diff --git a/Makefile b/Makefile index 2ac2a3a..c423d60 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,8 @@ help: ## Show this help .PHONY: deploy deploy: check-context secret ## Deploy the whole stack (runtimes + dashboards) - $(KUBECTL) apply -k $(K8S) + # --server-side for the same reason as the dashboards target below. + $(KUBECTL) apply -k $(K8S) --server-side --force-conflicts $(MAKE) --no-print-directory wait .PHONY: deploy-runtime @@ -89,7 +90,11 @@ secret: ## Generate k8s/secret.yaml (random admin password) if absent .PHONY: dashboards dashboards: check-context dashboards-check ## Apply dashboards only (no Grafana restart) - $(KUBECTL) apply -k $(DASHBOARDS) + # Server-side apply: client-side apply stores a full copy of the object in the + # kubectl.kubernetes.io/last-applied-configuration annotation, which is capped at + # 256KB. The dashboards passed that in aggregate. Server-side apply tracks field + # ownership in metadata instead, so the ceiling becomes etcd's ~1MB per object. + $(KUBECTL) apply -k $(DASHBOARDS) --server-side --force-conflicts printf "$(C_OK)grafana-dashboards ConfigMap updated$(C_OFF) -- visible in Grafana within ~70s\n" printf " (kubelet volume sync ~60s + provisioner poll 10s, no restart).\n" printf " For immediate effect: $(C_WARN)make restart-grafana$(C_OFF)\n" @@ -310,7 +315,7 @@ _flush: $(KUBECTL) -n $(NS) scale deployment/$(COMP) --replicas=0 $(KUBECTL) -n $(NS) wait --for=delete pod -l app=$(COMP) --timeout=120s 2>/dev/null || true $(KUBECTL) -n $(NS) delete pvc $(COMP)-data --ignore-not-found --wait=true - $(KUBECTL) apply -k $(K8S)/$(COMP) + $(KUBECTL) apply -k $(K8S)/$(COMP) --server-side --force-conflicts $(KUBECTL) -n $(NS) scale deployment/$(COMP) --replicas=1 ##@ Validation and teardown @@ -326,7 +331,7 @@ diff: check-context ## Diff the manifests against the cluster state .PHONY: validate validate: dashboards-check ## Validate the manifests (build + server dry-run) kustomize build $(K8S) >/dev/null && printf "$(C_OK)kustomize build OK$(C_OFF)\n" - $(KUBECTL) apply -k $(K8S) --dry-run=server >/dev/null && printf "$(C_OK)server dry-run OK$(C_OFF)\n" + $(KUBECTL) apply -k $(K8S) --server-side --force-conflicts --dry-run=server >/dev/null && printf "$(C_OK)server dry-run OK$(C_OFF)\n" .PHONY: undeploy undeploy: ## DESTRUCTIVE: delete the stack, PVCs kept (CONFIRM=yes) diff --git a/README.md b/README.md index 1ae5cc5..b5eb5df 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,13 @@ forces a pod restart when they change. | `tempo-data` | Tempo | 10Gi | WAL + trace blocks | 168 h | | `grafana-data` | Grafana | 5Gi | SQLite database (users, prefs) | — | +> **Applies are server-side.** Client-side `kubectl apply` stores a full copy of +> each object in the `kubectl.kubernetes.io/last-applied-configuration` +> annotation, capped at 256KB; the dashboard ConfigMap passed that in aggregate +> at eight dashboards. `--server-side` tracks field ownership in metadata +> instead, which moves the ceiling to etcd's ~1MB per object. Splitting into one +> ConfigMap per dashboard is the next step if that is ever reached. + StorageClass `local-path` (the cluster default), `ReadWriteOnce`. The Deployments use `strategy: Recreate`: with an RWO volume the old pod must release it before the new one can start. @@ -206,33 +213,45 @@ Each application dashboard carries the same four sections: an overview of RED indicators, HTTP traffic, a runtime section specific to the stack in use, and a **Logs** row, followed by container resources. -### Consolidated log search +### Log dashboards, split by kind -`obs-log-search` searches every source at once, and is deliberately separate -from the per-application dashboards: those answer "how is this component -behaving", this one answers "where did that string come from". +Logs are searched from three dashboards rather than one, because the three +questions have nothing in common and a single view compromised on all of them. -Four controls: a **scope** preset, a **source** picker chained to it, a -**level** filter, and a free-text **search** box. The search term is injected as -a case-insensitive line filter inside LogQL backticks — a raw string, so a quote -typed in the box cannot break the query, and an empty box yields `(?i)`, a valid -regex matching every line. That is what makes "no search" the natural default -rather than a special case. +| Dashboard | Question | Sources | +|---|---|---| +| `obs-access-logs` | Which HTTP requests came in? | Kong records, Keycloak access logger, Django gunicorn, ingress-nginx | +| `obs-application-logs` | What is the code reporting, and what broke? | Keycloak, LibreChat, Django, the Kong dataplanes' nginx error log | +| `obs-audit-logs` | Who authenticated, what changed, who called what? | Keycloak `org.keycloak.events`, Kong http-log records | -Scope is a curated preset list rather than a free namespace picker, because the -cluster carries 48 log sources and most are infrastructure noise. The presets -are job regexes matched against the `/` label Alloy sets — -except `kong-audit`, which Fluent Bit ships with no namespace at all, and which -is why a single naive selector cannot span both collection paths. +Each source keeps its own parser and they all emit the same field names, so the +frames merge into one table with consistent columns. Access logs converge on +method / path / status / client; application logs on level / message. -Two things this dashboard deliberately does not do. It does not parse per -format: its Message column holds the line as emitted, since the five sources -agree on nothing and that is what the per-application dashboards are for. And -its level filter is honest about being partial — around 40% of lines are -`unknown` because access lines and Kong records carry no severity for Loki to -infer, so for those the text is the only thing worth searching. +Three things this split fixes that a single consolidated view could not: -### Parsed log tables +**The kinds are subtracted, not just filtered.** Application logs exclude +Keycloak's `access-log` and `events` loggers by name, the Django container's +NCSA lines by shape, and the Kong containers' stray access lines by requiring +the nginx date prefix. Without that, access lines drown the signal. + +**Levels are read from the line, not inferred.** Loki's `detected_level` marks +78% of the Kong gateway output `unknown` even though every one of those lines +states `[notice]` or `[warn]`. Each parser extracts the level and lowercases it, +so nginx's `notice`, Keycloak's `INFO` and LibreChat's `info` land on one scale +— and the level selector has no `unknown` entry, because a line reaching the +table always states one. + +**The scope selector actually scopes.** Each query carries both +`job=~"${source:regex}"` and a literal job matcher. With a single selector, +choosing "All" widened the query back to every stream in the cluster — which is +how MongoDB and cert-manager ended up in an application view. + +Latency is populated for Kong only: the NCSA format the other sources use +carries no timing field. That is a property of the format, not a gap in the +query. + +### Parsed log tables### Parsed log tables Every dashboard shows its logs twice: a **table** with one column per parsed field, and, in a collapsed row underneath, the original **logs panel**. diff --git a/k8s/grafana/dashboards/kustomization.yaml b/k8s/grafana/dashboards/kustomization.yaml index ffe16a0..c8f2e6f 100644 --- a/k8s/grafana/dashboards/kustomization.yaml +++ b/k8s/grafana/dashboards/kustomization.yaml @@ -17,11 +17,13 @@ configMapGenerator: - name: grafana-dashboards files: # >>> dashboards (generated by `make dashboards-sync`) >>> + - obs-access-logs.json + - obs-application-logs.json + - obs-audit-logs.json - obs-classic-models-api.json - obs-keycloak.json - obs-kong-gateways.json - obs-librechat.json - - obs-log-search.json - obs-stack-health.json # <<< dashboards <<< diff --git a/k8s/grafana/dashboards/obs-log-search.json b/k8s/grafana/dashboards/obs-access-logs.json similarity index 56% rename from k8s/grafana/dashboards/obs-log-search.json rename to k8s/grafana/dashboards/obs-access-logs.json index 784308b..3f52a4b 100644 --- a/k8s/grafana/dashboards/obs-log-search.json +++ b/k8s/grafana/dashboards/obs-access-logs.json @@ -1,11 +1,11 @@ { - "uid": "obs-log-search", - "title": "Log Search", - "description": "Free-text search across every log source, scoped by preset, source and level. Complements the per-application dashboards rather than replacing them.", + "uid": "obs-access-logs", + "title": "Access Logs", + "description": "One row per HTTP request, across the Kong gateways, Keycloak, the Classic Models API and the ingress controller. Each source is parsed by its own format and normalised to the same columns.", "tags": [ "observability", "logs", - "search" + "access" ], "editable": false, "schemaVersion": 39, @@ -18,54 +18,14 @@ "timezone": "browser", "templating": { "list": [ - { - "name": "scope", - "label": "Scope", - "type": "custom", - "query": "Applications : keycloak/keycloak|librechat/librechat|classic-models/classic-models-api,Gateways : kong-audit|catalog/proxy|customers/proxy|payments/proxy|sales/proxy|workforce/proxy|llm/proxy,Applications and gateways : keycloak/keycloak|librechat/librechat|classic-models/classic-models-api|kong-audit|catalog/proxy|customers/proxy|payments/proxy|sales/proxy|workforce/proxy|llm/proxy,Observability platform : observability/.+,Everything : .+", - "current": { - "text": "Applications and gateways", - "value": "keycloak/keycloak|librechat/librechat|classic-models/classic-models-api|kong-audit|catalog/proxy|customers/proxy|payments/proxy|sales/proxy|workforce/proxy|llm/proxy" - }, - "includeAll": false, - "multi": false - }, { "name": "source", "label": "Source", - "type": "query", - "datasource": { - "type": "loki", - "uid": "loki" - }, - "query": { - "label": "job", - "stream": "{job=~\"$scope\"}", - "type": 1, - "refId": "source-variable" - }, - "refresh": 2, - "sort": 1, - "multi": true, - "includeAll": true, - "allValue": ".+", - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - } - }, - { - "name": "level", - "label": "Level", "type": "custom", - "query": "info,warn,error,critical,debug,unknown", + "query": "Kong gateways : kong-audit,Keycloak : keycloak/keycloak,Classic Models API : classic-models/classic-models-api,Ingress NGINX : ingress-nginx/controller", "multi": true, "includeAll": true, - "allValue": ".*", + "allValue": "kong-audit|keycloak/keycloak|classic-models/classic-models-api|ingress-nginx/controller", "current": { "text": [ "All" @@ -84,7 +44,7 @@ "text": "", "value": "" }, - "description": "Case-insensitive regex, matched against the whole line. Empty matches everything." + "description": "Case-insensitive regex over the whole line. Empty matches everything." } ] }, @@ -92,7 +52,7 @@ { "id": 1, "type": "stat", - "title": "Matching lines", + "title": "Requests", "datasource": { "type": "loki", "uid": "loki" @@ -110,7 +70,7 @@ "type": "loki", "uid": "loki" }, - "expr": "sum(count_over_time({job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search` [$__range])) or vector(0)", + "expr": "sum(count_over_time({job=~\"${source:regex}\"} |~ `(?i)$search` [$__range])) or vector(0)", "queryType": "instant" } ], @@ -147,12 +107,12 @@ "graphMode": "none", "textMode": "auto" }, - "description": "Over the dashboard's whole time range, not per interval. `or vector(0)` makes an empty result read as 0 rather than \"No data\" — on a search box, finding nothing is an answer, not a failure." + "description": "Every line the selected sources emitted over the dashboard's time range." }, { "id": 2, "type": "stat", - "title": "Sources matched", + "title": "Client errors (4xx)", "datasource": { "type": "loki", "uid": "loki" @@ -170,130 +130,7 @@ "type": "loki", "uid": "loki" }, - "expr": "count(sum by (job) (count_over_time({job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search` [$__range]))) or vector(0)", - "queryType": "instant" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "mappings": [], - "color": { - "mode": "fixed", - "fixedColor": "purple" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto" - }, - "description": "How many distinct containers the current filter touches." - }, - { - "id": 3, - "type": "stat", - "title": "Errors", - "datasource": { - "type": "loki", - "uid": "loki" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "loki" - }, - "expr": "sum(count_over_time({job=~\"${source:regex}\"} |~ `(?i)$search` | detected_level =~ \"error|critical\" [$__range])) or vector(0)", - "queryType": "instant" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "mappings": [], - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto" - }, - "description": "Deliberately ignores the Level selector, so it still answers \"is anything failing in here?\" while you are filtered on info." - }, - { - "id": 4, - "type": "stat", - "title": "Warnings", - "datasource": { - "type": "loki", - "uid": "loki" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 0 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "loki" - }, - "expr": "sum(count_over_time({job=~\"${source:regex}\"} |~ `(?i)$search` | detected_level = \"warn\" [$__range])) or vector(0)", + "expr": "sum(count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | status =~ \"4..\" [$__range])) or vector(0)", "queryType": "instant" } ], @@ -333,12 +170,133 @@ "graphMode": "none", "textMode": "auto" }, - "description": "Same: independent of the Level selector." + "description": "Counted on the Kong records only, the one source that exposes a parsed status without ambiguity. The status column of the table covers the others." + }, + { + "id": 3, + "type": "stat", + "title": "Server errors (5xx)", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | status =~ \"5..\" [$__range])) or vector(0)", + "queryType": "instant" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [], + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 4, + "type": "stat", + "title": "Sources active", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "count(sum by (job) (count_over_time({job=~\"${source:regex}\"} |~ `(?i)$search` [$__range]))) or vector(0)", + "queryType": "instant" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [], + "color": { + "mode": "fixed", + "fixedColor": "purple" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + } }, { "id": 5, "type": "timeseries", - "title": "Volume by source", + "title": "Requests by source", "datasource": { "type": "loki", "uid": "loki" @@ -356,7 +314,37 @@ "type": "loki", "uid": "loki" }, - "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search` [$__auto]))", + "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" [$__auto]))", + "legendFormat": "{{job}}", + "queryType": "range" + }, + { + "refId": "B", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern ` - - [<_>] \" <_>\" ` [$__auto]))", + "legendFormat": "{{job}}", + "queryType": "range" + }, + { + "refId": "C", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" ` [$__auto]))", + "legendFormat": "{{job}}", + "queryType": "range" + }, + { + "refId": "D", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" ` [$__auto]))", "legendFormat": "{{job}}", "queryType": "range" } @@ -397,12 +385,12 @@ "sort": "desc" } }, - "description": "Bars rather than lines: a line count per bucket is a run of discrete events, not a continuous signal." + "description": "Bars rather than lines: a request count per bucket is a run of discrete events." }, { "id": 6, "type": "timeseries", - "title": "Volume by level", + "title": "Status classes", "datasource": { "type": "loki", "uid": "loki" @@ -420,8 +408,38 @@ "type": "loki", "uid": "loki" }, - "expr": "sum by (detected_level) (count_over_time({job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search` [$__auto]))", - "legendFormat": "{{detected_level}}", + "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))", + "legendFormat": "{{class}}", + "queryType": "range" + }, + { + "refId": "B", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern ` - - [<_>] \" <_>\" ` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))", + "legendFormat": "{{class}}", + "queryType": "range" + }, + { + "refId": "C", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" ` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))", + "legendFormat": "{{class}}", + "queryType": "range" + }, + { + "refId": "D", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" ` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))", + "legendFormat": "{{class}}", "queryType": "range" } ], @@ -449,52 +467,7 @@ { "matcher": { "id": "byName", - "options": "critical" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "dark-red" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "error" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "red" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "warn" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "orange" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "info" + "options": "2xx" }, "properties": [ { @@ -509,7 +482,7 @@ { "matcher": { "id": "byName", - "options": "debug" + "options": "3xx" }, "properties": [ { @@ -524,14 +497,29 @@ { "matcher": { "id": "byName", - "options": "unknown" + "options": "4xx" }, "properties": [ { "id": "color", "value": { "mode": "fixed", - "fixedColor": "text" + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" } } ] @@ -552,23 +540,22 @@ "sort": "desc" } }, - "description": "`unknown` is expected to dominate: access lines and Kong records carry no severity of their own, so Loki has nothing to infer a level from. Search the text rather than the level for those." + "description": "Derived from each source's parsed status, so the four formats end up on the same scale." }, { "id": 7, "type": "table", - "title": "Results", + "title": "Requests", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { - "h": 13, + "h": 14, "w": 24, "x": 0, "y": 12 }, - "description": "One row per matching line, sorted newest first. Source and Level are columns you can sort and filter; Message holds the line as emitted. Click any cell to inspect the full value — these are deliberately not parsed per format here, that is what the per-application dashboards do.", "targets": [ { "refId": "A", @@ -576,7 +563,37 @@ "type": "loki", "uid": "loki" }, - "expr": "{job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search`", + "expr": "{job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\"", + "queryType": "range", + "maxLines": 1000 + }, + { + "refId": "B", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern ` - - [<_>] \" <_>\" `", + "queryType": "range", + "maxLines": 1000 + }, + { + "refId": "C", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" `", + "queryType": "range", + "maxLines": 1000 + }, + { + "refId": "D", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" `", "queryType": "range", "maxLines": 1000 } @@ -589,11 +606,15 @@ "replace": false } }, + { + "id": "merge", + "options": {} + }, { "id": "filterFieldsByName", "options": { "include": { - "pattern": "^(timestamp|body|job|detected_level)$" + "pattern": "^(timestamp|body|job|method|path|status|client|latency|size)$" } } }, @@ -604,14 +625,24 @@ "indexByName": { "Time": 0, "Source": 1, - "Level": 2, - "Message": 3 + "Method": 2, + "Path": 3, + "Status": 4, + "Client": 5, + "Latency (ms)": 6, + "Bytes": 7, + "Raw line": 8 }, "renameByName": { "timestamp": "Time", + "body": "Raw line", "job": "Source", - "detected_level": "Level", - "body": "Message" + "method": "Method", + "path": "Path", + "status": "Status", + "client": "Client", + "latency": "Latency (ms)", + "size": "Bytes" } } } @@ -637,19 +668,31 @@ "properties": [ { "id": "custom.width", - "value": 260 + "value": 210 } ] }, { "matcher": { "id": "byName", - "options": "Level" + "options": "Method" }, "properties": [ { "id": "custom.width", - "value": 90 + "value": 80 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 }, { "id": "mappings", @@ -657,9 +700,9 @@ { "type": "regex", "options": { - "pattern": "(?i)^(fatal|critical)$", + "pattern": "^2..$", "result": { - "color": "dark-red", + "color": "green", "index": 0 } } @@ -667,9 +710,9 @@ { "type": "regex", "options": { - "pattern": "(?i)^error$", + "pattern": "^3..$", "result": { - "color": "red", + "color": "blue", "index": 1 } } @@ -677,7 +720,7 @@ { "type": "regex", "options": { - "pattern": "(?i)^warn(ing)?$", + "pattern": "^4..$", "result": { "color": "orange", "index": 2 @@ -687,32 +730,12 @@ { "type": "regex", "options": { - "pattern": "(?i)^info$", + "pattern": "^5..$", "result": { - "color": "green", + "color": "red", "index": 3 } } - }, - { - "type": "regex", - "options": { - "pattern": "(?i)^debug$", - "result": { - "color": "blue", - "index": 4 - } - } - }, - { - "type": "regex", - "options": { - "pattern": "^unknown$", - "result": { - "color": "text", - "index": 5 - } - } } ] }, @@ -723,6 +746,54 @@ } } ] + }, + { + "matcher": { + "id": "byName", + "options": "Client" + }, + "properties": [ + { + "id": "custom.width", + "value": 130 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Latency (ms)" + }, + "properties": [ + { + "id": "custom.width", + "value": 110 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Bytes" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Raw line" + }, + "properties": [ + { + "id": "custom.width", + "value": 240 + } + ] } ] }, @@ -743,21 +814,22 @@ "desc": true } ] - } + }, + "description": "One row per request across all four sources. Latency is populated for Kong only — the NCSA format the others use carries no timing field at all, which is a property of the format, not a gap in the query." }, { "id": 8, "type": "logs", - "title": "Results as a log stream", + "title": "Requests as a log stream", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { - "h": 12, + "h": 10, "w": 24, "x": 0, - "y": 25 + "y": 26 }, "targets": [ { @@ -766,13 +838,40 @@ "type": "loki", "uid": "loki" }, - "expr": "{job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search`", + "expr": "{job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\"", + "queryType": "range" + }, + { + "refId": "B", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern ` - - [<_>] \" <_>\" `", + "queryType": "range" + }, + { + "refId": "C", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" `", + "queryType": "range" + }, + { + "refId": "D", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" `", "queryType": "range" } ], "options": { "showTime": true, - "showLabels": true, + "showLabels": false, "showCommonLabels": false, "wrapLogMessage": true, "prettifyLogMessage": true, @@ -780,7 +879,7 @@ "dedupStrategy": "none", "enableLogDetails": true }, - "description": "The same result set as the table above. Kept because it renders ANSI colour, expands a line's labels inline, tails live, and turns a trace id into a link to Tempo — none of which a table can do." + "description": "Same result set. Kept for live tail and for the trace link the Loki datasource derives from a trace id, neither of which a table can do." } ] } diff --git a/k8s/grafana/dashboards/obs-application-logs.json b/k8s/grafana/dashboards/obs-application-logs.json new file mode 100644 index 0000000..ce621e5 --- /dev/null +++ b/k8s/grafana/dashboards/obs-application-logs.json @@ -0,0 +1,926 @@ +{ + "uid": "obs-application-logs", + "title": "Application Logs", + "description": "What the applications themselves report, with access lines and audit events subtracted. Levels are read from the line rather than inferred. Defaults to errors and warnings.", + "tags": [ + "observability", + "logs", + "application" + ], + "editable": false, + "schemaVersion": 39, + "version": 1, + "refresh": "", + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "templating": { + "list": [ + { + "name": "source", + "label": "Source", + "type": "custom", + "query": "Keycloak : keycloak/keycloak,LibreChat : librechat/librechat,Classic Models API : classic-models/classic-models-api,Kong gateways : (catalog|customers|payments|sales|workforce|llm)/proxy", + "multi": true, + "includeAll": true, + "allValue": "keycloak/keycloak|librechat/librechat|classic-models/classic-models-api|(catalog|customers|payments|sales|workforce|llm)/proxy", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + } + }, + { + "name": "level", + "label": "Level", + "type": "custom", + "query": "error,warn,notice,info,debug,crit", + "multi": true, + "includeAll": true, + "allValue": "error|warn|notice|info|debug|crit|critical|fatal|emerg|alert", + "current": { + "text": [ + "error", + "warn" + ], + "value": [ + "error", + "warn" + ] + } + }, + { + "name": "search", + "label": "Search", + "type": "textbox", + "query": "", + "current": { + "text": "", + "value": "" + }, + "description": "Case-insensitive regex over the whole line. Empty matches everything." + } + ] + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Errors", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "(sum(count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> : ` | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [] <_>: ` | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__range])) or vector(0))", + "queryType": "instant" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [], + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + }, + "description": "Independent of the Level selector, so it still answers \"is anything broken here?\" while you are filtered on something else." + }, + { + "id": 2, + "type": "stat", + "title": "Warnings", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "(sum(count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level = \"warn\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> : ` | label_format level=`{{ lower .level }}` | level = \"warn\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level = \"warn\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [] <_>: ` | label_format level=`{{ lower .level }}` | level = \"warn\" [$__range])) or vector(0))", + "queryType": "instant" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [], + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "orange", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 3, + "type": "stat", + "title": "Lines matching", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "(sum(count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> : ` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [] <_>: ` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__range])) or vector(0))", + "queryType": "instant" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [], + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 4, + "type": "stat", + "title": "Sources active", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "count(sum by (job) (count_over_time({job=~\"${source:regex}\"} |~ `(?i)$search` [$__range]))) or vector(0)", + "queryType": "instant" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [], + "color": { + "mode": "fixed", + "fixedColor": "purple" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Volume by level", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (level) (count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__auto]))", + "legendFormat": "{{level}}", + "queryType": "range" + }, + { + "refId": "B", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (level) (count_over_time({job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> : ` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__auto]))", + "legendFormat": "{{level}}", + "queryType": "range" + }, + { + "refId": "C", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (level) (count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__auto]))", + "legendFormat": "{{level}}", + "queryType": "range" + }, + { + "refId": "D", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (level) (count_over_time({job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [] <_>: ` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__auto]))", + "legendFormat": "{{level}}", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 80, + "showPoints": "never", + "barAlignment": 0, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "critical" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "dark-red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "info" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "debug" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unknown" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "notice" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "text" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Levels as each runtime states them, lowercased so nginx's `notice` and Keycloak's `INFO` land on the same scale." + }, + { + "id": 6, + "type": "timeseries", + "title": "Errors by source", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__auto]))", + "legendFormat": "{{job}}", + "queryType": "range" + }, + { + "refId": "B", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> : ` | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__auto]))", + "legendFormat": "{{job}}", + "queryType": "range" + }, + { + "refId": "C", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__auto]))", + "legendFormat": "{{job}}", + "queryType": "range" + }, + { + "refId": "D", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [] <_>: ` | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__auto]))", + "legendFormat": "{{job}}", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 80, + "showPoints": "never", + "barAlignment": 0, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Empty is the healthy state." + }, + { + "id": 7, + "type": "table", + "title": "Application log lines", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 12 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"", + "queryType": "range", + "maxLines": 1000 + }, + { + "refId": "B", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> : ` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"", + "queryType": "range", + "maxLines": 1000 + }, + { + "refId": "C", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"", + "queryType": "range", + "maxLines": 1000 + }, + { + "refId": "D", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [] <_>: ` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"", + "queryType": "range", + "maxLines": 1000 + } + ], + "transformations": [ + { + "id": "extractFields", + "options": { + "source": "labels", + "replace": false + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "^(timestamp|body|job|level|msg|logger)$" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Time": 0, + "Source": 1, + "Level": 2, + "Message": 3, + "Logger": 4, + "Raw line": 5 + }, + "renameByName": { + "timestamp": "Time", + "body": "Raw line", + "job": "Source", + "level": "Level", + "msg": "Message", + "logger": "Logger" + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": true, + "filterable": true + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Source" + }, + "properties": [ + { + "id": "custom.width", + "value": 210 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Level" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + }, + { + "id": "mappings", + "value": [ + { + "type": "regex", + "options": { + "pattern": "^(fatal|FATAL|critical|CRITICAL)$", + "result": { + "color": "dark-red", + "index": 0 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^(error|ERROR|Error)$", + "result": { + "color": "red", + "index": 1 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^(warn|WARN|warning|WARNING)$", + "result": { + "color": "orange", + "index": 2 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^(info|INFO|Info)$", + "result": { + "color": "green", + "index": 3 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^(debug|DEBUG|Debug|trace|TRACE)$", + "result": { + "color": "blue", + "index": 4 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^unknown$", + "result": { + "color": "text", + "index": 5 + } + } + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Logger" + }, + "properties": [ + { + "id": "custom.width", + "value": 240 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Raw line" + }, + "properties": [ + { + "id": "custom.width", + "value": 240 + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + }, + "sortBy": [ + { + "displayName": "Time", + "desc": true + } + ] + }, + "description": "Each runtime parsed by its own format — Keycloak JSON, LibreChat colour-coded text, Django JSON, Kong's nginx error log — normalised to level and message. Logger is populated for Keycloak only, the one source that names the emitting class." + }, + { + "id": 8, + "type": "logs", + "title": "Application logs as a stream", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 26 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"", + "queryType": "range" + }, + { + "refId": "B", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> : ` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"", + "queryType": "range" + }, + { + "refId": "C", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"", + "queryType": "range" + }, + { + "refId": "D", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [] <_>: ` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": true, + "sortOrder": "Descending", + "dedupStrategy": "none", + "enableLogDetails": true + }, + "description": "Same result set, with ANSI colour and inline detail expansion." + } + ] +} diff --git a/k8s/grafana/dashboards/obs-audit-logs.json b/k8s/grafana/dashboards/obs-audit-logs.json new file mode 100644 index 0000000..442aa0e --- /dev/null +++ b/k8s/grafana/dashboards/obs-audit-logs.json @@ -0,0 +1,1325 @@ +{ + "uid": "obs-audit-logs", + "title": "Audit Logs", + "description": "Who authenticated, what changed in the realm, and which caller reached which API with which payload. Keycloak identity events and Kong access records, kept apart because they answer different questions.", + "tags": [ + "observability", + "logs", + "audit", + "security" + ], + "editable": false, + "schemaVersion": 39, + "version": 1, + "refresh": "", + "time": { + "from": "now-24h", + "to": "now" + }, + "timezone": "browser", + "templating": { + "list": [ + { + "name": "domain", + "label": "Kong domain", + "type": "custom", + "query": "catalog,customers,payments,sales,workforce,llm", + "multi": true, + "includeAll": true, + "allValue": ".+", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "description": "Applies to the Kong section only." + }, + { + "name": "search", + "label": "Search", + "type": "textbox", + "query": "", + "current": { + "text": "", + "value": "" + }, + "description": "Case-insensitive regex over the whole line. Empty matches everything." + } + ] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Identity and administration events (Keycloak)", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Events", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(count_over_time({job=\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\", lvl=\"level\" | logger=\"org.keycloak.events\" | line_format `{{.m}}` | logfmt [$__range])) or vector(0)", + "queryType": "instant" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [], + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 3, + "type": "stat", + "title": "Failed logins", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(count_over_time({job=\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\", lvl=\"level\" | logger=\"org.keycloak.events\" | line_format `{{.m}}` | logfmt | type = \"LOGIN_ERROR\" [$__range])) or vector(0)", + "queryType": "instant" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [], + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + }, + "description": "A sustained non-zero value here is the signal a credential-stuffing attempt would leave." + }, + { + "id": 4, + "type": "stat", + "title": "Successful logins", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(count_over_time({job=\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\", lvl=\"level\" | logger=\"org.keycloak.events\" | line_format `{{.m}}` | logfmt | type = \"LOGIN\" [$__range])) or vector(0)", + "queryType": "instant" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [], + "color": { + "mode": "fixed", + "fixedColor": "green" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 5, + "type": "stat", + "title": "Distinct clients", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "count(sum by (clientId) (count_over_time({job=\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\", lvl=\"level\" | logger=\"org.keycloak.events\" | line_format `{{.m}}` | logfmt [$__range]))) or vector(0)", + "queryType": "instant" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [], + "color": { + "mode": "fixed", + "fixedColor": "purple" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Events by type", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (type) (count_over_time({job=\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\", lvl=\"level\" | logger=\"org.keycloak.events\" | line_format `{{.m}}` | logfmt [$__auto]))", + "legendFormat": "{{type}}", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 80, + "showPoints": "never", + "barAlignment": 0, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "LOGIN, LOGIN_ERROR, LOGOUT, REFRESH_TOKEN, CLIENT_LOGIN and the admin events Keycloak raises when a realm object changes." + }, + { + "id": 7, + "type": "timeseries", + "title": "Events by client", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (clientId) (count_over_time({job=\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\", lvl=\"level\" | logger=\"org.keycloak.events\" | line_format `{{.m}}` | logfmt [$__auto]))", + "legendFormat": "{{clientId}}", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 80, + "showPoints": "never", + "barAlignment": 0, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "type": "table", + "title": "Identity events", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 13 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\", lvl=\"level\" | logger=\"org.keycloak.events\" | line_format `{{.m}}` | logfmt", + "queryType": "range", + "maxLines": 1000 + } + ], + "transformations": [ + { + "id": "extractFields", + "options": { + "source": "labels", + "replace": false + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "^(timestamp|body|type|realmName|clientId|userId|ipAddress|error|grant_type)$" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Time": 0, + "Event": 1, + "Realm": 2, + "Client": 3, + "User": 4, + "Source IP": 5, + "Error": 6, + "Grant": 7, + "Raw line": 8 + }, + "renameByName": { + "timestamp": "Time", + "body": "Raw line", + "type": "Event", + "realmName": "Realm", + "clientId": "Client", + "userId": "User", + "ipAddress": "Source IP", + "error": "Error", + "grant_type": "Grant" + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": true, + "filterable": true + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Event" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Realm" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Client" + }, + "properties": [ + { + "id": "custom.width", + "value": 170 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "User" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Source IP" + }, + "properties": [ + { + "id": "custom.width", + "value": 130 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Error" + }, + "properties": [ + { + "id": "custom.width", + "value": 220 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Grant" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Raw line" + }, + "properties": [ + { + "id": "custom.width", + "value": 240 + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + }, + "sortBy": [ + { + "displayName": "Time", + "desc": true + } + ] + }, + "description": "Keycloak writes these as key=\"value\" pairs inside its JSON message field, so the line is reformatted to the message and then read as logfmt. A userId of \"null\" means the subject was never resolved — the usual shape of a failed authentication." + }, + { + "id": 9, + "type": "row", + "title": "API access records (Kong)", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "panels": [] + }, + { + "id": 10, + "type": "stat", + "title": "Records", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 27 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(count_over_time({job=\"kong-audit\", domain=~\"${domain:regex}\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", svc=\"service.name\", route=\"route.name\", authz=\"request.headers.authorization\" [$__range])) or vector(0)", + "queryType": "instant" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [], + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 11, + "type": "stat", + "title": "Rejected (401/403)", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 27 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(count_over_time({job=\"kong-audit\", domain=~\"${domain:regex}\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", svc=\"service.name\", route=\"route.name\", authz=\"request.headers.authorization\" | status =~ \"40[13]\" [$__range])) or vector(0)", + "queryType": "instant" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [], + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "orange", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 12, + "type": "stat", + "title": "Distinct callers", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 27 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "count(sum by (client) (count_over_time({job=\"kong-audit\", domain=~\"${domain:regex}\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", svc=\"service.name\", route=\"route.name\", authz=\"request.headers.authorization\" [$__range]))) or vector(0)", + "queryType": "instant" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [], + "color": { + "mode": "fixed", + "fixedColor": "purple" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 13, + "type": "stat", + "title": "Domains covered", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 27 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "count(sum by (domain) (count_over_time({job=\"kong-audit\", domain=~\"${domain:regex}\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", svc=\"service.name\", route=\"route.name\", authz=\"request.headers.authorization\" [$__range]))) or vector(0)", + "queryType": "instant" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [], + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 14, + "type": "timeseries", + "title": "Records by domain", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (domain) (count_over_time({job=\"kong-audit\", domain=~\"${domain:regex}\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", svc=\"service.name\", route=\"route.name\", authz=\"request.headers.authorization\" [$__auto]))", + "legendFormat": "{{domain}}", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 80, + "showPoints": "never", + "barAlignment": 0, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 15, + "type": "timeseries", + "title": "Outcome by status class", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (class) (count_over_time({job=\"kong-audit\", domain=~\"${domain:regex}\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", svc=\"service.name\", route=\"route.name\", authz=\"request.headers.authorization\" | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))", + "legendFormat": "{{class}}", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 80, + "showPoints": "never", + "barAlignment": 0, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 16, + "type": "table", + "title": "API access records", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 39 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=\"kong-audit\", domain=~\"${domain:regex}\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", svc=\"service.name\", route=\"route.name\", authz=\"request.headers.authorization\"", + "queryType": "range", + "maxLines": 1000 + } + ], + "transformations": [ + { + "id": "extractFields", + "options": { + "source": "labels", + "replace": false + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "^(timestamp|body|domain|svc|route|method|path|status|client)$" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Time": 0, + "Domain": 1, + "Service": 2, + "Route": 3, + "Method": 4, + "Path": 5, + "Status": 6, + "Caller IP": 7, + "Raw line": 8 + }, + "renameByName": { + "timestamp": "Time", + "body": "Raw line", + "domain": "Domain", + "svc": "Service", + "route": "Route", + "method": "Method", + "path": "Path", + "status": "Status", + "client": "Caller IP" + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": true, + "filterable": true + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Domain" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Service" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Route" + }, + "properties": [ + { + "id": "custom.width", + "value": 230 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Method" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + }, + { + "id": "mappings", + "value": [ + { + "type": "regex", + "options": { + "pattern": "^2..$", + "result": { + "color": "green", + "index": 0 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^3..$", + "result": { + "color": "blue", + "index": 1 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^4..$", + "result": { + "color": "orange", + "index": 2 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^5..$", + "result": { + "color": "red", + "index": 3 + } + } + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Caller IP" + }, + "properties": [ + { + "id": "custom.width", + "value": 130 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Raw line" + }, + "properties": [ + { + "id": "custom.width", + "value": 240 + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + }, + "sortBy": [ + { + "displayName": "Time", + "desc": true + } + ] + }, + "description": "The Raw line column holds the complete record, request and response bodies included — that is the evidentiary part, and the reason these are kept beyond what an access log needs. Note the captured Authorization header is a bearer token in clear text: acceptable for this lab, not for anything else." + } + ] +} diff --git a/k8s/grafana/dashboards/obs-keycloak.json b/k8s/grafana/dashboards/obs-keycloak.json index 877cc79..d01c2e1 100644 --- a/k8s/grafana/dashboards/obs-keycloak.json +++ b/k8s/grafana/dashboards/obs-keycloak.json @@ -1480,7 +1480,7 @@ { "type": "regex", "options": { - "pattern": "(?i)^(fatal|critical)$", + "pattern": "^(fatal|FATAL|Fatal|critical|CRITICAL|Critical)$", "result": { "color": "dark-red", "index": 0 @@ -1490,7 +1490,7 @@ { "type": "regex", "options": { - "pattern": "(?i)^error$", + "pattern": "^(error|ERROR|Error)$", "result": { "color": "red", "index": 1 @@ -1500,7 +1500,7 @@ { "type": "regex", "options": { - "pattern": "(?i)^warn(ing)?$", + "pattern": "^(warn|WARN|Warn|warning|WARNING|Warning)$", "result": { "color": "orange", "index": 2 @@ -1510,7 +1510,7 @@ { "type": "regex", "options": { - "pattern": "(?i)^info$", + "pattern": "^(info|INFO|Info)$", "result": { "color": "green", "index": 3 @@ -1520,7 +1520,7 @@ { "type": "regex", "options": { - "pattern": "(?i)^debug$", + "pattern": "^(debug|DEBUG|Debug)$", "result": { "color": "blue", "index": 4 diff --git a/k8s/grafana/dashboards/obs-librechat.json b/k8s/grafana/dashboards/obs-librechat.json index d19d89a..556543c 100644 --- a/k8s/grafana/dashboards/obs-librechat.json +++ b/k8s/grafana/dashboards/obs-librechat.json @@ -1611,7 +1611,7 @@ { "type": "regex", "options": { - "pattern": "(?i)^(fatal|critical)$", + "pattern": "^(fatal|FATAL|Fatal|critical|CRITICAL|Critical)$", "result": { "color": "dark-red", "index": 0 @@ -1621,7 +1621,7 @@ { "type": "regex", "options": { - "pattern": "(?i)^error$", + "pattern": "^(error|ERROR|Error)$", "result": { "color": "red", "index": 1 @@ -1631,7 +1631,7 @@ { "type": "regex", "options": { - "pattern": "(?i)^warn(ing)?$", + "pattern": "^(warn|WARN|Warn|warning|WARNING|Warning)$", "result": { "color": "orange", "index": 2 @@ -1641,7 +1641,7 @@ { "type": "regex", "options": { - "pattern": "(?i)^info$", + "pattern": "^(info|INFO|Info)$", "result": { "color": "green", "index": 3 @@ -1651,7 +1651,7 @@ { "type": "regex", "options": { - "pattern": "(?i)^debug$", + "pattern": "^(debug|DEBUG|Debug)$", "result": { "color": "blue", "index": 4 diff --git a/k8s/grafana/dashboards/obs-stack-health.json b/k8s/grafana/dashboards/obs-stack-health.json index 722baf1..c17b4fd 100644 --- a/k8s/grafana/dashboards/obs-stack-health.json +++ b/k8s/grafana/dashboards/obs-stack-health.json @@ -507,7 +507,7 @@ { "type": "regex", "options": { - "pattern": "(?i)^(fatal|critical)$", + "pattern": "^(fatal|FATAL|Fatal|critical|CRITICAL|Critical)$", "result": { "color": "dark-red", "index": 0 @@ -517,7 +517,7 @@ { "type": "regex", "options": { - "pattern": "(?i)^error$", + "pattern": "^(error|ERROR|Error)$", "result": { "color": "red", "index": 1 @@ -527,7 +527,7 @@ { "type": "regex", "options": { - "pattern": "(?i)^warn(ing)?$", + "pattern": "^(warn|WARN|Warn|warning|WARNING|Warning)$", "result": { "color": "orange", "index": 2 @@ -537,7 +537,7 @@ { "type": "regex", "options": { - "pattern": "(?i)^info$", + "pattern": "^(info|INFO|Info)$", "result": { "color": "green", "index": 3 @@ -547,7 +547,7 @@ { "type": "regex", "options": { - "pattern": "(?i)^debug$", + "pattern": "^(debug|DEBUG|Debug)$", "result": { "color": "blue", "index": 4