From 40ffac4acf6de1ee651b7d3f8850d52ba72e776b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Tailland?= Date: Tue, 25 Aug 2026 18:23:48 +0200 Subject: [PATCH] Add a consolidated log search dashboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-application dashboards answer "how is this component behaving". This one answers "where did that string come from", across every source at once, and is kept separate so neither has to compromise for the other. Scope is a curated preset rather than a free namespace picker: the cluster carries 48 log sources and most are infrastructure noise. The presets are job regexes over the / label Alloy sets — except kong-audit, which Fluent Bit ships with no namespace at all. That mismatch between the two collection paths is the reason a single naive selector cannot span both, and why the presets spell the sources out. The search term is injected inside LogQL backticks, a raw string, so a quote typed in the box cannot break the query; an empty box yields `(?i)`, a valid regex matching every line, which makes "no search" the default rather than a special case. The four stat panels carry `or vector(0)` so an empty result reads as 0 rather than "No data" — on a search box, finding nothing is an answer, not a failure. Two deliberate omissions. No per-format parsing: the Message column holds the line as emitted, because the five formats agree on nothing and that is what the per-application dashboards are for. And the level filter is partial by nature — roughly 40% of lines are `unknown`, since access lines and Kong records carry no severity for Loki to infer. Co-Authored-By: Claude Opus 5 --- README.md | 26 + k8s/grafana/dashboards/kustomization.yaml | 1 + k8s/grafana/dashboards/obs-log-search.json | 786 +++++++++++++++++++++ 3 files changed, 813 insertions(+) create mode 100644 k8s/grafana/dashboards/obs-log-search.json diff --git a/README.md b/README.md index 7cec55e..1ae5cc5 100644 --- a/README.md +++ b/README.md @@ -206,6 +206,32 @@ Each application dashboard carries the same four sections: an overview of RED indicators, HTTP traffic, a runtime section specific to the stack in use, and a **Logs** row, followed by container resources. +### Consolidated log search + +`obs-log-search` searches every source at once, and is deliberately separate +from the per-application dashboards: those answer "how is this component +behaving", this one answers "where did that string come from". + +Four controls: a **scope** preset, a **source** picker chained to it, a +**level** filter, and a free-text **search** box. The search term is injected as +a case-insensitive line filter inside LogQL backticks — a raw string, so a quote +typed in the box cannot break the query, and an empty box yields `(?i)`, a valid +regex matching every line. That is what makes "no search" the natural default +rather than a special case. + +Scope is a curated preset list rather than a free namespace picker, because the +cluster carries 48 log sources and most are infrastructure noise. The presets +are job regexes matched against the `/` label Alloy sets — +except `kong-audit`, which Fluent Bit ships with no namespace at all, and which +is why a single naive selector cannot span both collection paths. + +Two things this dashboard deliberately does not do. It does not parse per +format: its Message column holds the line as emitted, since the five sources +agree on nothing and that is what the per-application dashboards are for. And +its level filter is honest about being partial — around 40% of lines are +`unknown` because access lines and Kong records carry no severity for Loki to +infer, so for those the text is the only thing worth searching. + ### Parsed log tables Every dashboard shows its logs twice: a **table** with one column per parsed diff --git a/k8s/grafana/dashboards/kustomization.yaml b/k8s/grafana/dashboards/kustomization.yaml index ae32b9b..ffe16a0 100644 --- a/k8s/grafana/dashboards/kustomization.yaml +++ b/k8s/grafana/dashboards/kustomization.yaml @@ -21,6 +21,7 @@ configMapGenerator: - obs-keycloak.json - obs-kong-gateways.json - obs-librechat.json + - obs-log-search.json - obs-stack-health.json # <<< dashboards <<< diff --git a/k8s/grafana/dashboards/obs-log-search.json b/k8s/grafana/dashboards/obs-log-search.json new file mode 100644 index 0000000..784308b --- /dev/null +++ b/k8s/grafana/dashboards/obs-log-search.json @@ -0,0 +1,786 @@ +{ + "uid": "obs-log-search", + "title": "Log Search", + "description": "Free-text search across every log source, scoped by preset, source and level. Complements the per-application dashboards rather than replacing them.", + "tags": [ + "observability", + "logs", + "search" + ], + "editable": false, + "schemaVersion": 39, + "version": 1, + "refresh": "", + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "templating": { + "list": [ + { + "name": "scope", + "label": "Scope", + "type": "custom", + "query": "Applications : keycloak/keycloak|librechat/librechat|classic-models/classic-models-api,Gateways : kong-audit|catalog/proxy|customers/proxy|payments/proxy|sales/proxy|workforce/proxy|llm/proxy,Applications and gateways : keycloak/keycloak|librechat/librechat|classic-models/classic-models-api|kong-audit|catalog/proxy|customers/proxy|payments/proxy|sales/proxy|workforce/proxy|llm/proxy,Observability platform : observability/.+,Everything : .+", + "current": { + "text": "Applications and gateways", + "value": "keycloak/keycloak|librechat/librechat|classic-models/classic-models-api|kong-audit|catalog/proxy|customers/proxy|payments/proxy|sales/proxy|workforce/proxy|llm/proxy" + }, + "includeAll": false, + "multi": false + }, + { + "name": "source", + "label": "Source", + "type": "query", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "query": { + "label": "job", + "stream": "{job=~\"$scope\"}", + "type": 1, + "refId": "source-variable" + }, + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".+", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + } + }, + { + "name": "level", + "label": "Level", + "type": "custom", + "query": "info,warn,error,critical,debug,unknown", + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + } + }, + { + "name": "search", + "label": "Search", + "type": "textbox", + "query": "", + "current": { + "text": "", + "value": "" + }, + "description": "Case-insensitive regex, matched against the whole line. Empty matches everything." + } + ] + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Matching lines", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(count_over_time({job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search` [$__range])) or vector(0)", + "queryType": "instant" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [], + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + }, + "description": "Over the dashboard's whole time range, not per interval. `or vector(0)` makes an empty result read as 0 rather than \"No data\" — on a search box, finding nothing is an answer, not a failure." + }, + { + "id": 2, + "type": "stat", + "title": "Sources matched", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "count(sum by (job) (count_over_time({job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search` [$__range]))) or vector(0)", + "queryType": "instant" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [], + "color": { + "mode": "fixed", + "fixedColor": "purple" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + }, + "description": "How many distinct containers the current filter touches." + }, + { + "id": 3, + "type": "stat", + "title": "Errors", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(count_over_time({job=~\"${source:regex}\"} |~ `(?i)$search` | detected_level =~ \"error|critical\" [$__range])) or vector(0)", + "queryType": "instant" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [], + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + }, + "description": "Deliberately ignores the Level selector, so it still answers \"is anything failing in here?\" while you are filtered on info." + }, + { + "id": 4, + "type": "stat", + "title": "Warnings", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(count_over_time({job=~\"${source:regex}\"} |~ `(?i)$search` | detected_level = \"warn\" [$__range])) or vector(0)", + "queryType": "instant" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [], + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "orange", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + }, + "description": "Same: independent of the Level selector." + }, + { + "id": 5, + "type": "timeseries", + "title": "Volume by source", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search` [$__auto]))", + "legendFormat": "{{job}}", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 80, + "showPoints": "never", + "barAlignment": 0, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Bars rather than lines: a line count per bucket is a run of discrete events, not a continuous signal." + }, + { + "id": 6, + "type": "timeseries", + "title": "Volume by level", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (detected_level) (count_over_time({job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search` [$__auto]))", + "legendFormat": "{{detected_level}}", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 80, + "showPoints": "never", + "barAlignment": 0, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "critical" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "dark-red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "info" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "debug" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unknown" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "text" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "`unknown` is expected to dominate: access lines and Kong records carry no severity of their own, so Loki has nothing to infer a level from. Search the text rather than the level for those." + }, + { + "id": 7, + "type": "table", + "title": "Results", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 12 + }, + "description": "One row per matching line, sorted newest first. Source and Level are columns you can sort and filter; Message holds the line as emitted. Click any cell to inspect the full value — these are deliberately not parsed per format here, that is what the per-application dashboards do.", + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search`", + "queryType": "range", + "maxLines": 1000 + } + ], + "transformations": [ + { + "id": "extractFields", + "options": { + "source": "labels", + "replace": false + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "^(timestamp|body|job|detected_level)$" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Time": 0, + "Source": 1, + "Level": 2, + "Message": 3 + }, + "renameByName": { + "timestamp": "Time", + "job": "Source", + "detected_level": "Level", + "body": "Message" + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": true, + "filterable": true + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Source" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Level" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + }, + { + "id": "mappings", + "value": [ + { + "type": "regex", + "options": { + "pattern": "(?i)^(fatal|critical)$", + "result": { + "color": "dark-red", + "index": 0 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "(?i)^error$", + "result": { + "color": "red", + "index": 1 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "(?i)^warn(ing)?$", + "result": { + "color": "orange", + "index": 2 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "(?i)^info$", + "result": { + "color": "green", + "index": 3 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "(?i)^debug$", + "result": { + "color": "blue", + "index": 4 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^unknown$", + "result": { + "color": "text", + "index": 5 + } + } + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + }, + "sortBy": [ + { + "displayName": "Time", + "desc": true + } + ] + } + }, + { + "id": 8, + "type": "logs", + "title": "Results as a log stream", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 25 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search`", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": true, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": true, + "sortOrder": "Descending", + "dedupStrategy": "none", + "enableLogDetails": true + }, + "description": "The same result set as the table above. Kept because it renders ANSI colour, expands a line's labels inline, tails live, and turns a trace id into a link to Tempo — none of which a table can do." + } + ] +}