diff --git a/README.md b/README.md index 7cec55e..1ae5cc5 100644 --- a/README.md +++ b/README.md @@ -206,6 +206,32 @@ Each application dashboard carries the same four sections: an overview of RED indicators, HTTP traffic, a runtime section specific to the stack in use, and a **Logs** row, followed by container resources. +### Consolidated log search + +`obs-log-search` searches every source at once, and is deliberately separate +from the per-application dashboards: those answer "how is this component +behaving", this one answers "where did that string come from". + +Four controls: a **scope** preset, a **source** picker chained to it, a +**level** filter, and a free-text **search** box. The search term is injected as +a case-insensitive line filter inside LogQL backticks — a raw string, so a quote +typed in the box cannot break the query, and an empty box yields `(?i)`, a valid +regex matching every line. That is what makes "no search" the natural default +rather than a special case. + +Scope is a curated preset list rather than a free namespace picker, because the +cluster carries 48 log sources and most are infrastructure noise. The presets +are job regexes matched against the `/` label Alloy sets — +except `kong-audit`, which Fluent Bit ships with no namespace at all, and which +is why a single naive selector cannot span both collection paths. + +Two things this dashboard deliberately does not do. It does not parse per +format: its Message column holds the line as emitted, since the five sources +agree on nothing and that is what the per-application dashboards are for. And +its level filter is honest about being partial — around 40% of lines are +`unknown` because access lines and Kong records carry no severity for Loki to +infer, so for those the text is the only thing worth searching. + ### Parsed log tables Every dashboard shows its logs twice: a **table** with one column per parsed diff --git a/k8s/grafana/dashboards/kustomization.yaml b/k8s/grafana/dashboards/kustomization.yaml index ae32b9b..ffe16a0 100644 --- a/k8s/grafana/dashboards/kustomization.yaml +++ b/k8s/grafana/dashboards/kustomization.yaml @@ -21,6 +21,7 @@ configMapGenerator: - obs-keycloak.json - obs-kong-gateways.json - obs-librechat.json + - obs-log-search.json - obs-stack-health.json # <<< dashboards <<< diff --git a/k8s/grafana/dashboards/obs-log-search.json b/k8s/grafana/dashboards/obs-log-search.json new file mode 100644 index 0000000..784308b --- /dev/null +++ b/k8s/grafana/dashboards/obs-log-search.json @@ -0,0 +1,786 @@ +{ + "uid": "obs-log-search", + "title": "Log Search", + "description": "Free-text search across every log source, scoped by preset, source and level. Complements the per-application dashboards rather than replacing them.", + "tags": [ + "observability", + "logs", + "search" + ], + "editable": false, + "schemaVersion": 39, + "version": 1, + "refresh": "", + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "templating": { + "list": [ + { + "name": "scope", + "label": "Scope", + "type": "custom", + "query": "Applications : keycloak/keycloak|librechat/librechat|classic-models/classic-models-api,Gateways : kong-audit|catalog/proxy|customers/proxy|payments/proxy|sales/proxy|workforce/proxy|llm/proxy,Applications and gateways : keycloak/keycloak|librechat/librechat|classic-models/classic-models-api|kong-audit|catalog/proxy|customers/proxy|payments/proxy|sales/proxy|workforce/proxy|llm/proxy,Observability platform : observability/.+,Everything : .+", + "current": { + "text": "Applications and gateways", + "value": "keycloak/keycloak|librechat/librechat|classic-models/classic-models-api|kong-audit|catalog/proxy|customers/proxy|payments/proxy|sales/proxy|workforce/proxy|llm/proxy" + }, + "includeAll": false, + "multi": false + }, + { + "name": "source", + "label": "Source", + "type": "query", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "query": { + "label": "job", + "stream": "{job=~\"$scope\"}", + "type": 1, + "refId": "source-variable" + }, + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".+", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + } + }, + { + "name": "level", + "label": "Level", + "type": "custom", + "query": "info,warn,error,critical,debug,unknown", + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + } + }, + { + "name": "search", + "label": "Search", + "type": "textbox", + "query": "", + "current": { + "text": "", + "value": "" + }, + "description": "Case-insensitive regex, matched against the whole line. Empty matches everything." + } + ] + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Matching lines", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(count_over_time({job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search` [$__range])) or vector(0)", + "queryType": "instant" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [], + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + }, + "description": "Over the dashboard's whole time range, not per interval. `or vector(0)` makes an empty result read as 0 rather than \"No data\" — on a search box, finding nothing is an answer, not a failure." + }, + { + "id": 2, + "type": "stat", + "title": "Sources matched", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "count(sum by (job) (count_over_time({job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search` [$__range]))) or vector(0)", + "queryType": "instant" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [], + "color": { + "mode": "fixed", + "fixedColor": "purple" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + }, + "description": "How many distinct containers the current filter touches." + }, + { + "id": 3, + "type": "stat", + "title": "Errors", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(count_over_time({job=~\"${source:regex}\"} |~ `(?i)$search` | detected_level =~ \"error|critical\" [$__range])) or vector(0)", + "queryType": "instant" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [], + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + }, + "description": "Deliberately ignores the Level selector, so it still answers \"is anything failing in here?\" while you are filtered on info." + }, + { + "id": 4, + "type": "stat", + "title": "Warnings", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(count_over_time({job=~\"${source:regex}\"} |~ `(?i)$search` | detected_level = \"warn\" [$__range])) or vector(0)", + "queryType": "instant" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [], + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "orange", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + }, + "description": "Same: independent of the Level selector." + }, + { + "id": 5, + "type": "timeseries", + "title": "Volume by source", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search` [$__auto]))", + "legendFormat": "{{job}}", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 80, + "showPoints": "never", + "barAlignment": 0, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Bars rather than lines: a line count per bucket is a run of discrete events, not a continuous signal." + }, + { + "id": 6, + "type": "timeseries", + "title": "Volume by level", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (detected_level) (count_over_time({job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search` [$__auto]))", + "legendFormat": "{{detected_level}}", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 80, + "showPoints": "never", + "barAlignment": 0, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "critical" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "dark-red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "info" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "debug" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unknown" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "text" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "`unknown` is expected to dominate: access lines and Kong records carry no severity of their own, so Loki has nothing to infer a level from. Search the text rather than the level for those." + }, + { + "id": 7, + "type": "table", + "title": "Results", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 12 + }, + "description": "One row per matching line, sorted newest first. Source and Level are columns you can sort and filter; Message holds the line as emitted. Click any cell to inspect the full value — these are deliberately not parsed per format here, that is what the per-application dashboards do.", + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search`", + "queryType": "range", + "maxLines": 1000 + } + ], + "transformations": [ + { + "id": "extractFields", + "options": { + "source": "labels", + "replace": false + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "^(timestamp|body|job|detected_level)$" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Time": 0, + "Source": 1, + "Level": 2, + "Message": 3 + }, + "renameByName": { + "timestamp": "Time", + "job": "Source", + "detected_level": "Level", + "body": "Message" + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": true, + "filterable": true + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Source" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Level" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + }, + { + "id": "mappings", + "value": [ + { + "type": "regex", + "options": { + "pattern": "(?i)^(fatal|critical)$", + "result": { + "color": "dark-red", + "index": 0 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "(?i)^error$", + "result": { + "color": "red", + "index": 1 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "(?i)^warn(ing)?$", + "result": { + "color": "orange", + "index": 2 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "(?i)^info$", + "result": { + "color": "green", + "index": 3 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "(?i)^debug$", + "result": { + "color": "blue", + "index": 4 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^unknown$", + "result": { + "color": "text", + "index": 5 + } + } + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + }, + "sortBy": [ + { + "displayName": "Time", + "desc": true + } + ] + } + }, + { + "id": 8, + "type": "logs", + "title": "Results as a log stream", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 25 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search`", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": true, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": true, + "sortOrder": "Descending", + "dedupStrategy": "none", + "enableLogDetails": true + }, + "description": "The same result set as the table above. Kept because it renders ANSI colour, expands a line's labels inline, tails live, and turns a trace id into a link to Tempo — none of which a table can do." + } + ] +}