From 7a8a476ef0a9cd4f30fd08f1e77145db6dafe18a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Tailland?= Date: Tue, 25 Aug 2026 18:11:53 +0200 Subject: [PATCH] Show parsed log tables instead of raw lines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Raw lines are close to unreadable when the payload is JSON, and the Kong audit records are the worst case: a wall of braces where the interesting fields are status, route and latency. Each dashboard now leads with a table carrying one column per parsed field, sortable and filterable, with the unparsed record as the last column and the cell inspector enabled on every cell. The original logs panel moves into a collapsed row underneath rather than disappearing: a table cannot render ANSI colour, expand a record inline, tail live, or turn a trace id into a link to Tempo. Parsing is per-format since the five sources agree on nothing — nested JSON for Kong, JSON for Keycloak, NCSA access lines for the Django API, ANSI-coloured text for LibreChat, logfmt for the stack itself. The Kong and stack parsers name their fields explicitly rather than extracting everything: a bare `| json` on a Kong record yields around seventy columns, and a bare `| logfmt` explodes Loki's own query-metrics lines into dozens. Columns are selected by include-pattern rather than by excluding names, because the stream labels that ride along are not a fixed set. Co-Authored-By: Claude Opus 5 --- README.md | 31 ++ .../dashboards/obs-classic-models-api.json | 291 +++++++++++++++-- k8s/grafana/dashboards/obs-keycloak.json | 261 +++++++++++++-- k8s/grafana/dashboards/obs-kong-gateways.json | 305 ++++++++++++++++-- k8s/grafana/dashboards/obs-librechat.json | 233 +++++++++++-- k8s/grafana/dashboards/obs-stack-health.json | 262 ++++++++++++++- 6 files changed, 1295 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index 17b6bb6..7cec55e 100644 --- a/README.md +++ b/README.md @@ -206,6 +206,37 @@ Each application dashboard carries the same four sections: an overview of RED indicators, HTTP traffic, a runtime section specific to the stack in use, and a **Logs** row, followed by container resources. +### Parsed log tables + +Every dashboard shows its logs twice: a **table** with one column per parsed +field, and, in a collapsed row underneath, the original **logs panel**. + +The table is what makes a JSON record readable — status, method, route, latency +as sortable, filterable columns rather than a wall of braces. Its last column +holds the unparsed record, and every cell has the inspector enabled, so the raw +line is one click away. + +The logs panel stays because a table cannot do what it does: render ANSI colour, +expand a record inline, tail live, and turn a `trace_id` into a link to Tempo. +It is collapsed so it costs nothing until needed. + +Parsing is per-format, because these five sources agree on nothing: + +| Source | Format | Parser | +|---|---|---| +| Kong audit | nested JSON | `\| json` with explicit field paths | +| Keycloak | JSON | `\| json` | +| classic-models-api | NCSA access lines | `\| pattern` | +| LibreChat | ANSI-coloured text | `line_format` to strip ANSI, then `\| pattern` | +| Observability stack | logfmt | `\| logfmt` with named fields | + +Two choices worth knowing. The Kong and stack parsers name their fields +explicitly rather than extracting everything: a bare `\| json` on a Kong record +yields about seventy columns, and a bare `\| logfmt` explodes Loki's own +query-metrics lines. And `\| pattern` keeps lines it cannot match, with empty +columns, rather than dropping them — so the Django table never silently hides +an application log among the access lines. + The Logs row leans on Loki's `detected_level`, which is inferred at ingestion time, so a single query works across all three log formats — Keycloak emits JSON, LibreChat colour-coded text, and the Django API plain gunicorn access lines. No diff --git a/k8s/grafana/dashboards/obs-classic-models-api.json b/k8s/grafana/dashboards/obs-classic-models-api.json index b076226..65f0a2c 100644 --- a/k8s/grafana/dashboards/obs-classic-models-api.json +++ b/k8s/grafana/dashboards/obs-classic-models-api.json @@ -1384,9 +1384,9 @@ "description": "Namespace-wide so the datastore's failures show up next to the application's." }, { - "id": 20, - "type": "logs", - "title": "Recent errors and warnings", + "id": 25, + "type": "table", + "title": "Access log", "datasource": { "type": "loki", "uid": "loki" @@ -1404,21 +1404,274 @@ "type": "loki", "uid": "loki" }, - "expr": "{namespace=\"classic-models\"} | detected_level =~ \"error|critical|warn\"", - "queryType": "range" + "expr": "{namespace=\"classic-models\", container=\"classic-models-api\"} | pattern ` - - [] \" \" <_> \"\"`", + "queryType": "range", + "maxLines": 500 } ], - "options": { - "showTime": true, - "showLabels": false, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": false, - "sortOrder": "Descending", - "dedupStrategy": "none", - "enableLogDetails": true + "transformations": [ + { + "id": "extractFields", + "options": { + "source": "labels", + "replace": false + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "^(timestamp|body|status|method|uri|size|client|agent)$" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Time": 0, + "status": 1, + "method": 2, + "uri": 3, + "size": 4, + "client": 5, + "agent": 6, + "body": 7 + }, + "renameByName": { + "status": "Status", + "method": "Method", + "uri": "URI", + "size": "Bytes", + "client": "Client", + "agent": "User agent", + "body": "Raw record", + "timestamp": "Time" + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": true, + "filterable": true + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + }, + { + "id": "mappings", + "value": [ + { + "type": "regex", + "options": { + "pattern": "^2..$", + "result": { + "color": "green", + "index": 0 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^3..$", + "result": { + "color": "blue", + "index": 1 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^4..$", + "result": { + "color": "orange", + "index": 2 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^5..$", + "result": { + "color": "red", + "index": 3 + } + } + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Method" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "URI" + }, + "properties": [ + { + "id": "custom.width", + "value": 300 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Bytes" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Client" + }, + "properties": [ + { + "id": "custom.width", + "value": 130 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "User agent" + }, + "properties": [ + { + "id": "custom.width", + "value": 200 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Raw record" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + } + ] }, - "description": "Click a line to expand it. Where the line carries a trace_id, the Loki datasource's derived field opens the matching trace in Tempo." + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + }, + "sortBy": [ + { + "displayName": "Time", + "desc": true + } + ] + }, + "description": "gunicorn writes NCSA access lines, not JSON, so these are split by pattern. Lines that do not match — application JSON logs — are kept with empty columns rather than dropped." + }, + { + "id": 26, + "type": "row", + "title": "Raw log lines", + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 58 + }, + "panels": [ + { + "id": 20, + "type": "logs", + "title": "Raw log lines", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 59 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{namespace=\"classic-models\"} | detected_level =~ \"error|critical|warn\"", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "sortOrder": "Descending", + "dedupStrategy": "none", + "enableLogDetails": true + }, + "description": "Kept alongside the table: the logs panel renders ANSI colour, expands a record's detail inline, tails live, and turns a trace id into a link to Tempo — none of which a table can do. Collapsed so it stays out of the way until needed." + } + ] }, { "id": 21, @@ -1429,7 +1682,7 @@ "h": 1, "w": 24, "x": 0, - "y": 58 + "y": 59 }, "panels": [] }, @@ -1445,7 +1698,7 @@ "h": 7, "w": 8, "x": 0, - "y": 59 + "y": 60 }, "targets": [ { @@ -1511,7 +1764,7 @@ "h": 7, "w": 8, "x": 8, - "y": 59 + "y": 60 }, "targets": [ { @@ -1576,7 +1829,7 @@ "h": 7, "w": 8, "x": 16, - "y": 59 + "y": 60 }, "targets": [ { diff --git a/k8s/grafana/dashboards/obs-keycloak.json b/k8s/grafana/dashboards/obs-keycloak.json index 81c6f4c..877cc79 100644 --- a/k8s/grafana/dashboards/obs-keycloak.json +++ b/k8s/grafana/dashboards/obs-keycloak.json @@ -1387,9 +1387,9 @@ "description": "Namespace-wide so the datastore's failures show up next to the application's." }, { - "id": 21, - "type": "logs", - "title": "Recent errors and warnings", + "id": 26, + "type": "table", + "title": "Errors and warnings", "datasource": { "type": "loki", "uid": "loki" @@ -1407,21 +1407,244 @@ "type": "loki", "uid": "loki" }, - "expr": "{namespace=\"keycloak\"} | detected_level =~ \"error|critical|warn\"", - "queryType": "range" + "expr": "{namespace=\"keycloak\", container=\"keycloak\"} | json | detected_level =~ \"error|critical|warn\"", + "queryType": "range", + "maxLines": 500 } ], - "options": { - "showTime": true, - "showLabels": false, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": false, - "sortOrder": "Descending", - "dedupStrategy": "none", - "enableLogDetails": true + "transformations": [ + { + "id": "extractFields", + "options": { + "source": "labels", + "replace": false + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "^(timestamp|body|level|loggerName|message|threadName)$" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Time": 0, + "level": 1, + "loggerName": 2, + "message": 3, + "threadName": 4, + "body": 5 + }, + "renameByName": { + "level": "Level", + "loggerName": "Logger", + "message": "Message", + "threadName": "Thread", + "body": "Raw record", + "timestamp": "Time" + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": true, + "filterable": true + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Level" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + }, + { + "id": "mappings", + "value": [ + { + "type": "regex", + "options": { + "pattern": "(?i)^(fatal|critical)$", + "result": { + "color": "dark-red", + "index": 0 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "(?i)^error$", + "result": { + "color": "red", + "index": 1 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "(?i)^warn(ing)?$", + "result": { + "color": "orange", + "index": 2 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "(?i)^info$", + "result": { + "color": "green", + "index": 3 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "(?i)^debug$", + "result": { + "color": "blue", + "index": 4 + } + } + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Logger" + }, + "properties": [ + { + "id": "custom.width", + "value": 280 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Thread" + }, + "properties": [ + { + "id": "custom.width", + "value": 160 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Raw record" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + } + ] }, - "description": "Click a line to expand it. Where the line carries a trace_id, the Loki datasource's derived field opens the matching trace in Tempo." + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + }, + "sortBy": [ + { + "displayName": "Time", + "desc": true + } + ] + }, + "description": "Keycloak logs JSON, so the fields come straight from `| json`. Scoped to the Keycloak container: PostgreSQL in the same namespace is not JSON and would parse to empty columns." + }, + { + "id": 27, + "type": "row", + "title": "Raw log lines", + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 60 + }, + "panels": [ + { + "id": 21, + "type": "logs", + "title": "Raw log lines", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 61 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{namespace=\"keycloak\"} | detected_level =~ \"error|critical|warn\"", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "sortOrder": "Descending", + "dedupStrategy": "none", + "enableLogDetails": true + }, + "description": "Kept alongside the table: the logs panel renders ANSI colour, expands a record's detail inline, tails live, and turns a trace id into a link to Tempo — none of which a table can do. Collapsed so it stays out of the way until needed." + } + ] }, { "id": 22, @@ -1432,7 +1655,7 @@ "h": 1, "w": 24, "x": 0, - "y": 60 + "y": 61 }, "panels": [] }, @@ -1448,7 +1671,7 @@ "h": 7, "w": 8, "x": 0, - "y": 61 + "y": 62 }, "targets": [ { @@ -1514,7 +1737,7 @@ "h": 7, "w": 8, "x": 8, - "y": 61 + "y": 62 }, "targets": [ { @@ -1579,7 +1802,7 @@ "h": 7, "w": 8, "x": 16, - "y": 61 + "y": 62 }, "targets": [ { diff --git a/k8s/grafana/dashboards/obs-kong-gateways.json b/k8s/grafana/dashboards/obs-kong-gateways.json index 00be19f..9d97d11 100644 --- a/k8s/grafana/dashboards/obs-kong-gateways.json +++ b/k8s/grafana/dashboards/obs-kong-gateways.json @@ -1189,9 +1189,9 @@ "description": "4xx and 5xx only, broken down by the Kong service that answered." }, { - "id": 16, - "type": "logs", - "title": "Recent audit records", + "id": 21, + "type": "table", + "title": "Audit records", "datasource": { "type": "loki", "uid": "loki" @@ -1209,21 +1209,288 @@ "type": "loki", "uid": "loki" }, - "expr": "{job=\"kong-audit\", domain=~\"${gateway:pipe}\"} | json", - "queryType": "range" + "expr": "{job=\"kong-audit\", domain=~\"${gateway:pipe}\"} | json status=\"response.status\", method=\"request.method\", uri=\"request.uri\", route=\"route.name\", svc=\"service.name\", latency=\"latencies.request\", client=\"client_ip\"", + "queryType": "range", + "maxLines": 500 } ], - "options": { - "showTime": true, - "showLabels": false, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": true, - "sortOrder": "Descending", - "dedupStrategy": "none", - "enableLogDetails": true + "transformations": [ + { + "id": "extractFields", + "options": { + "source": "labels", + "replace": false + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "^(timestamp|body|status|method|uri|route|svc|latency|client)$" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Time": 0, + "status": 1, + "method": 2, + "uri": 3, + "route": 4, + "svc": 5, + "latency": 6, + "client": 7, + "body": 8 + }, + "renameByName": { + "status": "Status", + "method": "Method", + "uri": "URI", + "route": "Route", + "svc": "Service", + "latency": "Latency (ms)", + "client": "Client", + "body": "Raw record", + "timestamp": "Time" + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": true, + "filterable": true + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + }, + { + "id": "mappings", + "value": [ + { + "type": "regex", + "options": { + "pattern": "^2..$", + "result": { + "color": "green", + "index": 0 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^3..$", + "result": { + "color": "blue", + "index": 1 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^4..$", + "result": { + "color": "orange", + "index": 2 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^5..$", + "result": { + "color": "red", + "index": 3 + } + } + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Method" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "URI" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Route" + }, + "properties": [ + { + "id": "custom.width", + "value": 220 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Service" + }, + "properties": [ + { + "id": "custom.width", + "value": 140 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Latency (ms)" + }, + "properties": [ + { + "id": "custom.width", + "value": 110 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Client" + }, + "properties": [ + { + "id": "custom.width", + "value": 130 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Raw record" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + } + ] }, - "description": "Full Kong record per request: route, status, latencies, client, and the captured request/response bodies. Where the record carries a trace id, the Loki datasource's derived field opens the matching trace in Tempo." + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + }, + "sortBy": [ + { + "displayName": "Time", + "desc": true + } + ] + }, + "description": "Each Kong record parsed into columns. The unparsed JSON stays in the last column — click any cell to inspect the full value." + }, + { + "id": 22, + "type": "row", + "title": "Raw log lines", + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 52 + }, + "panels": [ + { + "id": 16, + "type": "logs", + "title": "Raw log lines", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 53 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=\"kong-audit\", domain=~\"${gateway:pipe}\"} | json", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": true, + "sortOrder": "Descending", + "dedupStrategy": "none", + "enableLogDetails": true + }, + "description": "Kept alongside the table: the logs panel renders ANSI colour, expands a record's detail inline, tails live, and turns a trace id into a link to Tempo — none of which a table can do. Collapsed so it stays out of the way until needed." + } + ] }, { "id": 17, @@ -1234,7 +1501,7 @@ "h": 1, "w": 24, "x": 0, - "y": 52 + "y": 53 }, "panels": [] }, @@ -1250,7 +1517,7 @@ "h": 7, "w": 8, "x": 0, - "y": 53 + "y": 54 }, "targets": [ { @@ -1315,7 +1582,7 @@ "h": 7, "w": 8, "x": 8, - "y": 53 + "y": 54 }, "targets": [ { @@ -1380,7 +1647,7 @@ "h": 7, "w": 8, "x": 16, - "y": 53 + "y": 54 }, "targets": [ { diff --git a/k8s/grafana/dashboards/obs-librechat.json b/k8s/grafana/dashboards/obs-librechat.json index f67a4dc..d19d89a 100644 --- a/k8s/grafana/dashboards/obs-librechat.json +++ b/k8s/grafana/dashboards/obs-librechat.json @@ -1522,9 +1522,9 @@ "description": "Namespace-wide so the datastore's failures show up next to the application's." }, { - "id": 22, - "type": "logs", - "title": "Recent errors and warnings", + "id": 27, + "type": "table", + "title": "Errors and warnings", "datasource": { "type": "loki", "uid": "loki" @@ -1542,21 +1542,216 @@ "type": "loki", "uid": "loki" }, - "expr": "{namespace=\"librechat\"} | detected_level =~ \"error|critical|warn\"", - "queryType": "range" + "expr": "{namespace=\"librechat\", container=\"librechat\"} | detected_level =~ \"error|critical|warn\" | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `