diff --git a/README.md b/README.md index 17b6bb6..7cec55e 100644 --- a/README.md +++ b/README.md @@ -206,6 +206,37 @@ Each application dashboard carries the same four sections: an overview of RED indicators, HTTP traffic, a runtime section specific to the stack in use, and a **Logs** row, followed by container resources. +### Parsed log tables + +Every dashboard shows its logs twice: a **table** with one column per parsed +field, and, in a collapsed row underneath, the original **logs panel**. + +The table is what makes a JSON record readable — status, method, route, latency +as sortable, filterable columns rather than a wall of braces. Its last column +holds the unparsed record, and every cell has the inspector enabled, so the raw +line is one click away. + +The logs panel stays because a table cannot do what it does: render ANSI colour, +expand a record inline, tail live, and turn a `trace_id` into a link to Tempo. +It is collapsed so it costs nothing until needed. + +Parsing is per-format, because these five sources agree on nothing: + +| Source | Format | Parser | +|---|---|---| +| Kong audit | nested JSON | `\| json` with explicit field paths | +| Keycloak | JSON | `\| json` | +| classic-models-api | NCSA access lines | `\| pattern` | +| LibreChat | ANSI-coloured text | `line_format` to strip ANSI, then `\| pattern` | +| Observability stack | logfmt | `\| logfmt` with named fields | + +Two choices worth knowing. The Kong and stack parsers name their fields +explicitly rather than extracting everything: a bare `\| json` on a Kong record +yields about seventy columns, and a bare `\| logfmt` explodes Loki's own +query-metrics lines. And `\| pattern` keeps lines it cannot match, with empty +columns, rather than dropping them — so the Django table never silently hides +an application log among the access lines. + The Logs row leans on Loki's `detected_level`, which is inferred at ingestion time, so a single query works across all three log formats — Keycloak emits JSON, LibreChat colour-coded text, and the Django API plain gunicorn access lines. No diff --git a/k8s/grafana/dashboards/obs-classic-models-api.json b/k8s/grafana/dashboards/obs-classic-models-api.json index b076226..65f0a2c 100644 --- a/k8s/grafana/dashboards/obs-classic-models-api.json +++ b/k8s/grafana/dashboards/obs-classic-models-api.json @@ -1384,9 +1384,9 @@ "description": "Namespace-wide so the datastore's failures show up next to the application's." }, { - "id": 20, - "type": "logs", - "title": "Recent errors and warnings", + "id": 25, + "type": "table", + "title": "Access log", "datasource": { "type": "loki", "uid": "loki" @@ -1404,21 +1404,274 @@ "type": "loki", "uid": "loki" }, - "expr": "{namespace=\"classic-models\"} | detected_level =~ \"error|critical|warn\"", - "queryType": "range" + "expr": "{namespace=\"classic-models\", container=\"classic-models-api\"} | pattern ` - - [] \" \" <_> \"\"`", + "queryType": "range", + "maxLines": 500 } ], - "options": { - "showTime": true, - "showLabels": false, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": false, - "sortOrder": "Descending", - "dedupStrategy": "none", - "enableLogDetails": true + "transformations": [ + { + "id": "extractFields", + "options": { + "source": "labels", + "replace": false + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "^(timestamp|body|status|method|uri|size|client|agent)$" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Time": 0, + "status": 1, + "method": 2, + "uri": 3, + "size": 4, + "client": 5, + "agent": 6, + "body": 7 + }, + "renameByName": { + "status": "Status", + "method": "Method", + "uri": "URI", + "size": "Bytes", + "client": "Client", + "agent": "User agent", + "body": "Raw record", + "timestamp": "Time" + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": true, + "filterable": true + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + }, + { + "id": "mappings", + "value": [ + { + "type": "regex", + "options": { + "pattern": "^2..$", + "result": { + "color": "green", + "index": 0 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^3..$", + "result": { + "color": "blue", + "index": 1 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^4..$", + "result": { + "color": "orange", + "index": 2 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^5..$", + "result": { + "color": "red", + "index": 3 + } + } + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Method" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "URI" + }, + "properties": [ + { + "id": "custom.width", + "value": 300 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Bytes" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Client" + }, + "properties": [ + { + "id": "custom.width", + "value": 130 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "User agent" + }, + "properties": [ + { + "id": "custom.width", + "value": 200 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Raw record" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + } + ] }, - "description": "Click a line to expand it. Where the line carries a trace_id, the Loki datasource's derived field opens the matching trace in Tempo." + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + }, + "sortBy": [ + { + "displayName": "Time", + "desc": true + } + ] + }, + "description": "gunicorn writes NCSA access lines, not JSON, so these are split by pattern. Lines that do not match — application JSON logs — are kept with empty columns rather than dropped." + }, + { + "id": 26, + "type": "row", + "title": "Raw log lines", + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 58 + }, + "panels": [ + { + "id": 20, + "type": "logs", + "title": "Raw log lines", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 59 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{namespace=\"classic-models\"} | detected_level =~ \"error|critical|warn\"", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "sortOrder": "Descending", + "dedupStrategy": "none", + "enableLogDetails": true + }, + "description": "Kept alongside the table: the logs panel renders ANSI colour, expands a record's detail inline, tails live, and turns a trace id into a link to Tempo — none of which a table can do. Collapsed so it stays out of the way until needed." + } + ] }, { "id": 21, @@ -1429,7 +1682,7 @@ "h": 1, "w": 24, "x": 0, - "y": 58 + "y": 59 }, "panels": [] }, @@ -1445,7 +1698,7 @@ "h": 7, "w": 8, "x": 0, - "y": 59 + "y": 60 }, "targets": [ { @@ -1511,7 +1764,7 @@ "h": 7, "w": 8, "x": 8, - "y": 59 + "y": 60 }, "targets": [ { @@ -1576,7 +1829,7 @@ "h": 7, "w": 8, "x": 16, - "y": 59 + "y": 60 }, "targets": [ { diff --git a/k8s/grafana/dashboards/obs-keycloak.json b/k8s/grafana/dashboards/obs-keycloak.json index 81c6f4c..877cc79 100644 --- a/k8s/grafana/dashboards/obs-keycloak.json +++ b/k8s/grafana/dashboards/obs-keycloak.json @@ -1387,9 +1387,9 @@ "description": "Namespace-wide so the datastore's failures show up next to the application's." }, { - "id": 21, - "type": "logs", - "title": "Recent errors and warnings", + "id": 26, + "type": "table", + "title": "Errors and warnings", "datasource": { "type": "loki", "uid": "loki" @@ -1407,21 +1407,244 @@ "type": "loki", "uid": "loki" }, - "expr": "{namespace=\"keycloak\"} | detected_level =~ \"error|critical|warn\"", - "queryType": "range" + "expr": "{namespace=\"keycloak\", container=\"keycloak\"} | json | detected_level =~ \"error|critical|warn\"", + "queryType": "range", + "maxLines": 500 } ], - "options": { - "showTime": true, - "showLabels": false, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": false, - "sortOrder": "Descending", - "dedupStrategy": "none", - "enableLogDetails": true + "transformations": [ + { + "id": "extractFields", + "options": { + "source": "labels", + "replace": false + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "^(timestamp|body|level|loggerName|message|threadName)$" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Time": 0, + "level": 1, + "loggerName": 2, + "message": 3, + "threadName": 4, + "body": 5 + }, + "renameByName": { + "level": "Level", + "loggerName": "Logger", + "message": "Message", + "threadName": "Thread", + "body": "Raw record", + "timestamp": "Time" + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": true, + "filterable": true + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Level" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + }, + { + "id": "mappings", + "value": [ + { + "type": "regex", + "options": { + "pattern": "(?i)^(fatal|critical)$", + "result": { + "color": "dark-red", + "index": 0 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "(?i)^error$", + "result": { + "color": "red", + "index": 1 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "(?i)^warn(ing)?$", + "result": { + "color": "orange", + "index": 2 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "(?i)^info$", + "result": { + "color": "green", + "index": 3 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "(?i)^debug$", + "result": { + "color": "blue", + "index": 4 + } + } + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Logger" + }, + "properties": [ + { + "id": "custom.width", + "value": 280 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Thread" + }, + "properties": [ + { + "id": "custom.width", + "value": 160 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Raw record" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + } + ] }, - "description": "Click a line to expand it. Where the line carries a trace_id, the Loki datasource's derived field opens the matching trace in Tempo." + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + }, + "sortBy": [ + { + "displayName": "Time", + "desc": true + } + ] + }, + "description": "Keycloak logs JSON, so the fields come straight from `| json`. Scoped to the Keycloak container: PostgreSQL in the same namespace is not JSON and would parse to empty columns." + }, + { + "id": 27, + "type": "row", + "title": "Raw log lines", + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 60 + }, + "panels": [ + { + "id": 21, + "type": "logs", + "title": "Raw log lines", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 61 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{namespace=\"keycloak\"} | detected_level =~ \"error|critical|warn\"", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "sortOrder": "Descending", + "dedupStrategy": "none", + "enableLogDetails": true + }, + "description": "Kept alongside the table: the logs panel renders ANSI colour, expands a record's detail inline, tails live, and turns a trace id into a link to Tempo — none of which a table can do. Collapsed so it stays out of the way until needed." + } + ] }, { "id": 22, @@ -1432,7 +1655,7 @@ "h": 1, "w": 24, "x": 0, - "y": 60 + "y": 61 }, "panels": [] }, @@ -1448,7 +1671,7 @@ "h": 7, "w": 8, "x": 0, - "y": 61 + "y": 62 }, "targets": [ { @@ -1514,7 +1737,7 @@ "h": 7, "w": 8, "x": 8, - "y": 61 + "y": 62 }, "targets": [ { @@ -1579,7 +1802,7 @@ "h": 7, "w": 8, "x": 16, - "y": 61 + "y": 62 }, "targets": [ { diff --git a/k8s/grafana/dashboards/obs-kong-gateways.json b/k8s/grafana/dashboards/obs-kong-gateways.json index 00be19f..9d97d11 100644 --- a/k8s/grafana/dashboards/obs-kong-gateways.json +++ b/k8s/grafana/dashboards/obs-kong-gateways.json @@ -1189,9 +1189,9 @@ "description": "4xx and 5xx only, broken down by the Kong service that answered." }, { - "id": 16, - "type": "logs", - "title": "Recent audit records", + "id": 21, + "type": "table", + "title": "Audit records", "datasource": { "type": "loki", "uid": "loki" @@ -1209,21 +1209,288 @@ "type": "loki", "uid": "loki" }, - "expr": "{job=\"kong-audit\", domain=~\"${gateway:pipe}\"} | json", - "queryType": "range" + "expr": "{job=\"kong-audit\", domain=~\"${gateway:pipe}\"} | json status=\"response.status\", method=\"request.method\", uri=\"request.uri\", route=\"route.name\", svc=\"service.name\", latency=\"latencies.request\", client=\"client_ip\"", + "queryType": "range", + "maxLines": 500 } ], - "options": { - "showTime": true, - "showLabels": false, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": true, - "sortOrder": "Descending", - "dedupStrategy": "none", - "enableLogDetails": true + "transformations": [ + { + "id": "extractFields", + "options": { + "source": "labels", + "replace": false + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "pattern": "^(timestamp|body|status|method|uri|route|svc|latency|client)$" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Time": 0, + "status": 1, + "method": 2, + "uri": 3, + "route": 4, + "svc": 5, + "latency": 6, + "client": 7, + "body": 8 + }, + "renameByName": { + "status": "Status", + "method": "Method", + "uri": "URI", + "route": "Route", + "svc": "Service", + "latency": "Latency (ms)", + "client": "Client", + "body": "Raw record", + "timestamp": "Time" + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": true, + "filterable": true + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + }, + { + "id": "mappings", + "value": [ + { + "type": "regex", + "options": { + "pattern": "^2..$", + "result": { + "color": "green", + "index": 0 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^3..$", + "result": { + "color": "blue", + "index": 1 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^4..$", + "result": { + "color": "orange", + "index": 2 + } + } + }, + { + "type": "regex", + "options": { + "pattern": "^5..$", + "result": { + "color": "red", + "index": 3 + } + } + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Method" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "URI" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Route" + }, + "properties": [ + { + "id": "custom.width", + "value": 220 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Service" + }, + "properties": [ + { + "id": "custom.width", + "value": 140 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Latency (ms)" + }, + "properties": [ + { + "id": "custom.width", + "value": 110 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Client" + }, + "properties": [ + { + "id": "custom.width", + "value": 130 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Raw record" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + } + ] }, - "description": "Full Kong record per request: route, status, latencies, client, and the captured request/response bodies. Where the record carries a trace id, the Loki datasource's derived field opens the matching trace in Tempo." + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + }, + "sortBy": [ + { + "displayName": "Time", + "desc": true + } + ] + }, + "description": "Each Kong record parsed into columns. The unparsed JSON stays in the last column — click any cell to inspect the full value." + }, + { + "id": 22, + "type": "row", + "title": "Raw log lines", + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 52 + }, + "panels": [ + { + "id": 16, + "type": "logs", + "title": "Raw log lines", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 53 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=\"kong-audit\", domain=~\"${gateway:pipe}\"} | json", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": true, + "sortOrder": "Descending", + "dedupStrategy": "none", + "enableLogDetails": true + }, + "description": "Kept alongside the table: the logs panel renders ANSI colour, expands a record's detail inline, tails live, and turns a trace id into a link to Tempo — none of which a table can do. Collapsed so it stays out of the way until needed." + } + ] }, { "id": 17, @@ -1234,7 +1501,7 @@ "h": 1, "w": 24, "x": 0, - "y": 52 + "y": 53 }, "panels": [] }, @@ -1250,7 +1517,7 @@ "h": 7, "w": 8, "x": 0, - "y": 53 + "y": 54 }, "targets": [ { @@ -1315,7 +1582,7 @@ "h": 7, "w": 8, "x": 8, - "y": 53 + "y": 54 }, "targets": [ { @@ -1380,7 +1647,7 @@ "h": 7, "w": 8, "x": 16, - "y": 53 + "y": 54 }, "targets": [ { diff --git a/k8s/grafana/dashboards/obs-librechat.json b/k8s/grafana/dashboards/obs-librechat.json index f67a4dc..d19d89a 100644 --- a/k8s/grafana/dashboards/obs-librechat.json +++ b/k8s/grafana/dashboards/obs-librechat.json @@ -1522,9 +1522,9 @@ "description": "Namespace-wide so the datastore's failures show up next to the application's." }, { - "id": 22, - "type": "logs", - "title": "Recent errors and warnings", + "id": 27, + "type": "table", + "title": "Errors and warnings", "datasource": { "type": "loki", "uid": "loki" @@ -1542,21 +1542,216 @@ "type": "loki", "uid": "loki" }, - "expr": "{namespace=\"librechat\"} | detected_level =~ \"error|critical|warn\"", - "queryType": "range" + "expr": "{namespace=\"librechat\", container=\"librechat\"} | detected_level =~ \"error|critical|warn\" | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `