From 7c416ce06f433ef8b0ba99e0d653029da6992447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Tailland?= Date: Tue, 25 Aug 2026 18:56:27 +0200 Subject: [PATCH] Fix column order and byte counts in the log tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three defects, all mine. Column order was ignored because organize's indexByName was keyed on the renamed display labels. Grafana keys it on the original field names, so the mapping matched nothing and the order fell back to whatever order the frames arrived in — which is why the raw line landed second. Every table is affected, not just the access one. The byte count was wrong for the NCSA sources: a trailing `pattern` capture takes the rest of the line, so `` held `16 "-" "kube-probe/1.35"` rather than 16, and for nginx it swallowed the whole referer and user-agent. Replaced with a `regexp` parser bounding the capture on \d+, which also handles both line shapes — Keycloak's access line ends at the size while gunicorn's and nginx's carry more fields after it. Access log columns are now time, status, method, path, source, client, latency, bytes, raw line, with the raw record last in every table. gunicorn and nginx now share one query since their format is identical, which also halves the duplicate class names the status panel showed — one series per query returning the same class. Co-Authored-By: Claude Opus 5 --- README.md | 11 ++ k8s/grafana/dashboards/obs-access-logs.json | 145 +++++++----------- .../dashboards/obs-application-logs.json | 12 +- k8s/grafana/dashboards/obs-audit-logs.json | 36 ++--- .../dashboards/obs-classic-models-api.json | 2 +- k8s/grafana/dashboards/obs-keycloak.json | 2 +- k8s/grafana/dashboards/obs-kong-gateways.json | 2 +- k8s/grafana/dashboards/obs-librechat.json | 2 +- k8s/grafana/dashboards/obs-stack-health.json | 2 +- 9 files changed, 93 insertions(+), 121 deletions(-) diff --git a/README.md b/README.md index b5eb5df..929aeb2 100644 --- a/README.md +++ b/README.md @@ -251,6 +251,17 @@ Latency is populated for Kong only: the NCSA format the other sources use carries no timing field. That is a property of the format, not a gap in the query. +The NCSA sources are read with `regexp`, not `pattern`. A trailing `pattern` +capture consumes the rest of the line, so a `` at the end swallowed the +referer and user-agent along with the byte count. Bounding the capture on `\d+` +stops it at the number, and `regexp` tolerates both shapes — Keycloak's access +line ends at the size, gunicorn's and nginx's carry more fields after it. + +In every table the raw record is the last column, and the `organize` +transformation's `indexByName` keys on the original field names rather than the +renamed labels — keying it on the display names silently leaves the column order +to whatever order the frames arrived in. + ### Parsed log tables### Parsed log tables Every dashboard shows its logs twice: a **table** with one column per parsed diff --git a/k8s/grafana/dashboards/obs-access-logs.json b/k8s/grafana/dashboards/obs-access-logs.json index 3f52a4b..23ad931 100644 --- a/k8s/grafana/dashboards/obs-access-logs.json +++ b/k8s/grafana/dashboards/obs-access-logs.json @@ -130,7 +130,7 @@ "type": "loki", "uid": "loki" }, - "expr": "sum(count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | status =~ \"4..\" [$__range])) or vector(0)", + "expr": "(sum(count_over_time({job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | status =~ \"4..\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)` | status =~ \"4..\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)` | status =~ \"4..\" [$__range])) or vector(0))", "queryType": "instant" } ], @@ -170,7 +170,7 @@ "graphMode": "none", "textMode": "auto" }, - "description": "Counted on the Kong records only, the one source that exposes a parsed status without ambiguity. The status column of the table covers the others." + "description": "Summed across all four sources, each parsed by its own format." }, { "id": 3, @@ -193,7 +193,7 @@ "type": "loki", "uid": "loki" }, - "expr": "sum(count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | status =~ \"5..\" [$__range])) or vector(0)", + "expr": "(sum(count_over_time({job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | status =~ \"5..\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)` | status =~ \"5..\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)` | status =~ \"5..\" [$__range])) or vector(0))", "queryType": "instant" } ], @@ -314,7 +314,7 @@ "type": "loki", "uid": "loki" }, - "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" [$__auto]))", + "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" [$__auto]))", "legendFormat": "{{job}}", "queryType": "range" }, @@ -324,7 +324,7 @@ "type": "loki", "uid": "loki" }, - "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern ` - - [<_>] \" <_>\" ` [$__auto]))", + "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)` [$__auto]))", "legendFormat": "{{job}}", "queryType": "range" }, @@ -334,17 +334,7 @@ "type": "loki", "uid": "loki" }, - "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" ` [$__auto]))", - "legendFormat": "{{job}}", - "queryType": "range" - }, - { - "refId": "D", - "datasource": { - "type": "loki", - "uid": "loki" - }, - "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" ` [$__auto]))", + "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)` [$__auto]))", "legendFormat": "{{job}}", "queryType": "range" } @@ -408,7 +398,7 @@ "type": "loki", "uid": "loki" }, - "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))", + "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))", "legendFormat": "{{class}}", "queryType": "range" }, @@ -418,7 +408,7 @@ "type": "loki", "uid": "loki" }, - "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern ` - - [<_>] \" <_>\" ` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))", + "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))", "legendFormat": "{{class}}", "queryType": "range" }, @@ -428,17 +418,7 @@ "type": "loki", "uid": "loki" }, - "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" ` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))", - "legendFormat": "{{class}}", - "queryType": "range" - }, - { - "refId": "D", - "datasource": { - "type": "loki", - "uid": "loki" - }, - "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" ` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))", + "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))", "legendFormat": "{{class}}", "queryType": "range" } @@ -540,7 +520,7 @@ "sort": "desc" } }, - "description": "Derived from each source's parsed status, so the four formats end up on the same scale." + "description": "Derived from each source's parsed status, so the formats land on one scale. A class can appear once per query in the legend; the stack itself is the correct total." }, { "id": 7, @@ -563,7 +543,7 @@ "type": "loki", "uid": "loki" }, - "expr": "{job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\"", + "expr": "{job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\"", "queryType": "range", "maxLines": 1000 }, @@ -573,7 +553,7 @@ "type": "loki", "uid": "loki" }, - "expr": "{job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern ` - - [<_>] \" <_>\" `", + "expr": "{job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)`", "queryType": "range", "maxLines": 1000 }, @@ -583,17 +563,7 @@ "type": "loki", "uid": "loki" }, - "expr": "{job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" `", - "queryType": "range", - "maxLines": 1000 - }, - { - "refId": "D", - "datasource": { - "type": "loki", - "uid": "loki" - }, - "expr": "{job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" `", + "expr": "{job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)`", "queryType": "range", "maxLines": 1000 } @@ -614,7 +584,7 @@ "id": "filterFieldsByName", "options": { "include": { - "pattern": "^(timestamp|body|job|method|path|status|client|latency|size)$" + "pattern": "^(timestamp|body|status|method|path|job|client|latency|size)$" } } }, @@ -623,23 +593,23 @@ "options": { "excludeByName": {}, "indexByName": { - "Time": 0, - "Source": 1, - "Method": 2, - "Path": 3, - "Status": 4, - "Client": 5, - "Latency (ms)": 6, - "Bytes": 7, - "Raw line": 8 + "timestamp": 0, + "status": 1, + "method": 2, + "path": 3, + "job": 4, + "client": 5, + "latency": 6, + "size": 7, + "body": 8 }, "renameByName": { "timestamp": "Time", "body": "Raw line", - "job": "Source", + "status": "Status", "method": "Method", "path": "Path", - "status": "Status", + "job": "Source", "client": "Client", "latency": "Latency (ms)", "size": "Bytes" @@ -660,30 +630,6 @@ "mappings": [] }, "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Source" - }, - "properties": [ - { - "id": "custom.width", - "value": 210 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Method" - }, - "properties": [ - { - "id": "custom.width", - "value": 80 - } - ] - }, { "matcher": { "id": "byName", @@ -747,6 +693,30 @@ } ] }, + { + "matcher": { + "id": "byName", + "options": "Method" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Source" + }, + "properties": [ + { + "id": "custom.width", + "value": 230 + } + ] + }, { "matcher": { "id": "byName", @@ -815,7 +785,7 @@ } ] }, - "description": "One row per request across all four sources. Latency is populated for Kong only — the NCSA format the others use carries no timing field at all, which is a property of the format, not a gap in the query." + "description": "One row per request across every source. Latency is populated for Kong only — the NCSA format the others use carries no timing field, which is a property of the format rather than a gap in the query. Raw line holds the record as emitted." }, { "id": 8, @@ -838,7 +808,7 @@ "type": "loki", "uid": "loki" }, - "expr": "{job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\"", + "expr": "{job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\"", "queryType": "range" }, { @@ -847,7 +817,7 @@ "type": "loki", "uid": "loki" }, - "expr": "{job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern ` - - [<_>] \" <_>\" `", + "expr": "{job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)`", "queryType": "range" }, { @@ -856,16 +826,7 @@ "type": "loki", "uid": "loki" }, - "expr": "{job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" `", - "queryType": "range" - }, - { - "refId": "D", - "datasource": { - "type": "loki", - "uid": "loki" - }, - "expr": "{job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" `", + "expr": "{job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)`", "queryType": "range" } ], diff --git a/k8s/grafana/dashboards/obs-application-logs.json b/k8s/grafana/dashboards/obs-application-logs.json index ce621e5..2d32f04 100644 --- a/k8s/grafana/dashboards/obs-application-logs.json +++ b/k8s/grafana/dashboards/obs-application-logs.json @@ -686,12 +686,12 @@ "options": { "excludeByName": {}, "indexByName": { - "Time": 0, - "Source": 1, - "Level": 2, - "Message": 3, - "Logger": 4, - "Raw line": 5 + "timestamp": 0, + "job": 1, + "level": 2, + "msg": 3, + "logger": 4, + "body": 5 }, "renameByName": { "timestamp": "Time", diff --git a/k8s/grafana/dashboards/obs-audit-logs.json b/k8s/grafana/dashboards/obs-audit-logs.json index 442aa0e..65ccbf6 100644 --- a/k8s/grafana/dashboards/obs-audit-logs.json +++ b/k8s/grafana/dashboards/obs-audit-logs.json @@ -482,15 +482,15 @@ "options": { "excludeByName": {}, "indexByName": { - "Time": 0, - "Event": 1, - "Realm": 2, - "Client": 3, - "User": 4, - "Source IP": 5, - "Error": 6, - "Grant": 7, - "Raw line": 8 + "timestamp": 0, + "type": 1, + "realmName": 2, + "clientId": 3, + "userId": 4, + "ipAddress": 5, + "error": 6, + "grant_type": 7, + "body": 8 }, "renameByName": { "timestamp": "Time", @@ -1127,15 +1127,15 @@ "options": { "excludeByName": {}, "indexByName": { - "Time": 0, - "Domain": 1, - "Service": 2, - "Route": 3, - "Method": 4, - "Path": 5, - "Status": 6, - "Caller IP": 7, - "Raw line": 8 + "timestamp": 0, + "domain": 1, + "svc": 2, + "route": 3, + "method": 4, + "path": 5, + "status": 6, + "client": 7, + "body": 8 }, "renameByName": { "timestamp": "Time", diff --git a/k8s/grafana/dashboards/obs-classic-models-api.json b/k8s/grafana/dashboards/obs-classic-models-api.json index 65f0a2c..01f2069 100644 --- a/k8s/grafana/dashboards/obs-classic-models-api.json +++ b/k8s/grafana/dashboards/obs-classic-models-api.json @@ -1430,7 +1430,7 @@ "options": { "excludeByName": {}, "indexByName": { - "Time": 0, + "timestamp": 0, "status": 1, "method": 2, "uri": 3, diff --git a/k8s/grafana/dashboards/obs-keycloak.json b/k8s/grafana/dashboards/obs-keycloak.json index d01c2e1..109e9d7 100644 --- a/k8s/grafana/dashboards/obs-keycloak.json +++ b/k8s/grafana/dashboards/obs-keycloak.json @@ -1433,7 +1433,7 @@ "options": { "excludeByName": {}, "indexByName": { - "Time": 0, + "timestamp": 0, "level": 1, "loggerName": 2, "message": 3, diff --git a/k8s/grafana/dashboards/obs-kong-gateways.json b/k8s/grafana/dashboards/obs-kong-gateways.json index 9d97d11..3cc2867 100644 --- a/k8s/grafana/dashboards/obs-kong-gateways.json +++ b/k8s/grafana/dashboards/obs-kong-gateways.json @@ -1235,7 +1235,7 @@ "options": { "excludeByName": {}, "indexByName": { - "Time": 0, + "timestamp": 0, "status": 1, "method": 2, "uri": 3, diff --git a/k8s/grafana/dashboards/obs-librechat.json b/k8s/grafana/dashboards/obs-librechat.json index 556543c..38d9bbc 100644 --- a/k8s/grafana/dashboards/obs-librechat.json +++ b/k8s/grafana/dashboards/obs-librechat.json @@ -1568,7 +1568,7 @@ "options": { "excludeByName": {}, "indexByName": { - "Time": 0, + "timestamp": 0, "lvl": 1, "msg": 2, "body": 3 diff --git a/k8s/grafana/dashboards/obs-stack-health.json b/k8s/grafana/dashboards/obs-stack-health.json index c17b4fd..60f498d 100644 --- a/k8s/grafana/dashboards/obs-stack-health.json +++ b/k8s/grafana/dashboards/obs-stack-health.json @@ -446,7 +446,7 @@ "options": { "excludeByName": {}, "indexByName": { - "Time": 0, + "timestamp": 0, "container": 1, "level": 2, "msg": 3,