stt/infra-observability
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix column order and byte counts in the log tables

Browse Source 
Three defects, all mine.

Column order was ignored because organize's indexByName was keyed on the
renamed display labels. Grafana keys it on the original field names, so the
mapping matched nothing and the order fell back to whatever order the frames
arrived in — which is why the raw line landed second. Every table is affected,
not just the access one.

The byte count was wrong for the NCSA sources: a trailing `pattern` capture
takes the rest of the line, so `<size>` held `16 "-" "kube-probe/1.35"` rather
than 16, and for nginx it swallowed the whole referer and user-agent. Replaced
with a `regexp` parser bounding the capture on \d+, which also handles both
line shapes — Keycloak's access line ends at the size while gunicorn's and
nginx's carry more fields after it.

Access log columns are now time, status, method, path, source, client, latency,
bytes, raw line, with the raw record last in every table.

gunicorn and nginx now share one query since their format is identical, which
also halves the duplicate class names the status panel showed — one series per
query returning the same class.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Stéphane Tailland
2026-08-25 18:56:27 +02:00
parent eb35419ad0
commit 7c416ce06f
9 changed files with 93 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
k8s/grafana/dashboards/obs-access-logs.json
+53 -92
View File
@@ -130,7 +130,7 @@
"type": "loki",
"uid": "loki"
},
"expr": "sum(count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | status =~ \"4..\" [$__range])) or vector(0)",
"expr": "(sum(count_over_time({job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | status =~ \"4..\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)` | status =~ \"4..\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)` | status =~ \"4..\" [$__range])) or vector(0))",
"queryType": "instant"
}
],
@@ -170,7 +170,7 @@
"graphMode": "none",
"textMode": "auto"
},
"description": "Counted on the Kong records only, the one source that exposes a parsed status without ambiguity. The status column of the table covers the others."
"description": "Summed across all four sources, each parsed by its own format."
},
{
"id": 3,
@@ -193,7 +193,7 @@
"type": "loki",
"uid": "loki"
},
"expr": "sum(count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | status =~ \"5..\" [$__range])) or vector(0)",
"expr": "(sum(count_over_time({job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | status =~ \"5..\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)` | status =~ \"5..\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)` | status =~ \"5..\" [$__range])) or vector(0))",
"queryType": "instant"
}
],
@@ -314,7 +314,7 @@
"type": "loki",
"uid": "loki"
},
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" [$__auto]))",
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" [$__auto]))",
"legendFormat": "{{job}}",
"queryType": "range"
},
@@ -324,7 +324,7 @@
"type": "loki",
"uid": "loki"
},
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>` [$__auto]))",
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)` [$__auto]))",
"legendFormat": "{{job}}",
"queryType": "range"
},
@@ -334,17 +334,7 @@
"type": "loki",
"uid": "loki"
},
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>` [$__auto]))",
"legendFormat": "{{job}}",
"queryType": "range"
},
{
"refId": "D",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>` [$__auto]))",
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)` [$__auto]))",
"legendFormat": "{{job}}",
"queryType": "range"
}
@@ -408,7 +398,7 @@
"type": "loki",
"uid": "loki"
},
"expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))",
"expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))",
"legendFormat": "{{class}}",
"queryType": "range"
},
@@ -418,7 +408,7 @@
"type": "loki",
"uid": "loki"
},
"expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))",
"expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))",
"legendFormat": "{{class}}",
"queryType": "range"
},
@@ -428,17 +418,7 @@
"type": "loki",
"uid": "loki"
},
"expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))",
"legendFormat": "{{class}}",
"queryType": "range"
},
{
"refId": "D",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))",
"expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))",
"legendFormat": "{{class}}",
"queryType": "range"
}
@@ -540,7 +520,7 @@
"sort": "desc"
}
},
"description": "Derived from each source's parsed status, so the four formats end up on the same scale."
"description": "Derived from each source's parsed status, so the formats land on one scale. A class can appear once per query in the legend; the stack itself is the correct total."
},
{
"id": 7,
@@ -563,7 +543,7 @@
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\"",
"expr": "{job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\"",
"queryType": "range",
"maxLines": 1000
},
@@ -573,7 +553,7 @@
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>`",
"expr": "{job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)`",
"queryType": "range",
"maxLines": 1000
},
@@ -583,17 +563,7 @@
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>`",
"queryType": "range",
"maxLines": 1000
},
{
"refId": "D",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>`",
"expr": "{job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)`",
"queryType": "range",
"maxLines": 1000
}
@@ -614,7 +584,7 @@
"id": "filterFieldsByName",
"options": {
"include": {
"pattern": "^(timestamp|body|job|method|path|status|client|latency|size)$"
"pattern": "^(timestamp|body|status|method|path|job|client|latency|size)$"
}
}
},
@@ -623,23 +593,23 @@
"options": {
"excludeByName": {},
"indexByName": {
"Time": 0,
"Source": 1,
"Method": 2,
"Path": 3,
"Status": 4,
"Client": 5,
"Latency (ms)": 6,
"Bytes": 7,
"Raw line": 8
"timestamp": 0,
"status": 1,
"method": 2,
"path": 3,
"job": 4,
"client": 5,
"latency": 6,
"size": 7,
"body": 8
},
"renameByName": {
"timestamp": "Time",
"body": "Raw line",
"job": "Source",
"status": "Status",
"method": "Method",
"path": "Path",
"status": "Status",
"job": "Source",
"client": "Client",
"latency": "Latency (ms)",
"size": "Bytes"
@@ -660,30 +630,6 @@
"mappings": []
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Source"
},
"properties": [
{
"id": "custom.width",
"value": 210
}
]
},
{
"matcher": {
"id": "byName",
"options": "Method"
},
"properties": [
{
"id": "custom.width",
"value": 80
}
]
},
{
"matcher": {
"id": "byName",
@@ -747,6 +693,30 @@
}
]
},
{
"matcher": {
"id": "byName",
"options": "Method"
},
"properties": [
{
"id": "custom.width",
"value": 90
}
]
},
{
"matcher": {
"id": "byName",
"options": "Source"
},
"properties": [
{
"id": "custom.width",
"value": 230
}
]
},
{
"matcher": {
"id": "byName",
@@ -815,7 +785,7 @@
}
]
},
"description": "One row per request across all four sources. Latency is populated for Kong only — the NCSA format the others use carries no timing field at all, which is a property of the format, not a gap in the query."
"description": "One row per request across every source. Latency is populated for Kong only — the NCSA format the others use carries no timing field, which is a property of the format rather than a gap in the query. Raw line holds the record as emitted."
},
{
"id": 8,
@@ -838,7 +808,7 @@
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\"",
"expr": "{job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\"",
"queryType": "range"
},
{
@@ -847,7 +817,7 @@
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>`",
"expr": "{job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)`",
"queryType": "range"
},
{
@@ -856,16 +826,7 @@
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>`",
"queryType": "range"
},
{
"refId": "D",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>`",
"expr": "{job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)`",
"queryType": "range"
}
],