diff --git a/README.md b/README.md index b5eb5df..929aeb2 100644 --- a/README.md +++ b/README.md @@ -251,6 +251,17 @@ Latency is populated for Kong only: the NCSA format the other sources use carries no timing field. That is a property of the format, not a gap in the query. +The NCSA sources are read with `regexp`, not `pattern`. A trailing `pattern` +capture consumes the rest of the line, so a `` at the end swallowed the +referer and user-agent along with the byte count. Bounding the capture on `\d+` +stops it at the number, and `regexp` tolerates both shapes — Keycloak's access +line ends at the size, gunicorn's and nginx's carry more fields after it. + +In every table the raw record is the last column, and the `organize` +transformation's `indexByName` keys on the original field names rather than the +renamed labels — keying it on the display names silently leaves the column order +to whatever order the frames arrived in. + ### Parsed log tables### Parsed log tables Every dashboard shows its logs twice: a **table** with one column per parsed diff --git a/k8s/grafana/dashboards/obs-access-logs.json b/k8s/grafana/dashboards/obs-access-logs.json index 3f52a4b..23ad931 100644 --- a/k8s/grafana/dashboards/obs-access-logs.json +++ b/k8s/grafana/dashboards/obs-access-logs.json @@ -130,7 +130,7 @@ "type": "loki", "uid": "loki" }, - "expr": "sum(count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | status =~ \"4..\" [$__range])) or vector(0)", + "expr": "(sum(count_over_time({job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | status =~ \"4..\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)` | status =~ \"4..\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)` | status =~ \"4..\" [$__range])) or vector(0))", "queryType": "instant" } ], @@ -170,7 +170,7 @@ "graphMode": "none", "textMode": "auto" }, - "description": "Counted on the Kong records only, the one source that exposes a parsed status without ambiguity. The status column of the table covers the others." + "description": "Summed across all four sources, each parsed by its own format." }, { "id": 3, @@ -193,7 +193,7 @@ "type": "loki", "uid": "loki" }, - "expr": "sum(count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | status =~ \"5..\" [$__range])) or vector(0)", + "expr": "(sum(count_over_time({job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | status =~ \"5..\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)` | status =~ \"5..\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)` | status =~ \"5..\" [$__range])) or vector(0))", "queryType": "instant" } ], @@ -314,7 +314,7 @@ "type": "loki", "uid": "loki" }, - "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" [$__auto]))", + "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" [$__auto]))", "legendFormat": "{{job}}", "queryType": "range" }, @@ -324,7 +324,7 @@ "type": "loki", "uid": "loki" }, - "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern ` - - [<_>] \" <_>\" ` [$__auto]))", + "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)` [$__auto]))", "legendFormat": "{{job}}", "queryType": "range" }, @@ -334,17 +334,7 @@ "type": "loki", "uid": "loki" }, - "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" ` [$__auto]))", - "legendFormat": "{{job}}", - "queryType": "range" - }, - { - "refId": "D", - "datasource": { - "type": "loki", - "uid": "loki" - }, - "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" ` [$__auto]))", + "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)` [$__auto]))", "legendFormat": "{{job}}", "queryType": "range" } @@ -408,7 +398,7 @@ "type": "loki", "uid": "loki" }, - "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))", + "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))", "legendFormat": "{{class}}", "queryType": "range" }, @@ -418,7 +408,7 @@ "type": "loki", "uid": "loki" }, - "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern ` - - [<_>] \" <_>\" ` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))", + "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))", "legendFormat": "{{class}}", "queryType": "range" }, @@ -428,17 +418,7 @@ "type": "loki", "uid": "loki" }, - "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" ` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))", - "legendFormat": "{{class}}", - "queryType": "range" - }, - { - "refId": "D", - "datasource": { - "type": "loki", - "uid": "loki" - }, - "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" ` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))", + "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))", "legendFormat": "{{class}}", "queryType": "range" } @@ -540,7 +520,7 @@ "sort": "desc" } }, - "description": "Derived from each source's parsed status, so the four formats end up on the same scale." + "description": "Derived from each source's parsed status, so the formats land on one scale. A class can appear once per query in the legend; the stack itself is the correct total." }, { "id": 7, @@ -563,7 +543,7 @@ "type": "loki", "uid": "loki" }, - "expr": "{job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\"", + "expr": "{job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\"", "queryType": "range", "maxLines": 1000 }, @@ -573,7 +553,7 @@ "type": "loki", "uid": "loki" }, - "expr": "{job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern ` - - [<_>] \" <_>\" `", + "expr": "{job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)`", "queryType": "range", "maxLines": 1000 }, @@ -583,17 +563,7 @@ "type": "loki", "uid": "loki" }, - "expr": "{job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" `", - "queryType": "range", - "maxLines": 1000 - }, - { - "refId": "D", - "datasource": { - "type": "loki", - "uid": "loki" - }, - "expr": "{job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" `", + "expr": "{job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)`", "queryType": "range", "maxLines": 1000 } @@ -614,7 +584,7 @@ "id": "filterFieldsByName", "options": { "include": { - "pattern": "^(timestamp|body|job|method|path|status|client|latency|size)$" + "pattern": "^(timestamp|body|status|method|path|job|client|latency|size)$" } } }, @@ -623,23 +593,23 @@ "options": { "excludeByName": {}, "indexByName": { - "Time": 0, - "Source": 1, - "Method": 2, - "Path": 3, - "Status": 4, - "Client": 5, - "Latency (ms)": 6, - "Bytes": 7, - "Raw line": 8 + "timestamp": 0, + "status": 1, + "method": 2, + "path": 3, + "job": 4, + "client": 5, + "latency": 6, + "size": 7, + "body": 8 }, "renameByName": { "timestamp": "Time", "body": "Raw line", - "job": "Source", + "status": "Status", "method": "Method", "path": "Path", - "status": "Status", + "job": "Source", "client": "Client", "latency": "Latency (ms)", "size": "Bytes" @@ -660,30 +630,6 @@ "mappings": [] }, "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Source" - }, - "properties": [ - { - "id": "custom.width", - "value": 210 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Method" - }, - "properties": [ - { - "id": "custom.width", - "value": 80 - } - ] - }, { "matcher": { "id": "byName", @@ -747,6 +693,30 @@ } ] }, + { + "matcher": { + "id": "byName", + "options": "Method" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Source" + }, + "properties": [ + { + "id": "custom.width", + "value": 230 + } + ] + }, { "matcher": { "id": "byName", @@ -815,7 +785,7 @@ } ] }, - "description": "One row per request across all four sources. Latency is populated for Kong only — the NCSA format the others use carries no timing field at all, which is a property of the format, not a gap in the query." + "description": "One row per request across every source. Latency is populated for Kong only — the NCSA format the others use carries no timing field, which is a property of the format rather than a gap in the query. Raw line holds the record as emitted." }, { "id": 8, @@ -838,7 +808,7 @@ "type": "loki", "uid": "loki" }, - "expr": "{job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\"", + "expr": "{job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\"", "queryType": "range" }, { @@ -847,7 +817,7 @@ "type": "loki", "uid": "loki" }, - "expr": "{job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern ` - - [<_>] \" <_>\" `", + "expr": "{job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)`", "queryType": "range" }, { @@ -856,16 +826,7 @@ "type": "loki", "uid": "loki" }, - "expr": "{job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" `", - "queryType": "range" - }, - { - "refId": "D", - "datasource": { - "type": "loki", - "uid": "loki" - }, - "expr": "{job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern ` - - [<_>] \" <_>\" `", + "expr": "{job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P\\S+) (?P\\S+) [^\"]*\" (?P\\d{3}) (?P\\d+)`", "queryType": "range" } ], diff --git a/k8s/grafana/dashboards/obs-application-logs.json b/k8s/grafana/dashboards/obs-application-logs.json index ce621e5..2d32f04 100644 --- a/k8s/grafana/dashboards/obs-application-logs.json +++ b/k8s/grafana/dashboards/obs-application-logs.json @@ -686,12 +686,12 @@ "options": { "excludeByName": {}, "indexByName": { - "Time": 0, - "Source": 1, - "Level": 2, - "Message": 3, - "Logger": 4, - "Raw line": 5 + "timestamp": 0, + "job": 1, + "level": 2, + "msg": 3, + "logger": 4, + "body": 5 }, "renameByName": { "timestamp": "Time", diff --git a/k8s/grafana/dashboards/obs-audit-logs.json b/k8s/grafana/dashboards/obs-audit-logs.json index 442aa0e..65ccbf6 100644 --- a/k8s/grafana/dashboards/obs-audit-logs.json +++ b/k8s/grafana/dashboards/obs-audit-logs.json @@ -482,15 +482,15 @@ "options": { "excludeByName": {}, "indexByName": { - "Time": 0, - "Event": 1, - "Realm": 2, - "Client": 3, - "User": 4, - "Source IP": 5, - "Error": 6, - "Grant": 7, - "Raw line": 8 + "timestamp": 0, + "type": 1, + "realmName": 2, + "clientId": 3, + "userId": 4, + "ipAddress": 5, + "error": 6, + "grant_type": 7, + "body": 8 }, "renameByName": { "timestamp": "Time", @@ -1127,15 +1127,15 @@ "options": { "excludeByName": {}, "indexByName": { - "Time": 0, - "Domain": 1, - "Service": 2, - "Route": 3, - "Method": 4, - "Path": 5, - "Status": 6, - "Caller IP": 7, - "Raw line": 8 + "timestamp": 0, + "domain": 1, + "svc": 2, + "route": 3, + "method": 4, + "path": 5, + "status": 6, + "client": 7, + "body": 8 }, "renameByName": { "timestamp": "Time", diff --git a/k8s/grafana/dashboards/obs-classic-models-api.json b/k8s/grafana/dashboards/obs-classic-models-api.json index 65f0a2c..01f2069 100644 --- a/k8s/grafana/dashboards/obs-classic-models-api.json +++ b/k8s/grafana/dashboards/obs-classic-models-api.json @@ -1430,7 +1430,7 @@ "options": { "excludeByName": {}, "indexByName": { - "Time": 0, + "timestamp": 0, "status": 1, "method": 2, "uri": 3, diff --git a/k8s/grafana/dashboards/obs-keycloak.json b/k8s/grafana/dashboards/obs-keycloak.json index d01c2e1..109e9d7 100644 --- a/k8s/grafana/dashboards/obs-keycloak.json +++ b/k8s/grafana/dashboards/obs-keycloak.json @@ -1433,7 +1433,7 @@ "options": { "excludeByName": {}, "indexByName": { - "Time": 0, + "timestamp": 0, "level": 1, "loggerName": 2, "message": 3, diff --git a/k8s/grafana/dashboards/obs-kong-gateways.json b/k8s/grafana/dashboards/obs-kong-gateways.json index 9d97d11..3cc2867 100644 --- a/k8s/grafana/dashboards/obs-kong-gateways.json +++ b/k8s/grafana/dashboards/obs-kong-gateways.json @@ -1235,7 +1235,7 @@ "options": { "excludeByName": {}, "indexByName": { - "Time": 0, + "timestamp": 0, "status": 1, "method": 2, "uri": 3, diff --git a/k8s/grafana/dashboards/obs-librechat.json b/k8s/grafana/dashboards/obs-librechat.json index 556543c..38d9bbc 100644 --- a/k8s/grafana/dashboards/obs-librechat.json +++ b/k8s/grafana/dashboards/obs-librechat.json @@ -1568,7 +1568,7 @@ "options": { "excludeByName": {}, "indexByName": { - "Time": 0, + "timestamp": 0, "lvl": 1, "msg": 2, "body": 3 diff --git a/k8s/grafana/dashboards/obs-stack-health.json b/k8s/grafana/dashboards/obs-stack-health.json index c17b4fd..60f498d 100644 --- a/k8s/grafana/dashboards/obs-stack-health.json +++ b/k8s/grafana/dashboards/obs-stack-health.json @@ -446,7 +446,7 @@ "options": { "excludeByName": {}, "indexByName": { - "Time": 0, + "timestamp": 0, "container": 1, "level": 2, "msg": 3,