Fix column order and byte counts in the log tables

Three defects, all mine.

Column order was ignored because organize's indexByName was keyed on the
renamed display labels. Grafana keys it on the original field names, so the
mapping matched nothing and the order fell back to whatever order the frames
arrived in — which is why the raw line landed second. Every table is affected,
not just the access one.

The byte count was wrong for the NCSA sources: a trailing `pattern` capture
takes the rest of the line, so `<size>` held `16 "-" "kube-probe/1.35"` rather
than 16, and for nginx it swallowed the whole referer and user-agent. Replaced
with a `regexp` parser bounding the capture on \d+, which also handles both
line shapes — Keycloak's access line ends at the size while gunicorn's and
nginx's carry more fields after it.

Access log columns are now time, status, method, path, source, client, latency,
bytes, raw line, with the raw record last in every table.

gunicorn and nginx now share one query since their format is identical, which
also halves the duplicate class names the status panel showed — one series per
query returning the same class.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

README.md

@@ -251,6 +251,17 @@ Latency is populated for Kong only: the NCSA format the other sources use
 carries no timing field. That is a property of the format, not a gap in the
 query.
 
+The NCSA sources are read with `regexp`, not `pattern`. A trailing `pattern`
+capture consumes the rest of the line, so a `<size>` at the end swallowed the
+referer and user-agent along with the byte count. Bounding the capture on `\d+`
+stops it at the number, and `regexp` tolerates both shapes — Keycloak's access
+line ends at the size, gunicorn's and nginx's carry more fields after it.
+
+In every table the raw record is the last column, and the `organize`
+transformation's `indexByName` keys on the original field names rather than the
+renamed labels — keying it on the display names silently leaves the column order
+to whatever order the frames arrived in.
+
 ### Parsed log tables### Parsed log tables
 
 Every dashboard shows its logs twice: a **table** with one column per parsed

k8s/grafana/dashboards/obs-access-logs.json

@@ -130,7 +130,7 @@
             "type": "loki",
             "uid": "loki"
           },
-          "expr": "sum(count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | status =~ \"4..\" [$__range])) or vector(0)",
+          "expr": "(sum(count_over_time({job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | status =~ \"4..\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)` | status =~ \"4..\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)` | status =~ \"4..\" [$__range])) or vector(0))",
           "queryType": "instant"
         }
       ],
@@ -170,7 +170,7 @@
         "graphMode": "none",
         "textMode": "auto"
       },
-      "description": "Counted on the Kong records only, the one source that exposes a parsed status without ambiguity. The status column of the table covers the others."
+      "description": "Summed across all four sources, each parsed by its own format."
     },
     {
       "id": 3,
@@ -193,7 +193,7 @@
             "type": "loki",
             "uid": "loki"
           },
-          "expr": "sum(count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | status =~ \"5..\" [$__range])) or vector(0)",
+          "expr": "(sum(count_over_time({job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | status =~ \"5..\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)` | status =~ \"5..\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)` | status =~ \"5..\" [$__range])) or vector(0))",
           "queryType": "instant"
         }
       ],
@@ -314,7 +314,7 @@
             "type": "loki",
             "uid": "loki"
           },
-          "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" [$__auto]))",
+          "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" [$__auto]))",
           "legendFormat": "{{job}}",
           "queryType": "range"
         },
@@ -324,7 +324,7 @@
             "type": "loki",
             "uid": "loki"
           },
-          "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>` [$__auto]))",
+          "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)` [$__auto]))",
           "legendFormat": "{{job}}",
           "queryType": "range"
         },
@@ -334,17 +334,7 @@
             "type": "loki",
             "uid": "loki"
           },
-          "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>` [$__auto]))",
-          "legendFormat": "{{job}}",
-          "queryType": "range"
-        },
-        {
-          "refId": "D",
-          "datasource": {
-            "type": "loki",
-            "uid": "loki"
-          },
-          "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>` [$__auto]))",
+          "expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)` [$__auto]))",
           "legendFormat": "{{job}}",
           "queryType": "range"
         }
@@ -408,7 +398,7 @@
             "type": "loki",
             "uid": "loki"
           },
-          "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))",
+          "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))",
           "legendFormat": "{{class}}",
           "queryType": "range"
         },
@@ -418,7 +408,7 @@
             "type": "loki",
             "uid": "loki"
           },
-          "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))",
+          "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))",
           "legendFormat": "{{class}}",
           "queryType": "range"
         },
@@ -428,17 +418,7 @@
             "type": "loki",
             "uid": "loki"
           },
-          "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))",
-          "legendFormat": "{{class}}",
-          "queryType": "range"
-        },
-        {
-          "refId": "D",
-          "datasource": {
-            "type": "loki",
-            "uid": "loki"
-          },
-          "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))",
+          "expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))",
           "legendFormat": "{{class}}",
           "queryType": "range"
         }
@@ -540,7 +520,7 @@
           "sort": "desc"
         }
       },
-      "description": "Derived from each source's parsed status, so the four formats end up on the same scale."
+      "description": "Derived from each source's parsed status, so the formats land on one scale. A class can appear once per query in the legend; the stack itself is the correct total."
     },
     {
       "id": 7,
@@ -563,7 +543,7 @@
             "type": "loki",
             "uid": "loki"
           },
-          "expr": "{job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\"",
+          "expr": "{job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\"",
           "queryType": "range",
           "maxLines": 1000
         },
@@ -573,7 +553,7 @@
             "type": "loki",
             "uid": "loki"
           },
-          "expr": "{job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>`",
+          "expr": "{job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)`",
           "queryType": "range",
           "maxLines": 1000
         },
@@ -583,17 +563,7 @@
             "type": "loki",
             "uid": "loki"
           },
-          "expr": "{job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>`",
-          "queryType": "range",
-          "maxLines": 1000
-        },
-        {
-          "refId": "D",
-          "datasource": {
-            "type": "loki",
-            "uid": "loki"
-          },
-          "expr": "{job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>`",
+          "expr": "{job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)`",
           "queryType": "range",
           "maxLines": 1000
         }
@@ -614,7 +584,7 @@
           "id": "filterFieldsByName",
           "options": {
             "include": {
-              "pattern": "^(timestamp|body|job|method|path|status|client|latency|size)$"
+              "pattern": "^(timestamp|body|status|method|path|job|client|latency|size)$"
             }
           }
         },
@@ -623,23 +593,23 @@
           "options": {
             "excludeByName": {},
             "indexByName": {
-              "Time": 0,
-              "Source": 1,
-              "Method": 2,
-              "Path": 3,
-              "Status": 4,
-              "Client": 5,
-              "Latency (ms)": 6,
-              "Bytes": 7,
-              "Raw line": 8
+              "timestamp": 0,
+              "status": 1,
+              "method": 2,
+              "path": 3,
+              "job": 4,
+              "client": 5,
+              "latency": 6,
+              "size": 7,
+              "body": 8
             },
             "renameByName": {
               "timestamp": "Time",
               "body": "Raw line",
-              "job": "Source",
+              "status": "Status",
               "method": "Method",
               "path": "Path",
-              "status": "Status",
+              "job": "Source",
               "client": "Client",
               "latency": "Latency (ms)",
               "size": "Bytes"
@@ -660,30 +630,6 @@
           "mappings": []
         },
         "overrides": [
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Source"
-            },
-            "properties": [
-              {
-                "id": "custom.width",
-                "value": 210
-              }
-            ]
-          },
-          {
-            "matcher": {
-              "id": "byName",
-              "options": "Method"
-            },
-            "properties": [
-              {
-                "id": "custom.width",
-                "value": 80
-              }
-            ]
-          },
           {
             "matcher": {
               "id": "byName",
@@ -747,6 +693,30 @@
               }
             ]
           },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Method"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 90
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Source"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 230
+              }
+            ]
+          },
           {
             "matcher": {
               "id": "byName",
@@ -815,7 +785,7 @@
           }
         ]
       },
-      "description": "One row per request across all four sources. Latency is populated for Kong only — the NCSA format the others use carries no timing field at all, which is a property of the format, not a gap in the query."
+      "description": "One row per request across every source. Latency is populated for Kong only — the NCSA format the others use carries no timing field, which is a property of the format rather than a gap in the query. Raw line holds the record as emitted."
     },
     {
       "id": 8,
@@ -838,7 +808,7 @@
             "type": "loki",
             "uid": "loki"
           },
-          "expr": "{job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\"",
+          "expr": "{job=~\"${source:regex}\", job=~\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\"",
           "queryType": "range"
         },
         {
@@ -847,7 +817,7 @@
             "type": "loki",
             "uid": "loki"
           },
-          "expr": "{job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>`",
+          "expr": "{job=~\"${source:regex}\", job=~\"keycloak/keycloak\"} |~ `(?i)$search` | json logger=\"loggerName\", m=\"message\" | logger=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)`",
           "queryType": "range"
         },
         {
@@ -856,16 +826,7 @@
             "type": "loki",
             "uid": "loki"
           },
-          "expr": "{job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>`",
-          "queryType": "range"
-        },
-        {
-          "refId": "D",
-          "datasource": {
-            "type": "loki",
-            "uid": "loki"
-          },
-          "expr": "{job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>`",
+          "expr": "{job=~\"${source:regex}\", job=~\"classic-models/classic-models-api|ingress-nginx/controller\"} |~ `(?i)$search` | regexp `^(?P<client>\\S+) \\S+ \\S+ \\[[^\\]]+\\] \"(?P<method>\\S+) (?P<path>\\S+) [^\"]*\" (?P<status>\\d{3}) (?P<size>\\d+)`",
           "queryType": "range"
         }
       ],

k8s/grafana/dashboards/obs-application-logs.json

@@ -686,12 +686,12 @@
           "options": {
             "excludeByName": {},
             "indexByName": {
-              "Time": 0,
-              "Source": 1,
-              "Level": 2,
-              "Message": 3,
-              "Logger": 4,
-              "Raw line": 5
+              "timestamp": 0,
+              "job": 1,
+              "level": 2,
+              "msg": 3,
+              "logger": 4,
+              "body": 5
             },
             "renameByName": {
               "timestamp": "Time",

k8s/grafana/dashboards/obs-audit-logs.json

@@ -482,15 +482,15 @@
           "options": {
             "excludeByName": {},
             "indexByName": {
-              "Time": 0,
-              "Event": 1,
-              "Realm": 2,
-              "Client": 3,
-              "User": 4,
-              "Source IP": 5,
-              "Error": 6,
-              "Grant": 7,
-              "Raw line": 8
+              "timestamp": 0,
+              "type": 1,
+              "realmName": 2,
+              "clientId": 3,
+              "userId": 4,
+              "ipAddress": 5,
+              "error": 6,
+              "grant_type": 7,
+              "body": 8
             },
             "renameByName": {
               "timestamp": "Time",
@@ -1127,15 +1127,15 @@
           "options": {
             "excludeByName": {},
             "indexByName": {
-              "Time": 0,
-              "Domain": 1,
-              "Service": 2,
-              "Route": 3,
-              "Method": 4,
-              "Path": 5,
-              "Status": 6,
-              "Caller IP": 7,
-              "Raw line": 8
+              "timestamp": 0,
+              "domain": 1,
+              "svc": 2,
+              "route": 3,
+              "method": 4,
+              "path": 5,
+              "status": 6,
+              "client": 7,
+              "body": 8
             },
             "renameByName": {
               "timestamp": "Time",

k8s/grafana/dashboards/obs-classic-models-api.json

@@ -1430,7 +1430,7 @@
           "options": {
             "excludeByName": {},
             "indexByName": {
-              "Time": 0,
+              "timestamp": 0,
               "status": 1,
               "method": 2,
               "uri": 3,

k8s/grafana/dashboards/obs-keycloak.json

@@ -1433,7 +1433,7 @@
           "options": {
             "excludeByName": {},
             "indexByName": {
-              "Time": 0,
+              "timestamp": 0,
               "level": 1,
               "loggerName": 2,
               "message": 3,

k8s/grafana/dashboards/obs-kong-gateways.json

@@ -1235,7 +1235,7 @@
           "options": {
             "excludeByName": {},
             "indexByName": {
-              "Time": 0,
+              "timestamp": 0,
               "status": 1,
               "method": 2,
               "uri": 3,

k8s/grafana/dashboards/obs-librechat.json

@@ -1568,7 +1568,7 @@
           "options": {
             "excludeByName": {},
             "indexByName": {
-              "Time": 0,
+              "timestamp": 0,
               "lvl": 1,
               "msg": 2,
               "body": 3

k8s/grafana/dashboards/obs-stack-health.json

@@ -446,7 +446,7 @@
           "options": {
             "excludeByName": {},
             "indexByName": {
-              "Time": 0,
+              "timestamp": 0,
               "container": 1,
               "level": 2,
               "msg": 3,