stt/infra-observability
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Split log search into access, application and audit dashboards

Browse Source 
One consolidated search compromised on all three questions at once. It also
had a real defect: the Source selector carried `.+` as its All value, so
choosing "All" widened the query back to every stream in the cluster and the
scope preset only ever populated the picker. MongoDB and cert-manager showed up
in what was meant to be an application view.

Each query now carries both `job=~"${source:regex}"` and a literal job matcher,
so the selector actually restricts rather than widens.

The three kinds are subtracted from each other rather than merely filtered:
application logs exclude Keycloak's access-log and events loggers by name, the
Django container's NCSA lines by shape, and the Kong containers' stray access
lines by requiring the nginx date prefix.

Levels are read from the line rather than taken from Loki's detected_level.
That inference marks 78% of the Kong gateway output `unknown` even though every
one of those lines states [notice] or [warn], so filtering on it hid most of
the gateway's own logs. Each parser extracts the level and lowercases it, which
puts nginx's notice, Keycloak's INFO and LibreChat's info on one scale — and
removes the need for an `unknown` entry in the selector.

Applies move to server-side. Client-side apply stores a full copy of each
object in the last-applied-configuration annotation, capped at 256KB, and the
dashboard ConfigMap passed that at eight dashboards; the failure surfaced as
"metadata.annotations: Too long". Server-side apply tracks field ownership in
metadata instead, moving the ceiling to etcd's ~1MB.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Stéphane Tailland
2026-08-25 18:48:11 +02:00
parent 40ffac4acf
commit eb35419ad0
9 changed files with 2696 additions and 320 deletions
Download Patch File Download Diff File Expand all files Collapse all files
k8s/grafana/dashboards/kustomization.yaml
+3 -1
View File
@@ -17,11 +17,13 @@ configMapGenerator:
- name: grafana-dashboards
files:
# >>> dashboards (generated by `make dashboards-sync`) >>>
- obs-access-logs.json
- obs-application-logs.json
- obs-audit-logs.json
- obs-classic-models-api.json
- obs-keycloak.json
- obs-kong-gateways.json
- obs-librechat.json
- obs-log-search.json
- obs-stack-health.json
# <<< dashboards <<<
k8s/grafana/dashboards/obs-log-search.json → k8s/grafana/dashboards/obs-access-logs.json
+377 -278
View File
@@ -1,11 +1,11 @@
{
"uid": "obs-log-search",
"title": "Log Search",
"description": "Free-text search across every log source, scoped by preset, source and level. Complements the per-application dashboards rather than replacing them.",
"uid": "obs-access-logs",
"title": "Access Logs",
"description": "One row per HTTP request, across the Kong gateways, Keycloak, the Classic Models API and the ingress controller. Each source is parsed by its own format and normalised to the same columns.",
"tags": [
"observability",
"logs",
"search"
"access"
],
"editable": false,
"schemaVersion": 39,
@@ -18,54 +18,14 @@
"timezone": "browser",
"templating": {
"list": [
{
"name": "scope",
"label": "Scope",
"type": "custom",
"query": "Applications : keycloak/keycloak|librechat/librechat|classic-models/classic-models-api,Gateways : kong-audit|catalog/proxy|customers/proxy|payments/proxy|sales/proxy|workforce/proxy|llm/proxy,Applications and gateways : keycloak/keycloak|librechat/librechat|classic-models/classic-models-api|kong-audit|catalog/proxy|customers/proxy|payments/proxy|sales/proxy|workforce/proxy|llm/proxy,Observability platform : observability/.+,Everything : .+",
"current": {
"text": "Applications and gateways",
"value": "keycloak/keycloak|librechat/librechat|classic-models/classic-models-api|kong-audit|catalog/proxy|customers/proxy|payments/proxy|sales/proxy|workforce/proxy|llm/proxy"
},
"includeAll": false,
"multi": false
},
{
"name": "source",
"label": "Source",
"type": "query",
"datasource": {
"type": "loki",
"uid": "loki"
},
"query": {
"label": "job",
"stream": "{job=~\"$scope\"}",
"type": 1,
"refId": "source-variable"
},
"refresh": 2,
"sort": 1,
"multi": true,
"includeAll": true,
"allValue": ".+",
"current": {
"text": [
"All"
],
"value": [
"$__all"
]
}
},
{
"name": "level",
"label": "Level",
"type": "custom",
"query": "info,warn,error,critical,debug,unknown",
"query": "Kong gateways : kong-audit,Keycloak : keycloak/keycloak,Classic Models API : classic-models/classic-models-api,Ingress NGINX : ingress-nginx/controller",
"multi": true,
"includeAll": true,
"allValue": ".*",
"allValue": "kong-audit|keycloak/keycloak|classic-models/classic-models-api|ingress-nginx/controller",
"current": {
"text": [
"All"
@@ -84,7 +44,7 @@
"text": "",
"value": ""
},
"description": "Case-insensitive regex, matched against the whole line. Empty matches everything."
"description": "Case-insensitive regex over the whole line. Empty matches everything."
}
]
},
@@ -92,7 +52,7 @@
{
"id": 1,
"type": "stat",
"title": "Matching lines",
"title": "Requests",
"datasource": {
"type": "loki",
"uid": "loki"
@@ -110,7 +70,7 @@
"type": "loki",
"uid": "loki"
},
"expr": "sum(count_over_time({job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search` [$__range])) or vector(0)",
"expr": "sum(count_over_time({job=~\"${source:regex}\"} |~ `(?i)$search` [$__range])) or vector(0)",
"queryType": "instant"
}
],
@@ -147,12 +107,12 @@
"graphMode": "none",
"textMode": "auto"
},
"description": "Over the dashboard's whole time range, not per interval. `or vector(0)` makes an empty result read as 0 rather than \"No data\" — on a search box, finding nothing is an answer, not a failure."
"description": "Every line the selected sources emitted over the dashboard's time range."
},
{
"id": 2,
"type": "stat",
"title": "Sources matched",
"title": "Client errors (4xx)",
"datasource": {
"type": "loki",
"uid": "loki"
@@ -170,130 +130,7 @@
"type": "loki",
"uid": "loki"
},
"expr": "count(sum by (job) (count_over_time({job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search` [$__range]))) or vector(0)",
"queryType": "instant"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"mappings": [],
"color": {
"mode": "fixed",
"fixedColor": "purple"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "purple",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "none",
"textMode": "auto"
},
"description": "How many distinct containers the current filter touches."
},
{
"id": 3,
"type": "stat",
"title": "Errors",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 12,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum(count_over_time({job=~\"${source:regex}\"} |~ `(?i)$search` | detected_level =~ \"error|critical\" [$__range])) or vector(0)",
"queryType": "instant"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"mappings": [],
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "text",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "none",
"textMode": "auto"
},
"description": "Deliberately ignores the Level selector, so it still answers \"is anything failing in here?\" while you are filtered on info."
},
{
"id": 4,
"type": "stat",
"title": "Warnings",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 18,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum(count_over_time({job=~\"${source:regex}\"} |~ `(?i)$search` | detected_level = \"warn\" [$__range])) or vector(0)",
"expr": "sum(count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | status =~ \"4..\" [$__range])) or vector(0)",
"queryType": "instant"
}
],
@@ -333,12 +170,133 @@
"graphMode": "none",
"textMode": "auto"
},
"description": "Same: independent of the Level selector."
"description": "Counted on the Kong records only, the one source that exposes a parsed status without ambiguity. The status column of the table covers the others."
},
{
"id": 3,
"type": "stat",
"title": "Server errors (5xx)",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 12,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum(count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | status =~ \"5..\" [$__range])) or vector(0)",
"queryType": "instant"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"mappings": [],
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "text",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "none",
"textMode": "auto"
}
},
{
"id": 4,
"type": "stat",
"title": "Sources active",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 18,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "count(sum by (job) (count_over_time({job=~\"${source:regex}\"} |~ `(?i)$search` [$__range]))) or vector(0)",
"queryType": "instant"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"mappings": [],
"color": {
"mode": "fixed",
"fixedColor": "purple"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "purple",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "none",
"textMode": "auto"
}
},
{
"id": 5,
"type": "timeseries",
"title": "Volume by source",
"title": "Requests by source",
"datasource": {
"type": "loki",
"uid": "loki"
@@ -356,7 +314,37 @@
"type": "loki",
"uid": "loki"
},
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search` [$__auto]))",
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" [$__auto]))",
"legendFormat": "{{job}}",
"queryType": "range"
},
{
"refId": "B",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>` [$__auto]))",
"legendFormat": "{{job}}",
"queryType": "range"
},
{
"refId": "C",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>` [$__auto]))",
"legendFormat": "{{job}}",
"queryType": "range"
},
{
"refId": "D",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>` [$__auto]))",
"legendFormat": "{{job}}",
"queryType": "range"
}
@@ -397,12 +385,12 @@
"sort": "desc"
}
},
"description": "Bars rather than lines: a line count per bucket is a run of discrete events, not a continuous signal."
"description": "Bars rather than lines: a request count per bucket is a run of discrete events."
},
{
"id": 6,
"type": "timeseries",
"title": "Volume by level",
"title": "Status classes",
"datasource": {
"type": "loki",
"uid": "loki"
@@ -420,8 +408,38 @@
"type": "loki",
"uid": "loki"
},
"expr": "sum by (detected_level) (count_over_time({job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search` [$__auto]))",
"legendFormat": "{{detected_level}}",
"expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\" | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))",
"legendFormat": "{{class}}",
"queryType": "range"
},
{
"refId": "B",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))",
"legendFormat": "{{class}}",
"queryType": "range"
},
{
"refId": "C",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))",
"legendFormat": "{{class}}",
"queryType": "range"
},
{
"refId": "D",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum by (class) (count_over_time({job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>` | label_format class=`{{ printf \"%.1sxx\" .status }}` [$__auto]))",
"legendFormat": "{{class}}",
"queryType": "range"
}
],
@@ -449,52 +467,7 @@
{
"matcher": {
"id": "byName",
"options": "critical"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "dark-red"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "error"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "red"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "warn"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "orange"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "info"
"options": "2xx"
},
"properties": [
{
@@ -509,7 +482,7 @@
{
"matcher": {
"id": "byName",
"options": "debug"
"options": "3xx"
},
"properties": [
{
@@ -524,14 +497,29 @@
{
"matcher": {
"id": "byName",
"options": "unknown"
"options": "4xx"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "text"
"fixedColor": "orange"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "5xx"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "red"
}
}
]
@@ -552,23 +540,22 @@
"sort": "desc"
}
},
"description": "`unknown` is expected to dominate: access lines and Kong records carry no severity of their own, so Loki has nothing to infer a level from. Search the text rather than the level for those."
"description": "Derived from each source's parsed status, so the four formats end up on the same scale."
},
{
"id": 7,
"type": "table",
"title": "Results",
"title": "Requests",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 13,
"h": 14,
"w": 24,
"x": 0,
"y": 12
},
"description": "One row per matching line, sorted newest first. Source and Level are columns you can sort and filter; Message holds the line as emitted. Click any cell to inspect the full value — these are deliberately not parsed per format here, that is what the per-application dashboards do.",
"targets": [
{
"refId": "A",
@@ -576,7 +563,37 @@
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search`",
"expr": "{job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\"",
"queryType": "range",
"maxLines": 1000
},
{
"refId": "B",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>`",
"queryType": "range",
"maxLines": 1000
},
{
"refId": "C",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>`",
"queryType": "range",
"maxLines": 1000
},
{
"refId": "D",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>`",
"queryType": "range",
"maxLines": 1000
}
@@ -589,11 +606,15 @@
"replace": false
}
},
{
"id": "merge",
"options": {}
},
{
"id": "filterFieldsByName",
"options": {
"include": {
"pattern": "^(timestamp|body|job|detected_level)$"
"pattern": "^(timestamp|body|job|method|path|status|client|latency|size)$"
}
}
},
@@ -604,14 +625,24 @@
"indexByName": {
"Time": 0,
"Source": 1,
"Level": 2,
"Message": 3
"Method": 2,
"Path": 3,
"Status": 4,
"Client": 5,
"Latency (ms)": 6,
"Bytes": 7,
"Raw line": 8
},
"renameByName": {
"timestamp": "Time",
"body": "Raw line",
"job": "Source",
"detected_level": "Level",
"body": "Message"
"method": "Method",
"path": "Path",
"status": "Status",
"client": "Client",
"latency": "Latency (ms)",
"size": "Bytes"
}
}
}
@@ -637,19 +668,31 @@
"properties": [
{
"id": "custom.width",
"value": 260
"value": 210
}
]
},
{
"matcher": {
"id": "byName",
"options": "Level"
"options": "Method"
},
"properties": [
{
"id": "custom.width",
"value": 90
"value": 80
}
]
},
{
"matcher": {
"id": "byName",
"options": "Status"
},
"properties": [
{
"id": "custom.width",
"value": 80
},
{
"id": "mappings",
@@ -657,9 +700,9 @@
{
"type": "regex",
"options": {
"pattern": "(?i)^(fatal|critical)$",
"pattern": "^2..$",
"result": {
"color": "dark-red",
"color": "green",
"index": 0
}
}
@@ -667,9 +710,9 @@
{
"type": "regex",
"options": {
"pattern": "(?i)^error$",
"pattern": "^3..$",
"result": {
"color": "red",
"color": "blue",
"index": 1
}
}
@@ -677,7 +720,7 @@
{
"type": "regex",
"options": {
"pattern": "(?i)^warn(ing)?$",
"pattern": "^4..$",
"result": {
"color": "orange",
"index": 2
@@ -687,32 +730,12 @@
{
"type": "regex",
"options": {
"pattern": "(?i)^info$",
"pattern": "^5..$",
"result": {
"color": "green",
"color": "red",
"index": 3
}
}
},
{
"type": "regex",
"options": {
"pattern": "(?i)^debug$",
"result": {
"color": "blue",
"index": 4
}
}
},
{
"type": "regex",
"options": {
"pattern": "^unknown$",
"result": {
"color": "text",
"index": 5
}
}
}
]
},
@@ -723,6 +746,54 @@
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Client"
},
"properties": [
{
"id": "custom.width",
"value": 130
}
]
},
{
"matcher": {
"id": "byName",
"options": "Latency (ms)"
},
"properties": [
{
"id": "custom.width",
"value": 110
}
]
},
{
"matcher": {
"id": "byName",
"options": "Bytes"
},
"properties": [
{
"id": "custom.width",
"value": 90
}
]
},
{
"matcher": {
"id": "byName",
"options": "Raw line"
},
"properties": [
{
"id": "custom.width",
"value": 240
}
]
}
]
},
@@ -743,21 +814,22 @@
"desc": true
}
]
}
},
"description": "One row per request across all four sources. Latency is populated for Kong only — the NCSA format the others use carries no timing field at all, which is a property of the format, not a gap in the query."
},
{
"id": 8,
"type": "logs",
"title": "Results as a log stream",
"title": "Requests as a log stream",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 12,
"h": 10,
"w": 24,
"x": 0,
"y": 25
"y": 26
},
"targets": [
{
@@ -766,13 +838,40 @@
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\"} | detected_level =~ \"${level:regex}\" |~ `(?i)$search`",
"expr": "{job=~\"${source:regex}\", job=\"kong-audit\"} |~ `(?i)$search` | json method=\"request.method\", path=\"request.uri\", status=\"response.status\", client=\"client_ip\", latency=\"latencies.request\", size=\"response.size\"",
"queryType": "range"
},
{
"refId": "B",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json loggerName=\"loggerName\", m=\"message\" | loggerName=\"org.keycloak.http.access-log\" | line_format `{{.m}}` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>`",
"queryType": "range"
},
{
"refId": "C",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>`",
"queryType": "range"
},
{
"refId": "D",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\", job=\"ingress-nginx/controller\"} |~ `(?i)$search` | pattern `<client> - - [<_>] \"<method> <path> <_>\" <status> <size>`",
"queryType": "range"
}
],
"options": {
"showTime": true,
"showLabels": true,
"showLabels": false,
"showCommonLabels": false,
"wrapLogMessage": true,
"prettifyLogMessage": true,
@@ -780,7 +879,7 @@
"dedupStrategy": "none",
"enableLogDetails": true
},
"description": "The same result set as the table above. Kept because it renders ANSI colour, expands a line's labels inline, tails live, and turns a trace id into a link to Tempo — none of which a table can do."
"description": "Same result set. Kept for live tail and for the trace link the Loki datasource derives from a trace id, neither of which a table can do."
}
]
}
k8s/grafana/dashboards/obs-application-logs.json
+926
View File
@@ -0,0 +1,926 @@
{
"uid": "obs-application-logs",
"title": "Application Logs",
"description": "What the applications themselves report, with access lines and audit events subtracted. Levels are read from the line rather than inferred. Defaults to errors and warnings.",
"tags": [
"observability",
"logs",
"application"
],
"editable": false,
"schemaVersion": 39,
"version": 1,
"refresh": "",
"time": {
"from": "now-1h",
"to": "now"
},
"timezone": "browser",
"templating": {
"list": [
{
"name": "source",
"label": "Source",
"type": "custom",
"query": "Keycloak : keycloak/keycloak,LibreChat : librechat/librechat,Classic Models API : classic-models/classic-models-api,Kong gateways : (catalog|customers|payments|sales|workforce|llm)/proxy",
"multi": true,
"includeAll": true,
"allValue": "keycloak/keycloak|librechat/librechat|classic-models/classic-models-api|(catalog|customers|payments|sales|workforce|llm)/proxy",
"current": {
"text": [
"All"
],
"value": [
"$__all"
]
}
},
{
"name": "level",
"label": "Level",
"type": "custom",
"query": "error,warn,notice,info,debug,crit",
"multi": true,
"includeAll": true,
"allValue": "error|warn|notice|info|debug|crit|critical|fatal|emerg|alert",
"current": {
"text": [
"error",
"warn"
],
"value": [
"error",
"warn"
]
}
},
{
"name": "search",
"label": "Search",
"type": "textbox",
"query": "",
"current": {
"text": "",
"value": ""
},
"description": "Case-insensitive regex over the whole line. Empty matches everything."
}
]
},
"panels": [
{
"id": 1,
"type": "stat",
"title": "Errors",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "(sum(count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> <level>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [<level>] <_>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__range])) or vector(0))",
"queryType": "instant"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"mappings": [],
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "text",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "none",
"textMode": "auto"
},
"description": "Independent of the Level selector, so it still answers \"is anything broken here?\" while you are filtered on something else."
},
{
"id": 2,
"type": "stat",
"title": "Warnings",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 6,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "(sum(count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level = \"warn\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> <level>: <msg>` | label_format level=`{{ lower .level }}` | level = \"warn\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level = \"warn\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [<level>] <_>: <msg>` | label_format level=`{{ lower .level }}` | level = \"warn\" [$__range])) or vector(0))",
"queryType": "instant"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"mappings": [],
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "text",
"value": null
},
{
"color": "orange",
"value": 1
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "none",
"textMode": "auto"
}
},
{
"id": 3,
"type": "stat",
"title": "Lines matching",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 12,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "(sum(count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> <level>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [<level>] <_>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__range])) or vector(0))",
"queryType": "instant"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"mappings": [],
"color": {
"mode": "fixed",
"fixedColor": "blue"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "blue",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "none",
"textMode": "auto"
}
},
{
"id": 4,
"type": "stat",
"title": "Sources active",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 18,
"y": 0
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "count(sum by (job) (count_over_time({job=~\"${source:regex}\"} |~ `(?i)$search` [$__range]))) or vector(0)",
"queryType": "instant"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"mappings": [],
"color": {
"mode": "fixed",
"fixedColor": "purple"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "purple",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "none",
"textMode": "auto"
}
},
{
"id": 5,
"type": "timeseries",
"title": "Volume by level",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 4
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum by (level) (count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__auto]))",
"legendFormat": "{{level}}",
"queryType": "range"
},
{
"refId": "B",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum by (level) (count_over_time({job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> <level>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__auto]))",
"legendFormat": "{{level}}",
"queryType": "range"
},
{
"refId": "C",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum by (level) (count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__auto]))",
"legendFormat": "{{level}}",
"queryType": "range"
},
{
"refId": "D",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum by (level) (count_over_time({job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [<level>] <_>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__auto]))",
"legendFormat": "{{level}}",
"queryType": "range"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0,
"mappings": [],
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "bars",
"lineWidth": 1,
"fillOpacity": 80,
"showPoints": "never",
"barAlignment": 0,
"stacking": {
"mode": "normal",
"group": "A"
}
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "critical"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "dark-red"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "error"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "red"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "warn"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "orange"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "info"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "green"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "debug"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "blue"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "unknown"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "text"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "notice"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "text"
}
}
]
}
]
},
"options": {
"legend": {
"showLegend": true,
"displayMode": "table",
"placement": "bottom",
"calcs": [
"sum"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"description": "Levels as each runtime states them, lowercased so nginx's `notice` and Keycloak's `INFO` land on the same scale."
},
{
"id": 6,
"type": "timeseries",
"title": "Errors by source",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 4
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__auto]))",
"legendFormat": "{{job}}",
"queryType": "range"
},
{
"refId": "B",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> <level>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__auto]))",
"legendFormat": "{{job}}",
"queryType": "range"
},
{
"refId": "C",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__auto]))",
"legendFormat": "{{job}}",
"queryType": "range"
},
{
"refId": "D",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [<level>] <_>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__auto]))",
"legendFormat": "{{job}}",
"queryType": "range"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0,
"mappings": [],
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "bars",
"lineWidth": 1,
"fillOpacity": 80,
"showPoints": "never",
"barAlignment": 0,
"stacking": {
"mode": "normal",
"group": "A"
}
}
},
"overrides": []
},
"options": {
"legend": {
"showLegend": true,
"displayMode": "table",
"placement": "bottom",
"calcs": [
"sum"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"description": "Empty is the healthy state."
},
{
"id": 7,
"type": "table",
"title": "Application log lines",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 14,
"w": 24,
"x": 0,
"y": 12
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
"queryType": "range",
"maxLines": 1000
},
{
"refId": "B",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> <level>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
"queryType": "range",
"maxLines": 1000
},
{
"refId": "C",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
"queryType": "range",
"maxLines": 1000
},
{
"refId": "D",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [<level>] <_>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
"queryType": "range",
"maxLines": 1000
}
],
"transformations": [
{
"id": "extractFields",
"options": {
"source": "labels",
"replace": false
}
},
{
"id": "merge",
"options": {}
},
{
"id": "filterFieldsByName",
"options": {
"include": {
"pattern": "^(timestamp|body|job|level|msg|logger)$"
}
}
},
{
"id": "organize",
"options": {
"excludeByName": {},
"indexByName": {
"Time": 0,
"Source": 1,
"Level": 2,
"Message": 3,
"Logger": 4,
"Raw line": 5
},
"renameByName": {
"timestamp": "Time",
"body": "Raw line",
"job": "Source",
"level": "Level",
"msg": "Message",
"logger": "Logger"
}
}
}
],
"fieldConfig": {
"defaults": {
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"inspect": true,
"filterable": true
},
"mappings": []
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Source"
},
"properties": [
{
"id": "custom.width",
"value": 210
}
]
},
{
"matcher": {
"id": "byName",
"options": "Level"
},
"properties": [
{
"id": "custom.width",
"value": 90
},
{
"id": "mappings",
"value": [
{
"type": "regex",
"options": {
"pattern": "^(fatal|FATAL|critical|CRITICAL)$",
"result": {
"color": "dark-red",
"index": 0
}
}
},
{
"type": "regex",
"options": {
"pattern": "^(error|ERROR|Error)$",
"result": {
"color": "red",
"index": 1
}
}
},
{
"type": "regex",
"options": {
"pattern": "^(warn|WARN|warning|WARNING)$",
"result": {
"color": "orange",
"index": 2
}
}
},
{
"type": "regex",
"options": {
"pattern": "^(info|INFO|Info)$",
"result": {
"color": "green",
"index": 3
}
}
},
{
"type": "regex",
"options": {
"pattern": "^(debug|DEBUG|Debug|trace|TRACE)$",
"result": {
"color": "blue",
"index": 4
}
}
},
{
"type": "regex",
"options": {
"pattern": "^unknown$",
"result": {
"color": "text",
"index": 5
}
}
}
]
},
{
"id": "custom.cellOptions",
"value": {
"type": "color-text"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Logger"
},
"properties": [
{
"id": "custom.width",
"value": 240
}
]
},
{
"matcher": {
"id": "byName",
"options": "Raw line"
},
"properties": [
{
"id": "custom.width",
"value": 240
}
]
}
]
},
"options": {
"showHeader": true,
"cellHeight": "sm",
"footer": {
"show": false,
"reducer": [
"sum"
],
"countRows": false,
"fields": ""
},
"sortBy": [
{
"displayName": "Time",
"desc": true
}
]
},
"description": "Each runtime parsed by its own format — Keycloak JSON, LibreChat colour-coded text, Django JSON, Kong's nginx error log — normalised to level and message. Logger is populated for Keycloak only, the one source that names the emitting class."
},
{
"id": 8,
"type": "logs",
"title": "Application logs as a stream",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 26
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
"queryType": "range"
},
{
"refId": "B",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> <level>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
"queryType": "range"
},
{
"refId": "C",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
"queryType": "range"
},
{
"refId": "D",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "{job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [<level>] <_>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
"queryType": "range"
}
],
"options": {
"showTime": true,
"showLabels": false,
"showCommonLabels": false,
"wrapLogMessage": true,
"prettifyLogMessage": true,
"sortOrder": "Descending",
"dedupStrategy": "none",
"enableLogDetails": true
},
"description": "Same result set, with ANSI colour and inline detail expansion."
}
]
}
k8s/grafana/dashboards/obs-audit-logs.json
+1325
View File
File diff suppressed because it is too large Load Diff
k8s/grafana/dashboards/obs-keycloak.json
+5 -5
View File
@@ -1480,7 +1480,7 @@
{
"type": "regex",
"options": {
"pattern": "(?i)^(fatal|critical)$",
"pattern": "^(fatal|FATAL|Fatal|critical|CRITICAL|Critical)$",
"result": {
"color": "dark-red",
"index": 0
@@ -1490,7 +1490,7 @@
{
"type": "regex",
"options": {
"pattern": "(?i)^error$",
"pattern": "^(error|ERROR|Error)$",
"result": {
"color": "red",
"index": 1
@@ -1500,7 +1500,7 @@
{
"type": "regex",
"options": {
"pattern": "(?i)^warn(ing)?$",
"pattern": "^(warn|WARN|Warn|warning|WARNING|Warning)$",
"result": {
"color": "orange",
"index": 2
@@ -1510,7 +1510,7 @@
{
"type": "regex",
"options": {
"pattern": "(?i)^info$",
"pattern": "^(info|INFO|Info)$",
"result": {
"color": "green",
"index": 3
@@ -1520,7 +1520,7 @@
{
"type": "regex",
"options": {
"pattern": "(?i)^debug$",
"pattern": "^(debug|DEBUG|Debug)$",
"result": {
"color": "blue",
"index": 4
k8s/grafana/dashboards/obs-librechat.json
+5 -5
View File
@@ -1611,7 +1611,7 @@
{
"type": "regex",
"options": {
"pattern": "(?i)^(fatal|critical)$",
"pattern": "^(fatal|FATAL|Fatal|critical|CRITICAL|Critical)$",
"result": {
"color": "dark-red",
"index": 0
@@ -1621,7 +1621,7 @@
{
"type": "regex",
"options": {
"pattern": "(?i)^error$",
"pattern": "^(error|ERROR|Error)$",
"result": {
"color": "red",
"index": 1
@@ -1631,7 +1631,7 @@
{
"type": "regex",
"options": {
"pattern": "(?i)^warn(ing)?$",
"pattern": "^(warn|WARN|Warn|warning|WARNING|Warning)$",
"result": {
"color": "orange",
"index": 2
@@ -1641,7 +1641,7 @@
{
"type": "regex",
"options": {
"pattern": "(?i)^info$",
"pattern": "^(info|INFO|Info)$",
"result": {
"color": "green",
"index": 3
@@ -1651,7 +1651,7 @@
{
"type": "regex",
"options": {
"pattern": "(?i)^debug$",
"pattern": "^(debug|DEBUG|Debug)$",
"result": {
"color": "blue",
"index": 4
k8s/grafana/dashboards/obs-stack-health.json
+5 -5
View File
@@ -507,7 +507,7 @@
{
"type": "regex",
"options": {
"pattern": "(?i)^(fatal|critical)$",
"pattern": "^(fatal|FATAL|Fatal|critical|CRITICAL|Critical)$",
"result": {
"color": "dark-red",
"index": 0
@@ -517,7 +517,7 @@
{
"type": "regex",
"options": {
"pattern": "(?i)^error$",
"pattern": "^(error|ERROR|Error)$",
"result": {
"color": "red",
"index": 1
@@ -527,7 +527,7 @@
{
"type": "regex",
"options": {
"pattern": "(?i)^warn(ing)?$",
"pattern": "^(warn|WARN|Warn|warning|WARNING|Warning)$",
"result": {
"color": "orange",
"index": 2
@@ -537,7 +537,7 @@
{
"type": "regex",
"options": {
"pattern": "(?i)^info$",
"pattern": "^(info|INFO|Info)$",
"result": {
"color": "green",
"index": 3
@@ -547,7 +547,7 @@
{
"type": "regex",
"options": {
"pattern": "(?i)^debug$",
"pattern": "^(debug|DEBUG|Debug)$",
"result": {
"color": "blue",
"index": 4