Split log search into access, application and audit dashboards
One consolidated search compromised on all three questions at once. It also
had a real defect: the Source selector carried `.+` as its All value, so
choosing "All" widened the query back to every stream in the cluster and the
scope preset only ever populated the picker. MongoDB and cert-manager showed up
in what was meant to be an application view.
Each query now carries both `job=~"${source:regex}"` and a literal job matcher,
so the selector actually restricts rather than widens.
The three kinds are subtracted from each other rather than merely filtered:
application logs exclude Keycloak's access-log and events loggers by name, the
Django container's NCSA lines by shape, and the Kong containers' stray access
lines by requiring the nginx date prefix.
Levels are read from the line rather than taken from Loki's detected_level.
That inference marks 78% of the Kong gateway output `unknown` even though every
one of those lines states [notice] or [warn], so filtering on it hid most of
the gateway's own logs. Each parser extracts the level and lowercases it, which
puts nginx's notice, Keycloak's INFO and LibreChat's info on one scale — and
removes the need for an `unknown` entry in the selector.
Applies move to server-side. Client-side apply stores a full copy of each
object in the last-applied-configuration annotation, capped at 256KB, and the
dashboard ConfigMap passed that at eight dashboards; the failure surfaced as
"metadata.annotations: Too long". Server-side apply tracks field ownership in
metadata instead, moving the ceiling to etcd's ~1MB.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Stéphane Tailland
@@ -0,0 +1,926 @@
|
||||
{
|
||||
"uid": "obs-application-logs",
|
||||
"title": "Application Logs",
|
||||
"description": "What the applications themselves report, with access lines and audit events subtracted. Levels are read from the line rather than inferred. Defaults to errors and warnings.",
|
||||
"tags": [
|
||||
"observability",
|
||||
"logs",
|
||||
"application"
|
||||
],
|
||||
"editable": false,
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "",
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timezone": "browser",
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "source",
|
||||
"label": "Source",
|
||||
"type": "custom",
|
||||
"query": "Keycloak : keycloak/keycloak,LibreChat : librechat/librechat,Classic Models API : classic-models/classic-models-api,Kong gateways : (catalog|customers|payments|sales|workforce|llm)/proxy",
|
||||
"multi": true,
|
||||
"includeAll": true,
|
||||
"allValue": "keycloak/keycloak|librechat/librechat|classic-models/classic-models-api|(catalog|customers|payments|sales|workforce|llm)/proxy",
|
||||
"current": {
|
||||
"text": [
|
||||
"All"
|
||||
],
|
||||
"value": [
|
||||
"$__all"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "level",
|
||||
"label": "Level",
|
||||
"type": "custom",
|
||||
"query": "error,warn,notice,info,debug,crit",
|
||||
"multi": true,
|
||||
"includeAll": true,
|
||||
"allValue": "error|warn|notice|info|debug|crit|critical|fatal|emerg|alert",
|
||||
"current": {
|
||||
"text": [
|
||||
"error",
|
||||
"warn"
|
||||
],
|
||||
"value": [
|
||||
"error",
|
||||
"warn"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "search",
|
||||
"label": "Search",
|
||||
"type": "textbox",
|
||||
"query": "",
|
||||
"current": {
|
||||
"text": "",
|
||||
"value": ""
|
||||
},
|
||||
"description": "Case-insensitive regex over the whole line. Empty matches everything."
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "stat",
|
||||
"title": "Errors",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "(sum(count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> <level>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [<level>] <_>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__range])) or vector(0))",
|
||||
"queryType": "instant"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"mappings": [],
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "text",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"textMode": "auto"
|
||||
},
|
||||
"description": "Independent of the Level selector, so it still answers \"is anything broken here?\" while you are filtered on something else."
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "Warnings",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"y": 0
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "(sum(count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level = \"warn\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> <level>: <msg>` | label_format level=`{{ lower .level }}` | level = \"warn\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level = \"warn\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [<level>] <_>: <msg>` | label_format level=`{{ lower .level }}` | level = \"warn\" [$__range])) or vector(0))",
|
||||
"queryType": "instant"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"mappings": [],
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "text",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "orange",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"textMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "stat",
|
||||
"title": "Lines matching",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "(sum(count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> <level>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [<level>] <_>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__range])) or vector(0))",
|
||||
"queryType": "instant"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"mappings": [],
|
||||
"color": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "blue"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "blue",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"textMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "stat",
|
||||
"title": "Sources active",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 18,
|
||||
"y": 0
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "count(sum by (job) (count_over_time({job=~\"${source:regex}\"} |~ `(?i)$search` [$__range]))) or vector(0)",
|
||||
"queryType": "instant"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"mappings": [],
|
||||
"color": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "purple"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "purple",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"textMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Volume by level",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 4
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "sum by (level) (count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__auto]))",
|
||||
"legendFormat": "{{level}}",
|
||||
"queryType": "range"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "sum by (level) (count_over_time({job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> <level>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__auto]))",
|
||||
"legendFormat": "{{level}}",
|
||||
"queryType": "range"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "sum by (level) (count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__auto]))",
|
||||
"legendFormat": "{{level}}",
|
||||
"queryType": "range"
|
||||
},
|
||||
{
|
||||
"refId": "D",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "sum by (level) (count_over_time({job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [<level>] <_>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__auto]))",
|
||||
"legendFormat": "{{level}}",
|
||||
"queryType": "range"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"min": 0,
|
||||
"mappings": [],
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"drawStyle": "bars",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 80,
|
||||
"showPoints": "never",
|
||||
"barAlignment": 0,
|
||||
"stacking": {
|
||||
"mode": "normal",
|
||||
"group": "A"
|
||||
}
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "critical"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "dark-red"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "error"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "red"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "warn"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "orange"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "info"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "green"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "debug"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "blue"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "unknown"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "text"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "notice"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "text"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"showLegend": true,
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"sum"
|
||||
]
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"description": "Levels as each runtime states them, lowercased so nginx's `notice` and Keycloak's `INFO` land on the same scale."
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Errors by source",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 4
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__auto]))",
|
||||
"legendFormat": "{{job}}",
|
||||
"queryType": "range"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> <level>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__auto]))",
|
||||
"legendFormat": "{{job}}",
|
||||
"queryType": "range"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__auto]))",
|
||||
"legendFormat": "{{job}}",
|
||||
"queryType": "range"
|
||||
},
|
||||
{
|
||||
"refId": "D",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [<level>] <_>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__auto]))",
|
||||
"legendFormat": "{{job}}",
|
||||
"queryType": "range"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"min": 0,
|
||||
"mappings": [],
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"drawStyle": "bars",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 80,
|
||||
"showPoints": "never",
|
||||
"barAlignment": 0,
|
||||
"stacking": {
|
||||
"mode": "normal",
|
||||
"group": "A"
|
||||
}
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"showLegend": true,
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"sum"
|
||||
]
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"description": "Empty is the healthy state."
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "table",
|
||||
"title": "Application log lines",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 14,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 12
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "{job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
|
||||
"queryType": "range",
|
||||
"maxLines": 1000
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "{job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> <level>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
|
||||
"queryType": "range",
|
||||
"maxLines": 1000
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "{job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
|
||||
"queryType": "range",
|
||||
"maxLines": 1000
|
||||
},
|
||||
{
|
||||
"refId": "D",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "{job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [<level>] <_>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
|
||||
"queryType": "range",
|
||||
"maxLines": 1000
|
||||
}
|
||||
],
|
||||
"transformations": [
|
||||
{
|
||||
"id": "extractFields",
|
||||
"options": {
|
||||
"source": "labels",
|
||||
"replace": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "merge",
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": "filterFieldsByName",
|
||||
"options": {
|
||||
"include": {
|
||||
"pattern": "^(timestamp|body|job|level|msg|logger)$"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": {},
|
||||
"indexByName": {
|
||||
"Time": 0,
|
||||
"Source": 1,
|
||||
"Level": 2,
|
||||
"Message": 3,
|
||||
"Logger": 4,
|
||||
"Raw line": 5
|
||||
},
|
||||
"renameByName": {
|
||||
"timestamp": "Time",
|
||||
"body": "Raw line",
|
||||
"job": "Source",
|
||||
"level": "Level",
|
||||
"msg": "Message",
|
||||
"logger": "Logger"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"align": "auto",
|
||||
"cellOptions": {
|
||||
"type": "auto"
|
||||
},
|
||||
"inspect": true,
|
||||
"filterable": true
|
||||
},
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Source"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.width",
|
||||
"value": 210
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Level"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.width",
|
||||
"value": 90
|
||||
},
|
||||
{
|
||||
"id": "mappings",
|
||||
"value": [
|
||||
{
|
||||
"type": "regex",
|
||||
"options": {
|
||||
"pattern": "^(fatal|FATAL|critical|CRITICAL)$",
|
||||
"result": {
|
||||
"color": "dark-red",
|
||||
"index": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "regex",
|
||||
"options": {
|
||||
"pattern": "^(error|ERROR|Error)$",
|
||||
"result": {
|
||||
"color": "red",
|
||||
"index": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "regex",
|
||||
"options": {
|
||||
"pattern": "^(warn|WARN|warning|WARNING)$",
|
||||
"result": {
|
||||
"color": "orange",
|
||||
"index": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "regex",
|
||||
"options": {
|
||||
"pattern": "^(info|INFO|Info)$",
|
||||
"result": {
|
||||
"color": "green",
|
||||
"index": 3
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "regex",
|
||||
"options": {
|
||||
"pattern": "^(debug|DEBUG|Debug|trace|TRACE)$",
|
||||
"result": {
|
||||
"color": "blue",
|
||||
"index": 4
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "regex",
|
||||
"options": {
|
||||
"pattern": "^unknown$",
|
||||
"result": {
|
||||
"color": "text",
|
||||
"index": 5
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "custom.cellOptions",
|
||||
"value": {
|
||||
"type": "color-text"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Logger"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.width",
|
||||
"value": 240
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Raw line"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.width",
|
||||
"value": 240
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"cellHeight": "sm",
|
||||
"footer": {
|
||||
"show": false,
|
||||
"reducer": [
|
||||
"sum"
|
||||
],
|
||||
"countRows": false,
|
||||
"fields": ""
|
||||
},
|
||||
"sortBy": [
|
||||
{
|
||||
"displayName": "Time",
|
||||
"desc": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "Each runtime parsed by its own format — Keycloak JSON, LibreChat colour-coded text, Django JSON, Kong's nginx error log — normalised to level and message. Logger is populated for Keycloak only, the one source that names the emitting class."
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "logs",
|
||||
"title": "Application logs as a stream",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 10,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 26
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "{job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
|
||||
"queryType": "range"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "{job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> <level>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
|
||||
"queryType": "range"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "{job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
|
||||
"queryType": "range"
|
||||
},
|
||||
{
|
||||
"refId": "D",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "{job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [<level>] <_>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
|
||||
"queryType": "range"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"showLabels": false,
|
||||
"showCommonLabels": false,
|
||||
"wrapLogMessage": true,
|
||||
"prettifyLogMessage": true,
|
||||
"sortOrder": "Descending",
|
||||
"dedupStrategy": "none",
|
||||
"enableLogDetails": true
|
||||
},
|
||||
"description": "Same result set, with ANSI colour and inline detail expansion."
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user