Stéphane Tailland
7c416ce06f
Three defects, all mine. Column order was ignored because organize's indexByName was keyed on the renamed display labels. Grafana keys it on the original field names, so the mapping matched nothing and the order fell back to whatever order the frames arrived in — which is why the raw line landed second. Every table is affected, not just the access one. The byte count was wrong for the NCSA sources: a trailing `pattern` capture takes the rest of the line, so `<size>` held `16 "-" "kube-probe/1.35"` rather than 16, and for nginx it swallowed the whole referer and user-agent. Replaced with a `regexp` parser bounding the capture on \d+, which also handles both line shapes — Keycloak's access line ends at the size while gunicorn's and nginx's carry more fields after it. Access log columns are now time, status, method, path, source, client, latency, bytes, raw line, with the raw record last in every table. gunicorn and nginx now share one query since their format is identical, which also halves the duplicate class names the status panel showed — one series per query returning the same class. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
927 lines
29 KiB
JSON
927 lines
29 KiB
JSON
{
|
|
"uid": "obs-application-logs",
|
|
"title": "Application Logs",
|
|
"description": "What the applications themselves report, with access lines and audit events subtracted. Levels are read from the line rather than inferred. Defaults to errors and warnings.",
|
|
"tags": [
|
|
"observability",
|
|
"logs",
|
|
"application"
|
|
],
|
|
"editable": false,
|
|
"schemaVersion": 39,
|
|
"version": 1,
|
|
"refresh": "",
|
|
"time": {
|
|
"from": "now-1h",
|
|
"to": "now"
|
|
},
|
|
"timezone": "browser",
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "source",
|
|
"label": "Source",
|
|
"type": "custom",
|
|
"query": "Keycloak : keycloak/keycloak,LibreChat : librechat/librechat,Classic Models API : classic-models/classic-models-api,Kong gateways : (catalog|customers|payments|sales|workforce|llm)/proxy",
|
|
"multi": true,
|
|
"includeAll": true,
|
|
"allValue": "keycloak/keycloak|librechat/librechat|classic-models/classic-models-api|(catalog|customers|payments|sales|workforce|llm)/proxy",
|
|
"current": {
|
|
"text": [
|
|
"All"
|
|
],
|
|
"value": [
|
|
"$__all"
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "level",
|
|
"label": "Level",
|
|
"type": "custom",
|
|
"query": "error,warn,notice,info,debug,crit",
|
|
"multi": true,
|
|
"includeAll": true,
|
|
"allValue": "error|warn|notice|info|debug|crit|critical|fatal|emerg|alert",
|
|
"current": {
|
|
"text": [
|
|
"error",
|
|
"warn"
|
|
],
|
|
"value": [
|
|
"error",
|
|
"warn"
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "search",
|
|
"label": "Search",
|
|
"type": "textbox",
|
|
"query": "",
|
|
"current": {
|
|
"text": "",
|
|
"value": ""
|
|
},
|
|
"description": "Case-insensitive regex over the whole line. Empty matches everything."
|
|
}
|
|
]
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"type": "stat",
|
|
"title": "Errors",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "(sum(count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> <level>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [<level>] <_>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__range])) or vector(0))",
|
|
"queryType": "instant"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "text",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"textMode": "auto"
|
|
},
|
|
"description": "Independent of the Level selector, so it still answers \"is anything broken here?\" while you are filtered on something else."
|
|
},
|
|
{
|
|
"id": 2,
|
|
"type": "stat",
|
|
"title": "Warnings",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "(sum(count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level = \"warn\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> <level>: <msg>` | label_format level=`{{ lower .level }}` | level = \"warn\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level = \"warn\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [<level>] <_>: <msg>` | label_format level=`{{ lower .level }}` | level = \"warn\" [$__range])) or vector(0))",
|
|
"queryType": "instant"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "text",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"textMode": "auto"
|
|
}
|
|
},
|
|
{
|
|
"id": 3,
|
|
"type": "stat",
|
|
"title": "Lines matching",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "(sum(count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> <level>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__range])) or vector(0)) + (sum(count_over_time({job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [<level>] <_>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__range])) or vector(0))",
|
|
"queryType": "instant"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "blue",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"textMode": "auto"
|
|
}
|
|
},
|
|
{
|
|
"id": 4,
|
|
"type": "stat",
|
|
"title": "Sources active",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "count(sum by (job) (count_over_time({job=~\"${source:regex}\"} |~ `(?i)$search` [$__range]))) or vector(0)",
|
|
"queryType": "instant"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "purple"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "purple",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"textMode": "auto"
|
|
}
|
|
},
|
|
{
|
|
"id": 5,
|
|
"type": "timeseries",
|
|
"title": "Volume by level",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 4
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum by (level) (count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__auto]))",
|
|
"legendFormat": "{{level}}",
|
|
"queryType": "range"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum by (level) (count_over_time({job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> <level>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__auto]))",
|
|
"legendFormat": "{{level}}",
|
|
"queryType": "range"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum by (level) (count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__auto]))",
|
|
"legendFormat": "{{level}}",
|
|
"queryType": "range"
|
|
},
|
|
{
|
|
"refId": "D",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum by (level) (count_over_time({job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [<level>] <_>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\" [$__auto]))",
|
|
"legendFormat": "{{level}}",
|
|
"queryType": "range"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"min": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "bars",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 80,
|
|
"showPoints": "never",
|
|
"barAlignment": 0,
|
|
"stacking": {
|
|
"mode": "normal",
|
|
"group": "A"
|
|
}
|
|
}
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "critical"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "dark-red"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "error"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "red"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "warn"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "orange"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "info"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "green"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "debug"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "unknown"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "text"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "notice"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"mode": "fixed",
|
|
"fixedColor": "text"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"sum"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "Levels as each runtime states them, lowercased so nginx's `notice` and Keycloak's `INFO` land on the same scale."
|
|
},
|
|
{
|
|
"id": 6,
|
|
"type": "timeseries",
|
|
"title": "Errors by source",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 4
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__auto]))",
|
|
"legendFormat": "{{job}}",
|
|
"queryType": "range"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> <level>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__auto]))",
|
|
"legendFormat": "{{job}}",
|
|
"queryType": "range"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__auto]))",
|
|
"legendFormat": "{{job}}",
|
|
"queryType": "range"
|
|
},
|
|
{
|
|
"refId": "D",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "sum by (job) (count_over_time({job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [<level>] <_>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"error|crit|critical|fatal|emerg|alert\" [$__auto]))",
|
|
"legendFormat": "{{job}}",
|
|
"queryType": "range"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"min": 0,
|
|
"mappings": [],
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"drawStyle": "bars",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 80,
|
|
"showPoints": "never",
|
|
"barAlignment": 0,
|
|
"stacking": {
|
|
"mode": "normal",
|
|
"group": "A"
|
|
}
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true,
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"sum"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"description": "Empty is the healthy state."
|
|
},
|
|
{
|
|
"id": 7,
|
|
"type": "table",
|
|
"title": "Application log lines",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 14,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 12
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
|
|
"queryType": "range",
|
|
"maxLines": 1000
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> <level>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
|
|
"queryType": "range",
|
|
"maxLines": 1000
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
|
|
"queryType": "range",
|
|
"maxLines": 1000
|
|
},
|
|
{
|
|
"refId": "D",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [<level>] <_>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
|
|
"queryType": "range",
|
|
"maxLines": 1000
|
|
}
|
|
],
|
|
"transformations": [
|
|
{
|
|
"id": "extractFields",
|
|
"options": {
|
|
"source": "labels",
|
|
"replace": false
|
|
}
|
|
},
|
|
{
|
|
"id": "merge",
|
|
"options": {}
|
|
},
|
|
{
|
|
"id": "filterFieldsByName",
|
|
"options": {
|
|
"include": {
|
|
"pattern": "^(timestamp|body|job|level|msg|logger)$"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {
|
|
"timestamp": 0,
|
|
"job": 1,
|
|
"level": 2,
|
|
"msg": 3,
|
|
"logger": 4,
|
|
"body": 5
|
|
},
|
|
"renameByName": {
|
|
"timestamp": "Time",
|
|
"body": "Raw line",
|
|
"job": "Source",
|
|
"level": "Level",
|
|
"msg": "Message",
|
|
"logger": "Logger"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"align": "auto",
|
|
"cellOptions": {
|
|
"type": "auto"
|
|
},
|
|
"inspect": true,
|
|
"filterable": true
|
|
},
|
|
"mappings": []
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Source"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 210
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Level"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 90
|
|
},
|
|
{
|
|
"id": "mappings",
|
|
"value": [
|
|
{
|
|
"type": "regex",
|
|
"options": {
|
|
"pattern": "^(fatal|FATAL|critical|CRITICAL)$",
|
|
"result": {
|
|
"color": "dark-red",
|
|
"index": 0
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "regex",
|
|
"options": {
|
|
"pattern": "^(error|ERROR|Error)$",
|
|
"result": {
|
|
"color": "red",
|
|
"index": 1
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "regex",
|
|
"options": {
|
|
"pattern": "^(warn|WARN|warning|WARNING)$",
|
|
"result": {
|
|
"color": "orange",
|
|
"index": 2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "regex",
|
|
"options": {
|
|
"pattern": "^(info|INFO|Info)$",
|
|
"result": {
|
|
"color": "green",
|
|
"index": 3
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "regex",
|
|
"options": {
|
|
"pattern": "^(debug|DEBUG|Debug|trace|TRACE)$",
|
|
"result": {
|
|
"color": "blue",
|
|
"index": 4
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "regex",
|
|
"options": {
|
|
"pattern": "^unknown$",
|
|
"result": {
|
|
"color": "text",
|
|
"index": 5
|
|
}
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": "custom.cellOptions",
|
|
"value": {
|
|
"type": "color-text"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Logger"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 240
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Raw line"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 240
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"showHeader": true,
|
|
"cellHeight": "sm",
|
|
"footer": {
|
|
"show": false,
|
|
"reducer": [
|
|
"sum"
|
|
],
|
|
"countRows": false,
|
|
"fields": ""
|
|
},
|
|
"sortBy": [
|
|
{
|
|
"displayName": "Time",
|
|
"desc": true
|
|
}
|
|
]
|
|
},
|
|
"description": "Each runtime parsed by its own format — Keycloak JSON, LibreChat colour-coded text, Django JSON, Kong's nginx error log — normalised to level and message. Logger is populated for Keycloak only, the one source that names the emitting class."
|
|
},
|
|
{
|
|
"id": 8,
|
|
"type": "logs",
|
|
"title": "Application logs as a stream",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 10,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 26
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{job=~\"${source:regex}\", job=\"keycloak/keycloak\"} |~ `(?i)$search` | json level=\"level\", logger=\"loggerName\", msg=\"message\" | logger !~ \"org.keycloak.http.access-log|org.keycloak.events.*\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
|
|
"queryType": "range"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{job=~\"${source:regex}\", job=\"librechat/librechat\"} |~ `(?i)$search` | line_format `{{ regexReplaceAll \"\\\\x1b\\\\[[0-9;]*m\" __line__ \"\" }}` | pattern `<_> <_> <level>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
|
|
"queryType": "range"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{job=~\"${source:regex}\", job=\"classic-models/classic-models-api\"} !~ `^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+ - - \\[` |~ `(?i)$search` | json level=\"levelname\", msg=\"message\" | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
|
|
"queryType": "range"
|
|
},
|
|
{
|
|
"refId": "D",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{job=~\"${source:regex}\", job=~\"(catalog|customers|payments|sales|workforce|llm)/proxy\"} |~ `^[0-9]{4}/[0-9]{2}/[0-9]{2} ` |~ `(?i)$search` | pattern `<_> <_> [<level>] <_>: <msg>` | label_format level=`{{ lower .level }}` | level =~ \"${level:regex}\"",
|
|
"queryType": "range"
|
|
}
|
|
],
|
|
"options": {
|
|
"showTime": true,
|
|
"showLabels": false,
|
|
"showCommonLabels": false,
|
|
"wrapLogMessage": true,
|
|
"prettifyLogMessage": true,
|
|
"sortOrder": "Descending",
|
|
"dedupStrategy": "none",
|
|
"enableLogDetails": true
|
|
},
|
|
"description": "Same result set, with ANSI colour and inline detail expansion."
|
|
}
|
|
]
|
|
}
|