Stéphane Tailland
7c416ce06f
Three defects, all mine. Column order was ignored because organize's indexByName was keyed on the renamed display labels. Grafana keys it on the original field names, so the mapping matched nothing and the order fell back to whatever order the frames arrived in — which is why the raw line landed second. Every table is affected, not just the access one. The byte count was wrong for the NCSA sources: a trailing `pattern` capture takes the rest of the line, so `<size>` held `16 "-" "kube-probe/1.35"` rather than 16, and for nginx it swallowed the whole referer and user-agent. Replaced with a `regexp` parser bounding the capture on \d+, which also handles both line shapes — Keycloak's access line ends at the size while gunicorn's and nginx's carry more fields after it. Access log columns are now time, status, method, path, source, client, latency, bytes, raw line, with the raw record last in every table. gunicorn and nginx now share one query since their format is identical, which also halves the duplicate class names the status panel showed — one series per query returning the same class. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
676 lines
16 KiB
JSON
676 lines
16 KiB
JSON
{
|
|
"uid": "obs-stack-health",
|
|
"title": "Observability — Stack Health",
|
|
"description": "Health of the Prometheus / Loki / Tempo / Alloy stack. Reference dashboard: stable UID, declaratively provisioned.",
|
|
"tags": [
|
|
"observability",
|
|
"stack",
|
|
"provisioned"
|
|
],
|
|
"editable": false,
|
|
"schemaVersion": 39,
|
|
"version": 1,
|
|
"refresh": "30s",
|
|
"time": {
|
|
"from": "now-1h",
|
|
"to": "now"
|
|
},
|
|
"timezone": "browser",
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "namespace",
|
|
"label": "Namespace",
|
|
"type": "query",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"query": "label_values(kube_pod_info, namespace)",
|
|
"refresh": 2,
|
|
"includeAll": true,
|
|
"multi": true,
|
|
"current": {
|
|
"text": "observability",
|
|
"value": "observability"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"type": "row",
|
|
"title": "Collection",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 2,
|
|
"type": "stat",
|
|
"title": "Prometheus targets UP",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(up)",
|
|
"legendFormat": "up"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "red",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "green",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"textMode": "auto"
|
|
}
|
|
},
|
|
{
|
|
"id": 3,
|
|
"type": "stat",
|
|
"title": "Prometheus targets DOWN",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(up == bool 0)",
|
|
"legendFormat": "down"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"textMode": "auto"
|
|
}
|
|
},
|
|
{
|
|
"id": 4,
|
|
"type": "stat",
|
|
"title": "Log lines ingested (/s)",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(rate(loki_distributor_lines_received_total[5m]))",
|
|
"legendFormat": "lines/s"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 1,
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "blue"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"textMode": "auto"
|
|
}
|
|
},
|
|
{
|
|
"id": 5,
|
|
"type": "stat",
|
|
"title": "Spans ingested (/s)",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum(rate(tempo_distributor_spans_received_total[5m]))",
|
|
"legendFormat": "spans/s"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"decimals": 1,
|
|
"color": {
|
|
"mode": "fixed",
|
|
"fixedColor": "purple"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"textMode": "auto"
|
|
}
|
|
},
|
|
{
|
|
"id": 6,
|
|
"type": "row",
|
|
"title": "Stack resources",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 6
|
|
},
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 7,
|
|
"type": "timeseries",
|
|
"title": "CPU per component (observability namespace)",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 7
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"observability\", container!=\"\"}[$__rate_interval]))",
|
|
"legendFormat": "{{pod}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 8,
|
|
"type": "timeseries",
|
|
"title": "Memory per component (observability namespace)",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 7
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"observability\", container!=\"\"})",
|
|
"legendFormat": "{{pod}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes",
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineWidth": 1,
|
|
"fillOpacity": 10,
|
|
"showPoints": "never"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 9,
|
|
"type": "row",
|
|
"title": "Logs",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 15
|
|
},
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 11,
|
|
"type": "table",
|
|
"title": "Stack errors and warnings",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 10,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 16
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{namespace=\"observability\"} | logfmt level, msg, caller, logger, component_id | detected_level =~ \"error|critical|warn\"",
|
|
"queryType": "range",
|
|
"maxLines": 500
|
|
}
|
|
],
|
|
"transformations": [
|
|
{
|
|
"id": "extractFields",
|
|
"options": {
|
|
"source": "labels",
|
|
"replace": false
|
|
}
|
|
},
|
|
{
|
|
"id": "filterFieldsByName",
|
|
"options": {
|
|
"include": {
|
|
"pattern": "^(timestamp|body|container|level|msg|caller|component_id)$"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {
|
|
"timestamp": 0,
|
|
"container": 1,
|
|
"level": 2,
|
|
"msg": 3,
|
|
"caller": 4,
|
|
"component_id": 5,
|
|
"body": 6
|
|
},
|
|
"renameByName": {
|
|
"container": "Component",
|
|
"level": "Level",
|
|
"msg": "Message",
|
|
"caller": "Caller",
|
|
"component_id": "Component id",
|
|
"body": "Raw record",
|
|
"timestamp": "Time"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"align": "auto",
|
|
"cellOptions": {
|
|
"type": "auto"
|
|
},
|
|
"inspect": true,
|
|
"filterable": true
|
|
},
|
|
"mappings": []
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Component"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 120
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Level"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 90
|
|
},
|
|
{
|
|
"id": "mappings",
|
|
"value": [
|
|
{
|
|
"type": "regex",
|
|
"options": {
|
|
"pattern": "^(fatal|FATAL|Fatal|critical|CRITICAL|Critical)$",
|
|
"result": {
|
|
"color": "dark-red",
|
|
"index": 0
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "regex",
|
|
"options": {
|
|
"pattern": "^(error|ERROR|Error)$",
|
|
"result": {
|
|
"color": "red",
|
|
"index": 1
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "regex",
|
|
"options": {
|
|
"pattern": "^(warn|WARN|Warn|warning|WARNING|Warning)$",
|
|
"result": {
|
|
"color": "orange",
|
|
"index": 2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "regex",
|
|
"options": {
|
|
"pattern": "^(info|INFO|Info)$",
|
|
"result": {
|
|
"color": "green",
|
|
"index": 3
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "regex",
|
|
"options": {
|
|
"pattern": "^(debug|DEBUG|Debug)$",
|
|
"result": {
|
|
"color": "blue",
|
|
"index": 4
|
|
}
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": "custom.cellOptions",
|
|
"value": {
|
|
"type": "color-text"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Caller"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 200
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Component id"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 200
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Raw record"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 260
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"showHeader": true,
|
|
"cellHeight": "sm",
|
|
"footer": {
|
|
"show": false,
|
|
"reducer": [
|
|
"sum"
|
|
],
|
|
"countRows": false,
|
|
"fields": ""
|
|
},
|
|
"sortBy": [
|
|
{
|
|
"displayName": "Time",
|
|
"desc": true
|
|
}
|
|
]
|
|
},
|
|
"description": "Prometheus, Loki, Tempo, Grafana and Alloy all write logfmt, so one selective parser covers the whole stack. Selective on purpose: a bare `| logfmt` would explode Loki's own query-metrics lines into dozens of columns."
|
|
},
|
|
{
|
|
"id": 12,
|
|
"type": "row",
|
|
"title": "Raw log lines",
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 26
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 10,
|
|
"type": "logs",
|
|
"title": "Raw log lines",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 10,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 27
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{namespace=\"observability\"} |~ \"(?i)(error|level=error|fatal)\"",
|
|
"queryType": "range"
|
|
}
|
|
],
|
|
"options": {
|
|
"showTime": true,
|
|
"showLabels": false,
|
|
"wrapLogMessage": true,
|
|
"sortOrder": "Descending",
|
|
"dedupStrategy": "none",
|
|
"enableLogDetails": true
|
|
},
|
|
"description": "Kept alongside the table: the logs panel renders ANSI colour, expands a record's detail inline, tails live, and turns a trace id into a link to Tempo — none of which a table can do. Collapsed so it stays out of the way until needed."
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|