{ "uid": "obs-stack-health", "title": "Observability — Stack Health", "description": "Health of the Prometheus / Loki / Tempo / Alloy stack. Reference dashboard: stable UID, declaratively provisioned.", "tags": [ "observability", "stack", "provisioned" ], "editable": false, "schemaVersion": 39, "version": 1, "refresh": "30s", "time": { "from": "now-1h", "to": "now" }, "timezone": "browser", "templating": { "list": [ { "name": "namespace", "label": "Namespace", "type": "query", "datasource": { "type": "prometheus", "uid": "prometheus" }, "query": "label_values(kube_pod_info, namespace)", "refresh": 2, "includeAll": true, "multi": true, "current": { "text": "observability", "value": "observability" } } ] }, "panels": [ { "id": 1, "type": "row", "title": "Collection", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "collapsed": false, "panels": [] }, { "id": 2, "type": "stat", "title": "Prometheus targets UP", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(up)", "legendFormat": "up" } ], "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "textMode": "auto" } }, { "id": 3, "type": "stat", "title": "Prometheus targets DOWN", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(up == bool 0)", "legendFormat": "down" } ], "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "none", "textMode": "auto" } }, { "id": 4, "type": "stat", "title": "Log lines ingested (/s)", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 5, "w": 6, "x": 12, "y": 1 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(loki_distributor_lines_received_total[5m]))", "legendFormat": "lines/s" } ], "fieldConfig": { "defaults": { "unit": "short", "decimals": 1, "color": { "mode": "fixed", "fixedColor": "blue" } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "textMode": "auto" } }, { "id": 5, "type": "stat", "title": "Spans ingested (/s)", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(tempo_distributor_spans_received_total[5m]))", "legendFormat": "spans/s" } ], "fieldConfig": { "defaults": { "unit": "short", "decimals": 1, "color": { "mode": "fixed", "fixedColor": "purple" } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "textMode": "auto" } }, { "id": 6, "type": "row", "title": "Stack resources", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 }, "collapsed": false, "panels": [] }, { "id": 7, "type": "timeseries", "title": "CPU per component (observability namespace)", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"observability\", container!=\"\"}[$__rate_interval]))", "legendFormat": "{{pod}}" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 8, "type": "timeseries", "title": "Memory per component (observability namespace)", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"observability\", container!=\"\"})", "legendFormat": "{{pod}}" } ], "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 9, "type": "row", "title": "Logs", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 }, "collapsed": false, "panels": [] }, { "id": 11, "type": "table", "title": "Stack errors and warnings", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 10, "w": 24, "x": 0, "y": 16 }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "loki" }, "expr": "{namespace=\"observability\"} | logfmt level, msg, caller, logger, component_id | detected_level =~ \"error|critical|warn\"", "queryType": "range", "maxLines": 500 } ], "transformations": [ { "id": "extractFields", "options": { "source": "labels", "replace": false } }, { "id": "filterFieldsByName", "options": { "include": { "pattern": "^(timestamp|body|container|level|msg|caller|component_id)$" } } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": { "Time": 0, "container": 1, "level": 2, "msg": 3, "caller": 4, "component_id": 5, "body": 6 }, "renameByName": { "container": "Component", "level": "Level", "msg": "Message", "caller": "Caller", "component_id": "Component id", "body": "Raw record", "timestamp": "Time" } } } ], "fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": true, "filterable": true }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byName", "options": "Component" }, "properties": [ { "id": "custom.width", "value": 120 } ] }, { "matcher": { "id": "byName", "options": "Level" }, "properties": [ { "id": "custom.width", "value": 90 }, { "id": "mappings", "value": [ { "type": "regex", "options": { "pattern": "^(fatal|FATAL|Fatal|critical|CRITICAL|Critical)$", "result": { "color": "dark-red", "index": 0 } } }, { "type": "regex", "options": { "pattern": "^(error|ERROR|Error)$", "result": { "color": "red", "index": 1 } } }, { "type": "regex", "options": { "pattern": "^(warn|WARN|Warn|warning|WARNING|Warning)$", "result": { "color": "orange", "index": 2 } } }, { "type": "regex", "options": { "pattern": "^(info|INFO|Info)$", "result": { "color": "green", "index": 3 } } }, { "type": "regex", "options": { "pattern": "^(debug|DEBUG|Debug)$", "result": { "color": "blue", "index": 4 } } } ] }, { "id": "custom.cellOptions", "value": { "type": "color-text" } } ] }, { "matcher": { "id": "byName", "options": "Caller" }, "properties": [ { "id": "custom.width", "value": 200 } ] }, { "matcher": { "id": "byName", "options": "Component id" }, "properties": [ { "id": "custom.width", "value": 200 } ] }, { "matcher": { "id": "byName", "options": "Raw record" }, "properties": [ { "id": "custom.width", "value": 260 } ] } ] }, "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false, "reducer": [ "sum" ], "countRows": false, "fields": "" }, "sortBy": [ { "displayName": "Time", "desc": true } ] }, "description": "Prometheus, Loki, Tempo, Grafana and Alloy all write logfmt, so one selective parser covers the whole stack. Selective on purpose: a bare `| logfmt` would explode Loki's own query-metrics lines into dozens of columns." }, { "id": 12, "type": "row", "title": "Raw log lines", "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 26 }, "panels": [ { "id": 10, "type": "logs", "title": "Raw log lines", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 10, "w": 24, "x": 0, "y": 27 }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "loki" }, "expr": "{namespace=\"observability\"} |~ \"(?i)(error|level=error|fatal)\"", "queryType": "range" } ], "options": { "showTime": true, "showLabels": false, "wrapLogMessage": true, "sortOrder": "Descending", "dedupStrategy": "none", "enableLogDetails": true }, "description": "Kept alongside the table: the logs panel renders ANSI colour, expands a record's detail inline, tails live, and turns a trace id into a link to Tempo — none of which a table can do. Collapsed so it stays out of the way until needed." } ] } ] }