From d3abde1040f99205c622bc9412163913e2453da7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Tailland?= Date: Wed, 26 Aug 2026 07:55:17 +0200 Subject: [PATCH] Drop the traces dashboard, keep the datasource wiring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dashboard reimplemented, less well, what Explore already does: its search panel had no TraceQL query builder and no autocompletion, and its service dependency table replaced an interactive node graph with a list. Searching and exploring traces belongs in Explore. The datasource correlations this work also fixed are kept — they are what was actually missing, and they apply in Explore exactly as they would in a dashboard. Co-Authored-By: Claude Opus 5 --- README.md | 36 +- k8s/grafana/dashboards/kustomization.yaml | 1 - k8s/grafana/dashboards/obs-traces.json | 942 ---------------------- 3 files changed, 16 insertions(+), 963 deletions(-) delete mode 100644 k8s/grafana/dashboards/obs-traces.json diff --git a/README.md b/README.md index b9831dd..1e6987a 100644 --- a/README.md +++ b/README.md @@ -215,25 +215,13 @@ indicators, HTTP traffic, a runtime section specific to the stack in use, and a ### Distributed tracing -`obs-traces` searches traces with TraceQL, assembled from a service selector, a -minimum duration, a status, and a free-text field for conditions the selectors -do not cover. The status and free-text variables hold TraceQL *fragments* rather -than values, so an empty one contributes nothing and the surrounding query stays -valid — "no filter" is the default rather than a special case. +Traces are searched and explored in **Explore**, on the Tempo datasource, not +from a dashboard. Explore's TraceQL query builder, autocompletion, trace view +and interactive service graph have no dashboard equivalent worth maintaining — +a dashboard search panel is a worse copy of a tool that already exists. -Opening a trace gives the waterfall, every span's attributes and events, and -from a span its logs and its RED metrics. The rest of the dashboard is the -aggregate view: rate, errors and latency percentiles from the span metrics -Tempo's own `metrics_generator` derives from the traces, the slowest and most -failing operations by span name, and the service-to-service edges Tempo infers. - -Span name rather than endpoint is the unit worth looking at: a slow database -statement surfaces next to a slow HTTP handler, which is what tracing buys over -plain HTTP metrics. - -The interactive node graph stays in Explore — `serviceMap` is a query type -Grafana's frontend assembles and it returns nothing through the dashboard query -API, so the dashboard shows the same edges as a table instead. +What the repository does own is the datasource wiring, which applies in Explore +exactly as it would in a dashboard. #### Span-to-logs correlation @@ -254,10 +242,18 @@ Coverage today, each verified by taking a trace id and searching for it: | classic-models-api | yes | inbound `traceparent` in the gunicorn access line | | LibreChat | no | would need the upstream image patched | -`tracesToMetrics` likewise had no queries defined and rendered a bare datasource -link; it now carries rate, error-rate and p95 queries against the span metrics, +#### Span-to-metrics correlation + +`tracesToMetrics` had no queries defined and rendered a bare datasource link. It +now carries rate, error-rate and p95 queries against `traces_spanmetrics_*`, scoped to the span's own service. +Those span metrics are worth knowing about independently: Tempo's +`metrics_generator` derives them from the traces and remote-writes them to +Prometheus, keyed by `service` and `span_name`. They measure the *span*, so a +slow database statement appears next to a slow HTTP handler — a granularity the +gateway and per-application dashboards, which measure HTTP, do not have. + ### Log dashboards, split by kind Logs are searched from three dashboards rather than one, because the three diff --git a/k8s/grafana/dashboards/kustomization.yaml b/k8s/grafana/dashboards/kustomization.yaml index 9d74f08..c8f2e6f 100644 --- a/k8s/grafana/dashboards/kustomization.yaml +++ b/k8s/grafana/dashboards/kustomization.yaml @@ -25,7 +25,6 @@ configMapGenerator: - obs-kong-gateways.json - obs-librechat.json - obs-stack-health.json - - obs-traces.json # <<< dashboards <<< generatorOptions: diff --git a/k8s/grafana/dashboards/obs-traces.json b/k8s/grafana/dashboards/obs-traces.json deleted file mode 100644 index dcedba5..0000000 --- a/k8s/grafana/dashboards/obs-traces.json +++ /dev/null @@ -1,942 +0,0 @@ -{ - "uid": "obs-traces", - "title": "Distributed Traces", - "description": "Search traces with TraceQL, then open one to see its spans, their attributes, and the logs and metrics belonging to each span. RED figures come from the span metrics Tempo derives from the traces themselves.", - "tags": [ - "observability", - "traces", - "tempo" - ], - "editable": false, - "schemaVersion": 39, - "version": 1, - "refresh": "", - "time": { - "from": "now-1h", - "to": "now" - }, - "timezone": "browser", - "templating": { - "list": [ - { - "name": "service", - "label": "Service", - "type": "query", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "query": { - "type": 1, - "refId": "service-variable" - }, - "refresh": 2, - "sort": 1, - "multi": true, - "includeAll": true, - "allValue": ".+", - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "description": "Service names as Tempo knows them." - }, - { - "name": "minDuration", - "label": "Min duration", - "type": "custom", - "query": "0ms,10ms,50ms,100ms,500ms,1s,5s", - "current": { - "text": "0ms", - "value": "0ms" - }, - "multi": false, - "includeAll": false - }, - { - "name": "status", - "label": "Status", - "type": "custom", - "query": "Any : ,Error only : && status = error,OK only : && status = ok", - "current": { - "text": "Any", - "value": " " - }, - "multi": false, - "includeAll": false - }, - { - "name": "extra", - "label": "TraceQL", - "type": "textbox", - "query": "", - "current": { - "text": "", - "value": "" - }, - "description": "Extra TraceQL conditions, appended inside the braces. Example: && span.http.status_code = 500" - } - ] - }, - "panels": [ - { - "id": 1, - "type": "row", - "title": "Find traces", - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "panels": [] - }, - { - "id": 2, - "type": "table", - "title": "Trace search", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "gridPos": { - "h": 13, - "w": 24, - "x": 0, - "y": 1 - }, - "description": "TraceQL, assembled from the selectors above. Click a row to open the trace: the waterfall, every span's attributes and events, and — from a span — its logs and its RED metrics, wired on the Tempo datasource. The time range is the dashboard's, so narrowing the picker narrows the search.", - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{ resource.service.name =~ \"${service:regex}\" && duration >= ${minDuration} ${status} ${extra} }", - "limit": 100, - "tableType": "traces" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": true, - "filterable": true - }, - "mappings": [] - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "traceIdHidden" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "traceDuration" - }, - "properties": [ - { - "id": "unit", - "value": "ms" - }, - { - "id": "custom.width", - "value": 120 - } - ] - } - ] - }, - "options": { - "showHeader": true, - "cellHeight": "sm", - "footer": { - "show": false, - "reducer": [ - "sum" - ], - "countRows": false, - "fields": "" - }, - "sortBy": [ - { - "displayName": "Start time", - "desc": true - } - ] - } - }, - { - "id": 3, - "type": "row", - "title": "Rate, errors, duration (from span metrics)", - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "panels": [] - }, - { - "id": 4, - "type": "stat", - "title": "Spans / s", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 15 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "sum(rate(traces_spanmetrics_calls_total{service=~\"${service:regex}\"}[$__rate_interval]))" - } - ], - "fieldConfig": { - "defaults": { - "unit": "reqps", - "decimals": 2, - "mappings": [], - "color": { - "mode": "fixed", - "fixedColor": "blue" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "colorMode": "value", - "graphMode": "area", - "textMode": "auto" - }, - "description": "Derived by Tempo's metrics_generator from the traces themselves, then remote-written to Prometheus — no extra instrumentation in the applications." - }, - { - "id": 5, - "type": "stat", - "title": "Error rate", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 15 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "(sum(rate(traces_spanmetrics_calls_total{service=~\"${service:regex}\",status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval])) or vector(0)) / clamp_min(sum(rate(traces_spanmetrics_calls_total{service=~\"${service:regex}\"}[$__rate_interval])), 0.001)" - } - ], - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "decimals": 2, - "mappings": [], - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.01 - }, - { - "color": "red", - "value": 0.05 - } - ] - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "colorMode": "value", - "graphMode": "area", - "textMode": "auto" - }, - "description": "Share of spans whose status is STATUS_CODE_ERROR." - }, - { - "id": 6, - "type": "stat", - "title": "Latency p95", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 15 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "histogram_quantile(0.95, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=~\"${service:regex}\"}[$__rate_interval])))" - } - ], - "fieldConfig": { - "defaults": { - "unit": "ms", - "decimals": 0, - "mappings": [], - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 500 - }, - { - "color": "red", - "value": 2000 - } - ] - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "colorMode": "value", - "graphMode": "area", - "textMode": "auto" - } - }, - { - "id": 7, - "type": "stat", - "title": "Services traced", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 15 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "count(group by (service) (traces_spanmetrics_calls_total{service=~\"${service:regex}\"}))" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "mappings": [], - "color": { - "mode": "fixed", - "fixedColor": "purple" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "colorMode": "value", - "graphMode": "area", - "textMode": "auto" - } - }, - { - "id": 8, - "type": "timeseries", - "title": "Span rate by service", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 19 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "sum by (service) (rate(traces_spanmetrics_calls_total{service=~\"${service:regex}\"}[$__rate_interval]))", - "legendFormat": "{{service}}" - } - ], - "fieldConfig": { - "defaults": { - "unit": "reqps", - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 8, - "showPoints": "never", - "pointSize": 8, - "gradientMode": "none", - "spanNulls": false, - "axisLabel": "", - "axisPlacement": "auto", - "lineInterpolation": "smooth" - }, - "mappings": [], - "color": { - "mode": "palette-classic" - }, - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "showLegend": true, - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } - }, - { - "id": 9, - "type": "timeseries", - "title": "Latency percentiles", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 19 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "histogram_quantile(0.50, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=~\"${service:regex}\"}[$__rate_interval])))", - "legendFormat": "p50" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "histogram_quantile(0.95, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=~\"${service:regex}\"}[$__rate_interval])))", - "legendFormat": "p95" - }, - { - "refId": "C", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "histogram_quantile(0.99, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=~\"${service:regex}\"}[$__rate_interval])))", - "legendFormat": "p99" - } - ], - "fieldConfig": { - "defaults": { - "unit": "ms", - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 0, - "showPoints": "never", - "pointSize": 8, - "gradientMode": "none", - "spanNulls": false, - "axisLabel": "", - "axisPlacement": "auto", - "lineInterpolation": "smooth" - }, - "mappings": [], - "color": { - "mode": "palette-classic" - }, - "min": 0 - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "p50" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "light-blue" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "p95" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "blue" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "p99" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "dark-blue" - } - } - ] - } - ] - }, - "options": { - "legend": { - "showLegend": true, - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "description": "One hue light to dark: ordered magnitudes of the same measure, not distinct entities." - }, - { - "id": 10, - "type": "bargauge", - "title": "Slowest operations (p95)", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 27 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "topk(10, histogram_quantile(0.95, sum by (service, span_name, le) (rate(traces_spanmetrics_latency_bucket{service=~\"${service:regex}\"}[$__rate_interval]))))", - "legendFormat": "{{service}} · {{span_name}}", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "unit": "ms", - "min": 0, - "mappings": [], - "color": { - "mode": "continuous-BlPu" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "displayMode": "gradient", - "orientation": "horizontal", - "showUnfilled": true, - "valueMode": "text", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "description": "Span name, not endpoint: a slow database statement surfaces here next to a slow HTTP handler, which is the point of tracing over plain HTTP metrics." - }, - { - "id": 11, - "type": "bargauge", - "title": "Operations in error", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 27 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "topk(10, sum by (service, span_name) (rate(traces_spanmetrics_calls_total{service=~\"${service:regex}\",status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval])))", - "legendFormat": "{{service}} · {{span_name}}", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "unit": "reqps", - "min": 0, - "mappings": [], - "color": { - "mode": "continuous-BlPu" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - } - }, - "overrides": [] - }, - "options": { - "displayMode": "gradient", - "orientation": "horizontal", - "showUnfilled": true, - "valueMode": "text", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "description": "Empty is the healthy state." - }, - { - "id": 12, - "type": "row", - "title": "Service dependencies", - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 36 - }, - "panels": [] - }, - { - "id": 13, - "type": "table", - "title": "Calls between services", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 37 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "sum by (client, server) (rate(traces_service_graph_request_total[$__rate_interval]))", - "format": "table", - "instant": true - } - ], - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "__name__": true - }, - "indexByName": { - "client": 0, - "server": 1, - "Value": 2 - }, - "renameByName": { - "client": "Caller", - "server": "Callee", - "Value": "Requests/s" - } - } - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": true, - "filterable": true - }, - "mappings": [] - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Requests/s" - }, - "properties": [ - { - "id": "unit", - "value": "reqps" - }, - { - "id": "decimals", - "value": 3 - } - ] - } - ] - }, - "options": { - "showHeader": true, - "cellHeight": "sm", - "footer": { - "show": false, - "reducer": [ - "sum" - ], - "countRows": false, - "fields": "" - }, - "sortBy": [ - { - "displayName": "Requests/s", - "desc": true - } - ] - }, - "description": "Edges Tempo infers from the traces themselves — no service registry involved. `user` as caller means the span had no parent, so the call entered the system there. The interactive node graph lives in Explore, on the Tempo datasource's Service Graph tab." - }, - { - "id": 14, - "type": "timeseries", - "title": "Latency between services", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 37 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "histogram_quantile(0.95, sum by (client, server, le) (rate(traces_service_graph_request_server_seconds_bucket[$__rate_interval])))", - "legendFormat": "{{client}} → {{server}}" - } - ], - "fieldConfig": { - "defaults": { - "unit": "s", - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 0, - "showPoints": "never", - "pointSize": 8, - "gradientMode": "none", - "spanNulls": false, - "axisLabel": "", - "axisPlacement": "auto", - "lineInterpolation": "smooth" - }, - "mappings": [], - "color": { - "mode": "palette-classic" - }, - "min": 0 - }, - "overrides": [] - }, - "options": { - "legend": { - "showLegend": true, - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "description": "p95 as measured on the server side of each edge." - } - ] -}