{ "uid": "obs-traces", "title": "Distributed Traces", "description": "Search traces with TraceQL, then open one to see its spans, their attributes, and the logs and metrics belonging to each span. RED figures come from the span metrics Tempo derives from the traces themselves.", "tags": [ "observability", "traces", "tempo" ], "editable": false, "schemaVersion": 39, "version": 1, "refresh": "", "time": { "from": "now-1h", "to": "now" }, "timezone": "browser", "templating": { "list": [ { "name": "service", "label": "Service", "type": "query", "datasource": { "type": "tempo", "uid": "tempo" }, "query": { "type": 1, "refId": "service-variable" }, "refresh": 2, "sort": 1, "multi": true, "includeAll": true, "allValue": ".+", "current": { "text": [ "All" ], "value": [ "$__all" ] }, "description": "Service names as Tempo knows them." }, { "name": "minDuration", "label": "Min duration", "type": "custom", "query": "0ms,10ms,50ms,100ms,500ms,1s,5s", "current": { "text": "0ms", "value": "0ms" }, "multi": false, "includeAll": false }, { "name": "status", "label": "Status", "type": "custom", "query": "Any : ,Error only : && status = error,OK only : && status = ok", "current": { "text": "Any", "value": " " }, "multi": false, "includeAll": false }, { "name": "extra", "label": "TraceQL", "type": "textbox", "query": "", "current": { "text": "", "value": "" }, "description": "Extra TraceQL conditions, appended inside the braces. Example: && span.http.status_code = 500" } ] }, "panels": [ { "id": 1, "type": "row", "title": "Find traces", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "panels": [] }, { "id": 2, "type": "table", "title": "Trace search", "datasource": { "type": "tempo", "uid": "tempo" }, "gridPos": { "h": 13, "w": 24, "x": 0, "y": 1 }, "description": "TraceQL, assembled from the selectors above. Click a row to open the trace: the waterfall, every span's attributes and events, and — from a span — its logs and its RED metrics, wired on the Tempo datasource. The time range is the dashboard's, so narrowing the picker narrows the search.", "targets": [ { "refId": "A", "datasource": { "type": "tempo", "uid": "tempo" }, "queryType": "traceql", "query": "{ resource.service.name =~ \"${service:regex}\" && duration >= ${minDuration} ${status} ${extra} }", "limit": 100, "tableType": "traces" } ], "fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": true, "filterable": true }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byName", "options": "traceIdHidden" }, "properties": [ { "id": "custom.hidden", "value": true } ] }, { "matcher": { "id": "byName", "options": "traceDuration" }, "properties": [ { "id": "unit", "value": "ms" }, { "id": "custom.width", "value": 120 } ] } ] }, "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false, "reducer": [ "sum" ], "countRows": false, "fields": "" }, "sortBy": [ { "displayName": "Start time", "desc": true } ] } }, { "id": 3, "type": "row", "title": "Rate, errors, duration (from span metrics)", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, "panels": [] }, { "id": 4, "type": "stat", "title": "Spans / s", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 4, "w": 6, "x": 0, "y": 15 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(traces_spanmetrics_calls_total{service=~\"${service:regex}\"}[$__rate_interval]))" } ], "fieldConfig": { "defaults": { "unit": "reqps", "decimals": 2, "mappings": [], "color": { "mode": "fixed", "fixedColor": "blue" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "textMode": "auto" }, "description": "Derived by Tempo's metrics_generator from the traces themselves, then remote-written to Prometheus — no extra instrumentation in the applications." }, { "id": 5, "type": "stat", "title": "Error rate", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 4, "w": 6, "x": 6, "y": 15 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "(sum(rate(traces_spanmetrics_calls_total{service=~\"${service:regex}\",status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval])) or vector(0)) / clamp_min(sum(rate(traces_spanmetrics_calls_total{service=~\"${service:regex}\"}[$__rate_interval])), 0.001)" } ], "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "mappings": [], "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 0.01 }, { "color": "red", "value": 0.05 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "textMode": "auto" }, "description": "Share of spans whose status is STATUS_CODE_ERROR." }, { "id": 6, "type": "stat", "title": "Latency p95", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 4, "w": 6, "x": 12, "y": 15 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=~\"${service:regex}\"}[$__rate_interval])))" } ], "fieldConfig": { "defaults": { "unit": "ms", "decimals": 0, "mappings": [], "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 500 }, { "color": "red", "value": 2000 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "textMode": "auto" } }, { "id": 7, "type": "stat", "title": "Services traced", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 4, "w": 6, "x": 18, "y": 15 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "count(group by (service) (traces_spanmetrics_calls_total{service=~\"${service:regex}\"}))" } ], "fieldConfig": { "defaults": { "unit": "short", "decimals": 0, "mappings": [], "color": { "mode": "fixed", "fixedColor": "purple" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "purple", "value": null } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "textMode": "auto" } }, { "id": 8, "type": "timeseries", "title": "Span rate by service", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (service) (rate(traces_spanmetrics_calls_total{service=~\"${service:regex}\"}[$__rate_interval]))", "legendFormat": "{{service}}" } ], "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 8, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 9, "type": "timeseries", "title": "Latency percentiles", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=~\"${service:regex}\"}[$__rate_interval])))", "legendFormat": "p50" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=~\"${service:regex}\"}[$__rate_interval])))", "legendFormat": "p95" }, { "refId": "C", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=~\"${service:regex}\"}[$__rate_interval])))", "legendFormat": "p99" } ], "fieldConfig": { "defaults": { "unit": "ms", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [ { "matcher": { "id": "byName", "options": "p50" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "light-blue" } } ] }, { "matcher": { "id": "byName", "options": "p95" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } } ] }, { "matcher": { "id": "byName", "options": "p99" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "dark-blue" } } ] } ] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "One hue light to dark: ordered magnitudes of the same measure, not distinct entities." }, { "id": 10, "type": "bargauge", "title": "Slowest operations (p95)", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 27 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "topk(10, histogram_quantile(0.95, sum by (service, span_name, le) (rate(traces_spanmetrics_latency_bucket{service=~\"${service:regex}\"}[$__rate_interval]))))", "legendFormat": "{{service}} · {{span_name}}", "instant": true } ], "fieldConfig": { "defaults": { "unit": "ms", "min": 0, "mappings": [], "color": { "mode": "continuous-BlPu" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] } }, "overrides": [] }, "options": { "displayMode": "gradient", "orientation": "horizontal", "showUnfilled": true, "valueMode": "text", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "description": "Span name, not endpoint: a slow database statement surfaces here next to a slow HTTP handler, which is the point of tracing over plain HTTP metrics." }, { "id": 11, "type": "bargauge", "title": "Operations in error", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 27 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "topk(10, sum by (service, span_name) (rate(traces_spanmetrics_calls_total{service=~\"${service:regex}\",status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval])))", "legendFormat": "{{service}} · {{span_name}}", "instant": true } ], "fieldConfig": { "defaults": { "unit": "reqps", "min": 0, "mappings": [], "color": { "mode": "continuous-BlPu" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] } }, "overrides": [] }, "options": { "displayMode": "gradient", "orientation": "horizontal", "showUnfilled": true, "valueMode": "text", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "description": "Empty is the healthy state." }, { "id": 12, "type": "row", "title": "Service dependencies", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 36 }, "panels": [] }, { "id": 13, "type": "table", "title": "Calls between services", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 37 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (client, server) (rate(traces_service_graph_request_total[$__rate_interval]))", "format": "table", "instant": true } ], "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true }, "indexByName": { "client": 0, "server": 1, "Value": 2 }, "renameByName": { "client": "Caller", "server": "Callee", "Value": "Requests/s" } } } ], "fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": true, "filterable": true }, "mappings": [] }, "overrides": [ { "matcher": { "id": "byName", "options": "Requests/s" }, "properties": [ { "id": "unit", "value": "reqps" }, { "id": "decimals", "value": 3 } ] } ] }, "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false, "reducer": [ "sum" ], "countRows": false, "fields": "" }, "sortBy": [ { "displayName": "Requests/s", "desc": true } ] }, "description": "Edges Tempo infers from the traces themselves — no service registry involved. `user` as caller means the span had no parent, so the call entered the system there. The interactive node graph lives in Explore, on the Tempo datasource's Service Graph tab." }, { "id": 14, "type": "timeseries", "title": "Latency between services", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 37 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.95, sum by (client, server, le) (rate(traces_service_graph_request_server_seconds_bucket[$__rate_interval])))", "legendFormat": "{{client}} → {{server}}" } ], "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "pointSize": 8, "gradientMode": "none", "spanNulls": false, "axisLabel": "", "axisPlacement": "auto", "lineInterpolation": "smooth" }, "mappings": [], "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "description": "p95 as measured on the server side of each edge." } ] }