stt/infra-observability
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Select traces by component count, not by service

Browse Source 
The first version filtered the trace list by service name, which does not
remove the noise it was built to remove: Keycloak token calls and gateway
401s are services too, so with Service=All the panel showed exactly the
health-check traffic that makes Explore unusable, and the real LibreChat
transaction was nowhere in the first 50 rows.

Count SERVER spans instead. A token call has one, a health probe has one,
a gateway 401 has two; a real end-to-end transaction has four. So
{kind=server} | count() > 2 isolates them, and does it without naming a
single service — nothing to edit when an agent or a domain is added.
Verified against live data: it returns the LibreChat chat transactions
and nothing else.

The span panels were wrong in the same way: they showed the spans of
every trace at once, so the two tables were filled with unrelated
gateways. They now scope to {trace:id="$traceId"}, set by clicking a row
above, and sit empty until one is picked — which is the honest state, not
a wall of noise.

The service variable is gone; the trace filter is a named dropdown
instead, with the count-based default and a couple of looser options.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Stéphane Tailland
2026-08-27 07:53:47 +02:00
parent c95af8209e
commit 3988df4916
1 changed files with 56 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
k8s/grafana/dashboards/obs-distributed-traces.json
+54 -57
View File
@@ -1,7 +1,7 @@
{ {
"uid": "obs-distributed-traces", "uid": "obs-distributed-traces",
"title": "Distributed Traces", "title": "Distributed Traces",
"description": "Find a trace, and keep only the spans worth looking at. Built because Explore's unfiltered search is unusable here: Tempo does not return {} newest-first, so health checks crowd out real transactions.\n\nDeliberately carries no metric panels. A span's own duration is in the span tables, measured; aggregate RED by span name is what the per-application dashboards already show, and neither answers 'the metrics of this span'.", "description": "Find a real end-to-end transaction, then see only the spans worth looking at and the logs that go with it.\n\nBuilt because Explore's unfiltered search is unusable here: Tempo does not return {} newest-first, so raising the limit does not walk backwards in time, and health checks crowd real transactions out of the results.",
"tags": [ "tags": [
"observability", "observability",
"tracing", "tracing",
@@ -16,34 +16,43 @@
"to": "now" "to": "now"
}, },
"editable": true, "editable": true,
"graphTooltip": 1, "graphTooltip": 0,
"templating": { "templating": {
"list": [ "list": [
{ {
"name": "service", "name": "traceFilter",
"label": "Service", "label": "Trace filter",
"type": "query", "type": "custom",
"datasource": { "query": "End-to-end — 3+ components : {kind=server} | count() > 2,Multi-component — 2+ : {kind=server} | count() > 1,Crossing a Kong gateway : {name=~\"kong\\\\..*\"} && {kind=server && resource.service.name!~\".*-dev\"},Everything (noisy) : {}",
"type": "prometheus", "options": [
"uid": "prometheus" {
"text": "End-to-end — 3+ components",
"value": "{kind=server} | count() > 2",
"selected": true
}, },
"query": { {
"query": "label_values(traces_spanmetrics_calls_total, service)", "text": "Multi-component — 2+",
"refId": "service" "value": "{kind=server} | count() > 1",
"selected": false
}, },
"refresh": 2, {
"multi": true, "text": "Crossing a Kong gateway",
"includeAll": true, "value": "{name=~\"kong\\\\..*\"} && {kind=server && resource.service.name!~\".*-dev\"}",
"allValue": ".+", "selected": false
"current": { },
"text": [ {
"All" "text": "Everything (noisy)",
"value": "{}",
"selected": false
}
], ],
"value": [ "current": {
"$__all" "text": "End-to-end — 3+ components",
] "value": "{kind=server} | count() > 2"
}, },
"description": "Services known to Tempo's span metrics. Drives the trace, span and metric panels." "includeAll": false,
"multi": false,
"description": "Which traces count as interesting. Counting SERVER spans is what separates real transactions from token calls and health probes."
}, },
{ {
"name": "traceId", "name": "traceId",
@@ -54,7 +63,7 @@
"text": "", "text": "",
"value": "" "value": ""
}, },
"description": "Set by clicking a Trace ID in the top table, or paste one. Empty means the log filter matches everything." "description": "Set by clicking a Trace ID above, or paste one. Empty leaves the span panels empty."
} }
] ]
}, },
@@ -73,14 +82,14 @@
}, },
{ {
"type": "table", "type": "table",
"title": "Traces on the dashboard time range", "title": "End-to-end transactions",
"description": "TraceQL over the dashboard's time range. Filtered by service so the health-check noise stays out: an unfiltered {} query is NOT returned newest-first by Tempo and the limit truncates arbitrarily, which is why browsing raw Explore misses recent traces. Click a Trace ID to load it below.", "description": "Default filter is {kind=server} | count() > 2 — traces where at least three components received an HTTP request. That is what an end-to-end transaction is here, and it needs no service name hardcoded.\n\nIt exists because filtering by service does NOT remove the noise: Keycloak token calls and gateway 401s are services too. Counting server spans does — a token call has one, a health probe has one, a real LibreChat transaction has four.\n\nClick a Trace ID: the panels below scope to it.",
"datasource": { "datasource": {
"type": "tempo", "type": "tempo",
"uid": "tempo" "uid": "tempo"
}, },
"gridPos": { "gridPos": {
"h": 9, "h": 10,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 1 "y": 1
@@ -93,10 +102,10 @@
"uid": "tempo" "uid": "tempo"
}, },
"queryType": "traceql", "queryType": "traceql",
"query": "{resource.service.name=~\"$service\"}", "query": "$traceFilter",
"tableType": "traces", "tableType": "traces",
"limit": 50, "limit": 50,
"spss": 20 "spss": 50
} }
], ],
"fieldConfig": { "fieldConfig": {
@@ -107,18 +116,6 @@
} }
}, },
"overrides": [ "overrides": [
{
"matcher": {
"id": "byName",
"options": "traceIdHidden"
},
"properties": [
{
"id": "custom.hidden",
"value": true
}
]
},
{ {
"matcher": { "matcher": {
"id": "byName", "id": "byName",
@@ -129,8 +126,8 @@
"id": "links", "id": "links",
"value": [ "value": [
{ {
"title": "Load this trace below", "title": "Scope the panels below to this trace",
"url": "d/obs-distributed-traces/distributed-traces?${__url_time_range}&var-traceId=${__value.raw}&var-service=${service:queryparam}" "url": "d/obs-distributed-traces/distributed-traces?${__url_time_range}&var-traceId=${__value.raw}&var-traceFilter=${traceFilter:queryparam}"
} }
] ]
} }
@@ -150,12 +147,12 @@
}, },
{ {
"type": "row", "type": "row",
"title": "Spans that matter", "title": "Spans of the selected trace",
"gridPos": { "gridPos": {
"h": 1, "h": 1,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 10 "y": 11
}, },
"collapsed": false, "collapsed": false,
"panels": [] "panels": []
@@ -163,7 +160,7 @@
{ {
"type": "table", "type": "table",
"title": "HTTP hops between components", "title": "HTTP hops between components",
"description": "One row per component entry: these are the SERVER-kind spans, i.e. the inbound HTTP request each service received. Browser to LibreChat, LibreChat to the llm gateway, gateway to a domain gateway, gateway to the backend. Everything else — express middleware, DNS, SQL — is deliberately excluded.", "description": "The SERVER-kind spans of the selected trace: one row per component that received the request. Browser to LibreChat, LibreChat to the llm gateway, gateway to a domain gateway, gateway to the backend. Express middleware, DNS and SQL spans are excluded on purpose.\n\nEmpty until a Trace ID is selected above.",
"datasource": { "datasource": {
"type": "tempo", "type": "tempo",
"uid": "tempo" "uid": "tempo"
@@ -172,7 +169,7 @@
"h": 10, "h": 10,
"w": 12, "w": 12,
"x": 0, "x": 0,
"y": 11 "y": 12
}, },
"targets": [ "targets": [
{ {
@@ -182,10 +179,10 @@
"uid": "tempo" "uid": "tempo"
}, },
"queryType": "traceql", "queryType": "traceql",
"query": "{resource.service.name=~\"$service\" && kind=server} | select(span.http.method, span.http.status_code, span.http.route, span.http.url)", "query": "{trace:id=\"$traceId\" && kind=server} | select(span.http.method, span.http.status_code, span.http.route, span.http.url)",
"tableType": "spans", "tableType": "spans",
"limit": 50, "limit": 50,
"spss": 20 "spss": 50
} }
], ],
"fieldConfig": { "fieldConfig": {
@@ -204,7 +201,7 @@
{ {
"type": "table", "type": "table",
"title": "Kong plugin execution", "title": "Kong plugin execution",
"description": "Every plugin Kong ran, with its own duration — openid-connect, rate-limiting, request-transformer, opentelemetry, and so on. This is what shows which plugin is costing latency on a gateway hop.", "description": "Every plugin Kong ran on the selected trace, with its own duration — openid-connect, rate-limiting, the ai-* plugins, opentelemetry. This is what shows which plugin costs latency on a gateway hop.\n\nEmpty until a Trace ID is selected above.",
"datasource": { "datasource": {
"type": "tempo", "type": "tempo",
"uid": "tempo" "uid": "tempo"
@@ -213,7 +210,7 @@
"h": 10, "h": 10,
"w": 12, "w": 12,
"x": 12, "x": 12,
"y": 11 "y": 12
}, },
"targets": [ "targets": [
{ {
@@ -223,10 +220,10 @@
"uid": "tempo" "uid": "tempo"
}, },
"queryType": "traceql", "queryType": "traceql",
"query": "{resource.service.name=~\"$service\" && name=~\"kong\\\\..*plugin.*\"}", "query": "{trace:id=\"$traceId\" && name=~\"kong\\\\..*plugin.*\"}",
"tableType": "spans", "tableType": "spans",
"limit": 50, "limit": 50,
"spss": 20 "spss": 50
} }
], ],
"fieldConfig": { "fieldConfig": {
@@ -244,12 +241,12 @@
}, },
{ {
"type": "row", "type": "row",
"title": "Logs for the selected trace", "title": "Logs of the selected trace",
"gridPos": { "gridPos": {
"h": 1, "h": 1,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 21 "y": 22
}, },
"collapsed": false, "collapsed": false,
"panels": [] "panels": []
@@ -257,16 +254,16 @@
{ {
"type": "logs", "type": "logs",
"title": "Logs carrying trace $traceId", "title": "Logs carrying trace $traceId",
"description": "Correlation is by TRACE id, not span id: none of the emitters log a span id, so this is every log line of the whole trace rather than of one span.\n\nCoverage is uneven and worth knowing: kong-audit and the backend's access log carry the id on every line; the backend's application log does too since v1.0.4. LibreChat logs carry no trace id at all, so its lines never appear here.\n\nWith traceId empty the filter matches everything — paste an id or click one above.", "description": "Correlation is by TRACE id, not span id: no emitter logs a span id, so this is every line of the whole trace rather than of one span.\n\nCoverage is uneven: kong-audit and the backend's access log carry the id on every line, and the backend's application log does too since v1.0.4. LibreChat logs carry no trace id at all, so its lines never appear here.\n\nWith traceId empty the filter matches everything — pick a trace above.",
"datasource": { "datasource": {
"type": "loki", "type": "loki",
"uid": "loki" "uid": "loki"
}, },
"gridPos": { "gridPos": {
"h": 11, "h": 12,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 22 "y": 23
}, },
"targets": [ "targets": [
{ {