Stéphane Tailland
3988df4916
The first version filtered the trace list by service name, which does not
remove the noise it was built to remove: Keycloak token calls and gateway
401s are services too, so with Service=All the panel showed exactly the
health-check traffic that makes Explore unusable, and the real LibreChat
transaction was nowhere in the first 50 rows.
Count SERVER spans instead. A token call has one, a health probe has one,
a gateway 401 has two; a real end-to-end transaction has four. So
{kind=server} | count() > 2 isolates them, and does it without naming a
single service — nothing to edit when an agent or a domain is added.
Verified against live data: it returns the LibreChat chat transactions
and nothing else.
The span panels were wrong in the same way: they showed the spans of
every trace at once, so the two tables were filled with unrelated
gateways. They now scope to {trace:id="$traceId"}, set by clicking a row
above, and sit empty until one is picked — which is the honest state, not
a wall of noise.
The service variable is gone; the trace filter is a named dropdown
instead, with the count-based default and a couple of looser options.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
286 lines
8.3 KiB
JSON
286 lines
8.3 KiB
JSON
{
|
|
"uid": "obs-distributed-traces",
|
|
"title": "Distributed Traces",
|
|
"description": "Find a real end-to-end transaction, then see only the spans worth looking at and the logs that go with it.\n\nBuilt because Explore's unfiltered search is unusable here: Tempo does not return {} newest-first, so raising the limit does not walk backwards in time, and health checks crowd real transactions out of the results.",
|
|
"tags": [
|
|
"observability",
|
|
"tracing",
|
|
"tempo"
|
|
],
|
|
"timezone": "browser",
|
|
"schemaVersion": 39,
|
|
"version": 1,
|
|
"refresh": "",
|
|
"time": {
|
|
"from": "now-1h",
|
|
"to": "now"
|
|
},
|
|
"editable": true,
|
|
"graphTooltip": 0,
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "traceFilter",
|
|
"label": "Trace filter",
|
|
"type": "custom",
|
|
"query": "End-to-end — 3+ components : {kind=server} | count() > 2,Multi-component — 2+ : {kind=server} | count() > 1,Crossing a Kong gateway : {name=~\"kong\\\\..*\"} && {kind=server && resource.service.name!~\".*-dev\"},Everything (noisy) : {}",
|
|
"options": [
|
|
{
|
|
"text": "End-to-end — 3+ components",
|
|
"value": "{kind=server} | count() > 2",
|
|
"selected": true
|
|
},
|
|
{
|
|
"text": "Multi-component — 2+",
|
|
"value": "{kind=server} | count() > 1",
|
|
"selected": false
|
|
},
|
|
{
|
|
"text": "Crossing a Kong gateway",
|
|
"value": "{name=~\"kong\\\\..*\"} && {kind=server && resource.service.name!~\".*-dev\"}",
|
|
"selected": false
|
|
},
|
|
{
|
|
"text": "Everything (noisy)",
|
|
"value": "{}",
|
|
"selected": false
|
|
}
|
|
],
|
|
"current": {
|
|
"text": "End-to-end — 3+ components",
|
|
"value": "{kind=server} | count() > 2"
|
|
},
|
|
"includeAll": false,
|
|
"multi": false,
|
|
"description": "Which traces count as interesting. Counting SERVER spans is what separates real transactions from token calls and health probes."
|
|
},
|
|
{
|
|
"name": "traceId",
|
|
"label": "Trace ID",
|
|
"type": "textbox",
|
|
"query": "",
|
|
"current": {
|
|
"text": "",
|
|
"value": ""
|
|
},
|
|
"description": "Set by clicking a Trace ID above, or paste one. Empty leaves the span panels empty."
|
|
}
|
|
]
|
|
},
|
|
"panels": [
|
|
{
|
|
"type": "row",
|
|
"title": "Traces",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "table",
|
|
"title": "End-to-end transactions",
|
|
"description": "Default filter is {kind=server} | count() > 2 — traces where at least three components received an HTTP request. That is what an end-to-end transaction is here, and it needs no service name hardcoded.\n\nIt exists because filtering by service does NOT remove the noise: Keycloak token calls and gateway 401s are services too. Counting server spans does — a token call has one, a health probe has one, a real LibreChat transaction has four.\n\nClick a Trace ID: the panels below scope to it.",
|
|
"datasource": {
|
|
"type": "tempo",
|
|
"uid": "tempo"
|
|
},
|
|
"gridPos": {
|
|
"h": 10,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "tempo",
|
|
"uid": "tempo"
|
|
},
|
|
"queryType": "traceql",
|
|
"query": "$traceFilter",
|
|
"tableType": "traces",
|
|
"limit": 50,
|
|
"spss": 50
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"align": "auto",
|
|
"filterable": true
|
|
}
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "traceID"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "links",
|
|
"value": [
|
|
{
|
|
"title": "Scope the panels below to this trace",
|
|
"url": "d/obs-distributed-traces/distributed-traces?${__url_time_range}&var-traceId=${__value.raw}&var-traceFilter=${traceFilter:queryparam}"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"showHeader": true,
|
|
"sortBy": [
|
|
{
|
|
"displayName": "Start time",
|
|
"desc": true
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"type": "row",
|
|
"title": "Spans of the selected trace",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 11
|
|
},
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "table",
|
|
"title": "HTTP hops between components",
|
|
"description": "The SERVER-kind spans of the selected trace: one row per component that received the request. Browser to LibreChat, LibreChat to the llm gateway, gateway to a domain gateway, gateway to the backend. Express middleware, DNS and SQL spans are excluded on purpose.\n\nEmpty until a Trace ID is selected above.",
|
|
"datasource": {
|
|
"type": "tempo",
|
|
"uid": "tempo"
|
|
},
|
|
"gridPos": {
|
|
"h": 10,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 12
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "tempo",
|
|
"uid": "tempo"
|
|
},
|
|
"queryType": "traceql",
|
|
"query": "{trace:id=\"$traceId\" && kind=server} | select(span.http.method, span.http.status_code, span.http.route, span.http.url)",
|
|
"tableType": "spans",
|
|
"limit": 50,
|
|
"spss": 50
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"align": "auto",
|
|
"filterable": true
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"showHeader": true
|
|
}
|
|
},
|
|
{
|
|
"type": "table",
|
|
"title": "Kong plugin execution",
|
|
"description": "Every plugin Kong ran on the selected trace, with its own duration — openid-connect, rate-limiting, the ai-* plugins, opentelemetry. This is what shows which plugin costs latency on a gateway hop.\n\nEmpty until a Trace ID is selected above.",
|
|
"datasource": {
|
|
"type": "tempo",
|
|
"uid": "tempo"
|
|
},
|
|
"gridPos": {
|
|
"h": 10,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 12
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "tempo",
|
|
"uid": "tempo"
|
|
},
|
|
"queryType": "traceql",
|
|
"query": "{trace:id=\"$traceId\" && name=~\"kong\\\\..*plugin.*\"}",
|
|
"tableType": "spans",
|
|
"limit": 50,
|
|
"spss": 50
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"align": "auto",
|
|
"filterable": true
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"showHeader": true
|
|
}
|
|
},
|
|
{
|
|
"type": "row",
|
|
"title": "Logs of the selected trace",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 22
|
|
},
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "logs",
|
|
"title": "Logs carrying trace $traceId",
|
|
"description": "Correlation is by TRACE id, not span id: no emitter logs a span id, so this is every line of the whole trace rather than of one span.\n\nCoverage is uneven: kong-audit and the backend's access log carry the id on every line, and the backend's application log does too since v1.0.4. LibreChat logs carry no trace id at all, so its lines never appear here.\n\nWith traceId empty the filter matches everything — pick a trace above.",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 12,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 23
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{job=~\"kong-audit|classic-models/classic-models-api|librechat/librechat|keycloak/keycloak\"} |= \"$traceId\""
|
|
}
|
|
],
|
|
"options": {
|
|
"showTime": true,
|
|
"wrapLogMessage": true,
|
|
"sortOrder": "Ascending",
|
|
"enableLogDetails": true
|
|
}
|
|
}
|
|
]
|
|
} |