Stéphane Tailland
32995ff949
A chat transaction is 212 spans, and almost all of them are Express middleware, mongoose and mongodb calls, kong.dns lookups and raw SQL. The flow — LibreChat to the gateway to the backend — is buried in it. Set a default span filter on the panel: spanNameOperator "!~" against those names with matchesOnly, so the view opens on the spans that describe the exchange. "Show all spans" and clearing the filter both bring them back. Deliberately a display filter, not a source change. The spans are still produced and still stored; only this panel's default view is quieter. It is also the only lever that covers the LibreChat ones: LibreChat does not use auto-instrumentations-node — it registers express, mongodb and mongoose individually in its own code — so no environment variable disables them, and they are the bulk of the 212. The option names were checked against Grafana's own traces panel bundle rather than assumed: spanName, spanNameOperator, serviceName, serviceNameOperator, matchesOnly and criticalPathOnly all exist there, and "!~" is among the operators it offers. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
469 lines
14 KiB
JSON
469 lines
14 KiB
JSON
{
|
|
"uid": "obs-distributed-traces",
|
|
"title": "Distributed Traces",
|
|
"description": "Follow one transaction end to end: pick it, read every span in order, see which tokens were issued, and read what was exchanged at each hop.",
|
|
"tags": [
|
|
"observability",
|
|
"tracing",
|
|
"tempo"
|
|
],
|
|
"timezone": "browser",
|
|
"schemaVersion": 39,
|
|
"version": 1,
|
|
"refresh": "",
|
|
"time": {
|
|
"from": "now-1h",
|
|
"to": "now"
|
|
},
|
|
"editable": true,
|
|
"graphTooltip": 0,
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "traceFilter",
|
|
"label": "Show",
|
|
"type": "custom",
|
|
"query": "LibreChat transactions : {trace:rootService=\"librechat\"} && {resource.service.name!=\"librechat\"},Reaching a Kong gateway : {trace:rootService=\"librechat\"} && {name=~\"kong\\\\..*\"},All LibreChat traces (incl. /metrics scrapes) : {trace:rootService=\"librechat\"},Everything (very noisy) : {}",
|
|
"options": [
|
|
{
|
|
"text": "LibreChat transactions",
|
|
"value": "{trace:rootService=\"librechat\"} && {resource.service.name!=\"librechat\"}",
|
|
"selected": true
|
|
},
|
|
{
|
|
"text": "Reaching a Kong gateway",
|
|
"value": "{trace:rootService=\"librechat\"} && {name=~\"kong\\\\..*\"}",
|
|
"selected": false
|
|
},
|
|
{
|
|
"text": "All LibreChat traces (incl. /metrics scrapes)",
|
|
"value": "{trace:rootService=\"librechat\"}",
|
|
"selected": false
|
|
},
|
|
{
|
|
"text": "Everything (very noisy)",
|
|
"value": "{}",
|
|
"selected": false
|
|
}
|
|
],
|
|
"current": {
|
|
"text": "LibreChat transactions",
|
|
"value": "{trace:rootService=\"librechat\"} && {resource.service.name!=\"librechat\"}"
|
|
},
|
|
"includeAll": false,
|
|
"multi": false,
|
|
"description": "Traces started BY LibreChat and leaving it. Rooting on LibreChat is what makes this a user-facing transaction rather than infrastructure chatter; requiring a second service is what drops Prometheus scraping LibreChat's own /metrics, which otherwise fills the list."
|
|
},
|
|
{
|
|
"name": "traceId",
|
|
"label": "Trace ID",
|
|
"type": "textbox",
|
|
"query": "",
|
|
"current": {
|
|
"text": "",
|
|
"value": ""
|
|
},
|
|
"description": "Set by clicking a Trace ID above, or paste one."
|
|
},
|
|
{
|
|
"name": "userId",
|
|
"label": "User ID",
|
|
"type": "textbox",
|
|
"query": "",
|
|
"current": {
|
|
"text": "",
|
|
"value": ""
|
|
},
|
|
"description": "Set by clicking an identity in the panel below. Empty means the Keycloak panel is scoped only by time."
|
|
}
|
|
]
|
|
},
|
|
"panels": [
|
|
{
|
|
"type": "row",
|
|
"title": "1 — Pick a transaction",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "table",
|
|
"title": "Transactions",
|
|
"description": "Traces LibreChat started that reached another service — the user-facing transactions.\n\nCounting spans was the previous approach and it let LibreChat's own /metrics scrapes through: with 50 of them in six hours they crowded out the two real conversations. Rooting on the service that starts a transaction is both simpler and stricter.\n\nThe trade: anything not entering through LibreChat is invisible here — a direct call to a gateway, or a curl against an API. Use 'Everything' for those.\n\nClick a Trace ID to load it below.",
|
|
"datasource": {
|
|
"type": "tempo",
|
|
"uid": "tempo"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "tempo",
|
|
"uid": "tempo"
|
|
},
|
|
"queryType": "traceql",
|
|
"query": "$traceFilter",
|
|
"tableType": "traces",
|
|
"limit": 50
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"align": "auto",
|
|
"filterable": true
|
|
}
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "traceID"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "links",
|
|
"value": [
|
|
{
|
|
"title": "Open this trace below",
|
|
"url": "d/obs-distributed-traces/distributed-traces?${__url_time_range}&var-traceId=${__value.raw}"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"showHeader": true,
|
|
"sortBy": [
|
|
{
|
|
"displayName": "Start time",
|
|
"desc": true
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"type": "row",
|
|
"title": "2 — Every span of the trace, in order",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 9
|
|
},
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "traces",
|
|
"title": "Trace $traceId",
|
|
"description": "Grafana's own trace view: every span nested in order across LibreChat, Keycloak, the Kong gateways and the backend. Click a span to expand its attributes; from a span, 'Logs for this span' jumps to Loki.\n\nA default span filter hides the framework plumbing — Express middleware, mongoose/mongodb calls, kong.dns, raw SQL — which is most of a 212-span trace and none of the flow. Clear the filter in the panel, or toggle 'Show all spans', to get them back.\n\nThis is a DISPLAY filter: those spans are still produced and still stored in Tempo.",
|
|
"datasource": {
|
|
"type": "tempo",
|
|
"uid": "tempo"
|
|
},
|
|
"gridPos": {
|
|
"h": 22,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 10
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "tempo",
|
|
"uid": "tempo"
|
|
},
|
|
"queryType": "traceql",
|
|
"query": "$traceId",
|
|
"limit": 300
|
|
}
|
|
],
|
|
"options": {
|
|
"spanFilters": {
|
|
"spanName": "middleware - .*|mongoose\\..*|mongodb\\..*|kong\\.dns|SELECT|INSERT|UPDATE|DELETE|router - .*",
|
|
"spanNameOperator": "!~",
|
|
"serviceName": "",
|
|
"serviceNameOperator": "=",
|
|
"from": "",
|
|
"to": "",
|
|
"tags": [
|
|
{
|
|
"id": "1",
|
|
"operator": "="
|
|
}
|
|
],
|
|
"matchesOnly": true,
|
|
"criticalPathOnly": false,
|
|
"query": ""
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "row",
|
|
"title": "3 — Identity: what was issued, to whom, for what",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 32
|
|
},
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "table",
|
|
"title": "Identities authenticated in this trace",
|
|
"description": "Who Kong authenticated at each hop of the selected trace, read from the audit records' authenticated_entity.id. A transaction usually carries more than one: LibreChat's own identity on the llm hop, the end user's on the domain hops.\n\nClick a User ID to scope the Keycloak panel below to it. That click is the join Kong does not make for us — its openid-connect plugin calls Keycloak through its own HTTP client, which Kong's propagation module does not touch, so those token exchanges carry their own trace id. The user id is the same on both sides, which is what makes the join possible.",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 33
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"editorMode": "code",
|
|
"queryType": "range",
|
|
"expr": "{job=\"kong-audit\"} |= \"$traceId\" | json authuser=\"authenticated_entity.id\", svc=\"service.name\", uri=\"request.uri\", status=\"response.status\""
|
|
}
|
|
],
|
|
"transformations": [
|
|
{
|
|
"id": "extractFields",
|
|
"options": {
|
|
"source": "labels",
|
|
"replace": false
|
|
}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {
|
|
"labels": true,
|
|
"Line": true,
|
|
"tsNs": true,
|
|
"id": true
|
|
},
|
|
"indexByName": {
|
|
"Time": 0,
|
|
"authuser": 1,
|
|
"svc": 2,
|
|
"uri": 3,
|
|
"status": 4
|
|
},
|
|
"renameByName": {
|
|
"authuser": "User ID",
|
|
"svc": "Gateway service",
|
|
"uri": "Path",
|
|
"status": "Status"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": "groupBy",
|
|
"options": {
|
|
"fields": {
|
|
"User ID": {
|
|
"aggregations": [],
|
|
"operation": "groupby"
|
|
},
|
|
"Gateway service": {
|
|
"aggregations": [],
|
|
"operation": "groupby"
|
|
},
|
|
"Path": {
|
|
"aggregations": [],
|
|
"operation": "groupby"
|
|
},
|
|
"Status": {
|
|
"aggregations": [],
|
|
"operation": "groupby"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"align": "auto",
|
|
"filterable": true
|
|
}
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "User ID"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "links",
|
|
"value": [
|
|
{
|
|
"title": "Scope the Keycloak panel to this identity",
|
|
"url": "d/obs-distributed-traces/distributed-traces?${__url_time_range}&var-traceId=${traceId}&var-userId=${__value.raw}"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"showHeader": true
|
|
}
|
|
},
|
|
{
|
|
"type": "table",
|
|
"title": "Keycloak identity events",
|
|
"description": "Parsed from Keycloak's own event log: token exchanges, logins and code-to-token swaps, with the audience minted, the requesting client, the end user and the grant type. This is where the OIDC substance lives — the Authorization header is REDACTED in the Kong audit records, so claims cannot be read from there, and the raw token is deliberately not logged anywhere.\n\nScoped by USER, not by trace, because Kong does not propagate the trace when its openid-connect plugin calls Keycloak — verified: a request traced end to end produced a TOKEN_EXCHANGE that Keycloak recorded under its own trace id. Click a User ID in the panel above to fill it in; with it empty this shows every identity event in the time range.",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 41
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"editorMode": "code",
|
|
"queryType": "range",
|
|
"expr": "{job=\"keycloak/keycloak\"} |= \"org.keycloak.events\" |= \"$userId\" | regexp `type=\\\\\"(?P<event>[^\\\\\"]*)\\\\\"` | regexp `clientId=\\\\\"(?P<client>[^\\\\\"]*)\\\\\"` | regexp `audience=\\\\\"(?P<audience>[^\\\\\"]*)\\\\\"` | regexp `username=\\\\\"(?P<user>[^\\\\\"]*)\\\\\"` | regexp `grant_type=\\\\\"(?P<grant>[^\\\\\"]*)\\\\\"` | regexp `scope=\\\\\"(?P<scope>[^\\\\\"]*)\\\\\"`"
|
|
}
|
|
],
|
|
"transformations": [
|
|
{
|
|
"id": "extractFields",
|
|
"options": {
|
|
"source": "labels",
|
|
"replace": false
|
|
}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {
|
|
"labels": true,
|
|
"Line": true,
|
|
"tsNs": true,
|
|
"id": true,
|
|
"Time": false
|
|
},
|
|
"indexByName": {
|
|
"Time": 0,
|
|
"event": 1,
|
|
"client": 2,
|
|
"audience": 3,
|
|
"user": 4,
|
|
"grant": 5,
|
|
"scope": 6
|
|
},
|
|
"renameByName": {
|
|
"event": "Event",
|
|
"client": "Client",
|
|
"audience": "Audience",
|
|
"user": "User",
|
|
"grant": "Grant type",
|
|
"scope": "Scopes"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"align": "auto",
|
|
"filterable": true
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"showHeader": true,
|
|
"sortBy": [
|
|
{
|
|
"displayName": "Time",
|
|
"desc": true
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"type": "row",
|
|
"title": "4 — What was actually exchanged",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 52
|
|
},
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "logs",
|
|
"title": "Logs of trace $traceId",
|
|
"description": "Every log line carrying this trace id. The Kong audit records are the substance: request headers, response status, per-stage latencies, and on the domain gateways request_body and response_body — so the MCP tools/call payload and its result are readable in full. Expand a line to read it as fields.\n\nOne gap, and it is a capture gap rather than a display one: the llm gateway has no body capture, so LLM prompts and completions are not recorded.",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 14,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 53
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{job=~\"kong-audit|classic-models/classic-models-api|librechat/librechat|keycloak/keycloak\"} |= \"$traceId\""
|
|
}
|
|
],
|
|
"options": {
|
|
"showTime": true,
|
|
"wrapLogMessage": true,
|
|
"sortOrder": "Ascending",
|
|
"enableLogDetails": true,
|
|
"prettifyLogMessage": true
|
|
}
|
|
}
|
|
]
|
|
} |