Stéphane Tailland
5be2c76d94
Kong does not propagate the trace when its openid-connect plugin calls
Keycloak: verified by tracing a request end to end and watching the
TOKEN_EXCHANGE it triggered land under Keycloak's own trace id instead.
It is not a missing setting — Kong's propagation module injects through
kong.service.request.*, which is the proxied upstream request, while the
plugin reaches Keycloak with its own HTTP client. Nothing in the
opentelemetry schema or kong_defaults.lua covers that path.
There is a join key even so: the audit record's
authenticated_entity.id is the same value as Keycloak's userId. So add a
panel listing the identities Kong authenticated on the selected trace —
usually more than one, LibreChat's own on the llm hop and the end user's
on the domain hops — and let a click on one scope the Keycloak panel to
that user.
A Grafana query variable cannot derive this automatically: Loki's
label/values endpoint rejects a pipeline ("only label matchers are
supported"), so the extracted field cannot feed a variable. Hence the
click.
That gives the intended chain for one user: LOGIN, CODE_TO_TOKEN,
USER_INFO_REQUEST, then a TOKEN_EXCHANGE per audience — verified against
live data.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
445 lines
13 KiB
JSON
445 lines
13 KiB
JSON
{
|
|
"uid": "obs-distributed-traces",
|
|
"title": "Distributed Traces",
|
|
"description": "Follow one transaction end to end: pick it, read every span in order, see which tokens were issued, and read what was exchanged at each hop.",
|
|
"tags": [
|
|
"observability",
|
|
"tracing",
|
|
"tempo"
|
|
],
|
|
"timezone": "browser",
|
|
"schemaVersion": 39,
|
|
"version": 1,
|
|
"refresh": "",
|
|
"time": {
|
|
"from": "now-1h",
|
|
"to": "now"
|
|
},
|
|
"editable": true,
|
|
"graphTooltip": 0,
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "traceFilter",
|
|
"label": "Show",
|
|
"type": "custom",
|
|
"query": "Transactions — 3+ services : {kind=server} | count() > 2,2+ services : {kind=server} | count() > 1,Everything (noisy) : {}",
|
|
"options": [
|
|
{
|
|
"text": "Transactions — 3+ services",
|
|
"value": "{kind=server} | count() > 2",
|
|
"selected": true
|
|
},
|
|
{
|
|
"text": "2+ services",
|
|
"value": "{kind=server} | count() > 1",
|
|
"selected": false
|
|
},
|
|
{
|
|
"text": "Everything (noisy)",
|
|
"value": "{}",
|
|
"selected": false
|
|
}
|
|
],
|
|
"current": {
|
|
"text": "Transactions — 3+ services",
|
|
"value": "{kind=server} | count() > 2"
|
|
},
|
|
"includeAll": false,
|
|
"multi": false,
|
|
"description": "Which traces are worth listing."
|
|
},
|
|
{
|
|
"name": "traceId",
|
|
"label": "Trace ID",
|
|
"type": "textbox",
|
|
"query": "",
|
|
"current": {
|
|
"text": "",
|
|
"value": ""
|
|
},
|
|
"description": "Set by clicking a Trace ID above, or paste one."
|
|
},
|
|
{
|
|
"name": "userId",
|
|
"label": "User ID",
|
|
"type": "textbox",
|
|
"query": "",
|
|
"current": {
|
|
"text": "",
|
|
"value": ""
|
|
},
|
|
"description": "Set by clicking an identity in the panel below. Empty means the Keycloak panel is scoped only by time."
|
|
}
|
|
]
|
|
},
|
|
"panels": [
|
|
{
|
|
"type": "row",
|
|
"title": "1 — Pick a transaction",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "table",
|
|
"title": "Transactions",
|
|
"description": "Traces where at least three services received an HTTP request — a real end-to-end flow rather than a token call or a health probe. Filtering by service name does not achieve this: Keycloak and the gateways are services too.\n\nClick a Trace ID to load it below.",
|
|
"datasource": {
|
|
"type": "tempo",
|
|
"uid": "tempo"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 1
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "tempo",
|
|
"uid": "tempo"
|
|
},
|
|
"queryType": "traceql",
|
|
"query": "$traceFilter",
|
|
"tableType": "traces",
|
|
"limit": 50
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"align": "auto",
|
|
"filterable": true
|
|
}
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "traceID"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "links",
|
|
"value": [
|
|
{
|
|
"title": "Open this trace below",
|
|
"url": "d/obs-distributed-traces/distributed-traces?${__url_time_range}&var-traceId=${__value.raw}"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"showHeader": true,
|
|
"sortBy": [
|
|
{
|
|
"displayName": "Start time",
|
|
"desc": true
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"type": "row",
|
|
"title": "2 — Every span of the trace, in order",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 9
|
|
},
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "traces",
|
|
"title": "Trace $traceId",
|
|
"description": "Grafana's own trace view: the full waterfall, every span nested in order, across LibreChat, Keycloak, the Kong gateways and the backend — no split between HTTP hops and plugins, they are one tree.\n\nClick a span to expand its attributes: method, route, status, durations, and whatever the instrumentation attached. From a span, 'Logs for this span' jumps to Loki.",
|
|
"datasource": {
|
|
"type": "tempo",
|
|
"uid": "tempo"
|
|
},
|
|
"gridPos": {
|
|
"h": 22,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 10
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "tempo",
|
|
"uid": "tempo"
|
|
},
|
|
"queryType": "traceql",
|
|
"query": "$traceId",
|
|
"limit": 300
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"type": "row",
|
|
"title": "3 — Identity: what was issued, to whom, for what",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 32
|
|
},
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "table",
|
|
"title": "Identities authenticated in this trace",
|
|
"description": "Who Kong authenticated at each hop of the selected trace, read from the audit records' authenticated_entity.id. A transaction usually carries more than one: LibreChat's own identity on the llm hop, the end user's on the domain hops.\n\nClick a User ID to scope the Keycloak panel below to it. That click is the join Kong does not make for us — its openid-connect plugin calls Keycloak through its own HTTP client, which Kong's propagation module does not touch, so those token exchanges carry their own trace id. The user id is the same on both sides, which is what makes the join possible.",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 33
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"editorMode": "code",
|
|
"queryType": "range",
|
|
"expr": "{job=\"kong-audit\"} |= \"$traceId\" | json authuser=\"authenticated_entity.id\", svc=\"service.name\", uri=\"request.uri\", status=\"response.status\""
|
|
}
|
|
],
|
|
"transformations": [
|
|
{
|
|
"id": "extractFields",
|
|
"options": {
|
|
"source": "labels",
|
|
"replace": false
|
|
}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {
|
|
"labels": true,
|
|
"Line": true,
|
|
"tsNs": true,
|
|
"id": true
|
|
},
|
|
"indexByName": {
|
|
"Time": 0,
|
|
"authuser": 1,
|
|
"svc": 2,
|
|
"uri": 3,
|
|
"status": 4
|
|
},
|
|
"renameByName": {
|
|
"authuser": "User ID",
|
|
"svc": "Gateway service",
|
|
"uri": "Path",
|
|
"status": "Status"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": "groupBy",
|
|
"options": {
|
|
"fields": {
|
|
"User ID": {
|
|
"aggregations": [],
|
|
"operation": "groupby"
|
|
},
|
|
"Gateway service": {
|
|
"aggregations": [],
|
|
"operation": "groupby"
|
|
},
|
|
"Path": {
|
|
"aggregations": [],
|
|
"operation": "groupby"
|
|
},
|
|
"Status": {
|
|
"aggregations": [],
|
|
"operation": "groupby"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"align": "auto",
|
|
"filterable": true
|
|
}
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "User ID"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "links",
|
|
"value": [
|
|
{
|
|
"title": "Scope the Keycloak panel to this identity",
|
|
"url": "d/obs-distributed-traces/distributed-traces?${__url_time_range}&var-traceId=${traceId}&var-userId=${__value.raw}"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"showHeader": true
|
|
}
|
|
},
|
|
{
|
|
"type": "table",
|
|
"title": "Keycloak identity events",
|
|
"description": "Parsed from Keycloak's own event log: token exchanges, logins and code-to-token swaps, with the audience minted, the requesting client, the end user and the grant type. This is where the OIDC substance lives — the Authorization header is REDACTED in the Kong audit records, so claims cannot be read from there, and the raw token is deliberately not logged anywhere.\n\nScoped by USER, not by trace, because Kong does not propagate the trace when its openid-connect plugin calls Keycloak — verified: a request traced end to end produced a TOKEN_EXCHANGE that Keycloak recorded under its own trace id. Click a User ID in the panel above to fill it in; with it empty this shows every identity event in the time range.",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 41
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"editorMode": "code",
|
|
"queryType": "range",
|
|
"expr": "{job=\"keycloak/keycloak\"} |= \"org.keycloak.events\" |= \"$userId\" | regexp `type=\\\\\"(?P<event>[^\\\\\"]*)\\\\\"` | regexp `clientId=\\\\\"(?P<client>[^\\\\\"]*)\\\\\"` | regexp `audience=\\\\\"(?P<audience>[^\\\\\"]*)\\\\\"` | regexp `username=\\\\\"(?P<user>[^\\\\\"]*)\\\\\"` | regexp `grant_type=\\\\\"(?P<grant>[^\\\\\"]*)\\\\\"` | regexp `scope=\\\\\"(?P<scope>[^\\\\\"]*)\\\\\"`"
|
|
}
|
|
],
|
|
"transformations": [
|
|
{
|
|
"id": "extractFields",
|
|
"options": {
|
|
"source": "labels",
|
|
"replace": false
|
|
}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {
|
|
"labels": true,
|
|
"Line": true,
|
|
"tsNs": true,
|
|
"id": true,
|
|
"Time": false
|
|
},
|
|
"indexByName": {
|
|
"Time": 0,
|
|
"event": 1,
|
|
"client": 2,
|
|
"audience": 3,
|
|
"user": 4,
|
|
"grant": 5,
|
|
"scope": 6
|
|
},
|
|
"renameByName": {
|
|
"event": "Event",
|
|
"client": "Client",
|
|
"audience": "Audience",
|
|
"user": "User",
|
|
"grant": "Grant type",
|
|
"scope": "Scopes"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"align": "auto",
|
|
"filterable": true
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"showHeader": true,
|
|
"sortBy": [
|
|
{
|
|
"displayName": "Time",
|
|
"desc": true
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"type": "row",
|
|
"title": "4 — What was actually exchanged",
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 52
|
|
},
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "logs",
|
|
"title": "Logs of trace $traceId",
|
|
"description": "Every log line carrying this trace id. The Kong audit records are the substance: request headers, response status, per-stage latencies, and on the domain gateways request_body and response_body — so the MCP tools/call payload and its result are readable in full. Expand a line to read it as fields.\n\nOne gap, and it is a capture gap rather than a display one: the llm gateway has no body capture, so LLM prompts and completions are not recorded.",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"gridPos": {
|
|
"h": 14,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 53
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "loki"
|
|
},
|
|
"expr": "{job=~\"kong-audit|classic-models/classic-models-api|librechat/librechat|keycloak/keycloak\"} |= \"$traceId\""
|
|
}
|
|
],
|
|
"options": {
|
|
"showTime": true,
|
|
"wrapLogMessage": true,
|
|
"sortOrder": "Ascending",
|
|
"enableLogDetails": true,
|
|
"prettifyLogMessage": true
|
|
}
|
|
}
|
|
]
|
|
} |