diff --git a/k8s/grafana/dashboards/kustomization.yaml b/k8s/grafana/dashboards/kustomization.yaml index c8f2e6f..6a97da6 100644 --- a/k8s/grafana/dashboards/kustomization.yaml +++ b/k8s/grafana/dashboards/kustomization.yaml @@ -21,6 +21,7 @@ configMapGenerator: - obs-application-logs.json - obs-audit-logs.json - obs-classic-models-api.json + - obs-distributed-traces.json - obs-keycloak.json - obs-kong-gateways.json - obs-librechat.json diff --git a/k8s/grafana/dashboards/obs-distributed-traces.json b/k8s/grafana/dashboards/obs-distributed-traces.json new file mode 100644 index 0000000..7edd056 --- /dev/null +++ b/k8s/grafana/dashboards/obs-distributed-traces.json @@ -0,0 +1,289 @@ +{ + "uid": "obs-distributed-traces", + "title": "Distributed Traces", + "description": "Find a trace, and keep only the spans worth looking at. Built because Explore's unfiltered search is unusable here: Tempo does not return {} newest-first, so health checks crowd out real transactions.\n\nDeliberately carries no metric panels. A span's own duration is in the span tables, measured; aggregate RED by span name is what the per-application dashboards already show, and neither answers 'the metrics of this span'.", + "tags": [ + "observability", + "tracing", + "tempo" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "", + "time": { + "from": "now-1h", + "to": "now" + }, + "editable": true, + "graphTooltip": 1, + "templating": { + "list": [ + { + "name": "service", + "label": "Service", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "query": "label_values(traces_spanmetrics_calls_total, service)", + "refId": "service" + }, + "refresh": 2, + "multi": true, + "includeAll": true, + "allValue": ".+", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "description": "Services known to Tempo's span metrics. Drives the trace, span and metric panels." + }, + { + "name": "traceId", + "label": "Trace ID", + "type": "textbox", + "query": "", + "current": { + "text": "", + "value": "" + }, + "description": "Set by clicking a Trace ID in the top table, or paste one. Empty means the log filter matches everything." + } + ] + }, + "panels": [ + { + "type": "row", + "title": "Traces", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [] + }, + { + "type": "table", + "title": "Traces on the dashboard time range", + "description": "TraceQL over the dashboard's time range. Filtered by service so the health-check noise stays out: an unfiltered {} query is NOT returned newest-first by Tempo and the limit truncates arbitrarily, which is why browsing raw Explore misses recent traces. Click a Trace ID to load it below.", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "queryType": "traceql", + "query": "{resource.service.name=~\"$service\"}", + "tableType": "traces", + "limit": 50, + "spss": 20 + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "filterable": true + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "traceIdHidden" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "traceID" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Load this trace below", + "url": "d/obs-distributed-traces/distributed-traces?${__url_time_range}&var-traceId=${__value.raw}&var-service=${service:queryparam}" + } + ] + } + ] + } + ] + }, + "options": { + "showHeader": true, + "sortBy": [ + { + "displayName": "Start time", + "desc": true + } + ] + } + }, + { + "type": "row", + "title": "Spans that matter", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "collapsed": false, + "panels": [] + }, + { + "type": "table", + "title": "HTTP hops between components", + "description": "One row per component entry: these are the SERVER-kind spans, i.e. the inbound HTTP request each service received. Browser to LibreChat, LibreChat to the llm gateway, gateway to a domain gateway, gateway to the backend. Everything else — express middleware, DNS, SQL — is deliberately excluded.", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 11 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "queryType": "traceql", + "query": "{resource.service.name=~\"$service\" && kind=server} | select(span.http.method, span.http.status_code, span.http.route, span.http.url)", + "tableType": "spans", + "limit": 50, + "spss": 20 + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "filterable": true + } + }, + "overrides": [] + }, + "options": { + "showHeader": true + } + }, + { + "type": "table", + "title": "Kong plugin execution", + "description": "Every plugin Kong ran, with its own duration — openid-connect, rate-limiting, request-transformer, opentelemetry, and so on. This is what shows which plugin is costing latency on a gateway hop.", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 11 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "queryType": "traceql", + "query": "{resource.service.name=~\"$service\" && name=~\"kong\\\\..*plugin.*\"}", + "tableType": "spans", + "limit": 50, + "spss": 20 + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "filterable": true + } + }, + "overrides": [] + }, + "options": { + "showHeader": true + } + }, + { + "type": "row", + "title": "Logs for the selected trace", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "collapsed": false, + "panels": [] + }, + { + "type": "logs", + "title": "Logs carrying trace $traceId", + "description": "Correlation is by TRACE id, not span id: none of the emitters log a span id, so this is every log line of the whole trace rather than of one span.\n\nCoverage is uneven and worth knowing: kong-audit and the backend's access log carry the id on every line; the backend's application log does too since v1.0.4. LibreChat logs carry no trace id at all, so its lines never appear here.\n\nWith traceId empty the filter matches everything — paste an id or click one above.", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 22 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{job=~\"kong-audit|classic-models/classic-models-api|librechat/librechat|keycloak/keycloak\"} |= \"$traceId\"" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "sortOrder": "Ascending", + "enableLogDetails": true + } + } + ] +} \ No newline at end of file