Stéphane Tailland
367ebd1da5
Trace search lived only in Explore. `obs-traces` brings it into a dashboard:
TraceQL assembled from a service selector, a minimum duration, a status and a
free-text field, plus the aggregate RED view from the span metrics Tempo derives
from the traces themselves.
The status and free-text variables hold TraceQL fragments rather than values, so
an empty one contributes nothing and the query stays valid — the same reason the
log dashboards inject their search term the way they do.
Both span correlations were declared but non-functional.
tracesToLogsV2 mapped service.name onto Loki's `app` label, which Alloy derives
from app.kubernetes.io/name. That label exists for Keycloak and the Classic
Models API but not for the six Kong gateways or LibreChat — and the gateways are
the only place the trace id actually appears in the logs, so the mapping pointed
squarely at the services where it could not resolve. Replaced with a custom
query on ${__trace.traceId}, which matches any source writing the id anywhere in
the line.
tracesToMetrics had no queries at all and rendered a bare datasource link. It
now carries rate, error-rate and p95 against traces_spanmetrics_*, scoped to the
span's own service.
The service graph stays a table: serviceMap is a query type Grafana's frontend
assembles for Explore and returns no frames through the dashboard query API.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
106 lines
3.5 KiB
YAML
106 lines
3.5 KiB
YAML
# Declarative datasource provisioning.
|
|
# The `uid` values are STABLE and are what dashboard JSON files reference:
|
|
# "datasource": { "type": "prometheus", "uid": "prometheus" }
|
|
apiVersion: 1
|
|
|
|
deleteDatasources:
|
|
- name: Prometheus
|
|
orgId: 1
|
|
- name: Loki
|
|
orgId: 1
|
|
- name: Tempo
|
|
orgId: 1
|
|
|
|
datasources:
|
|
- name: Prometheus
|
|
uid: prometheus
|
|
type: prometheus
|
|
access: proxy
|
|
orgId: 1
|
|
url: http://prometheus:9090
|
|
isDefault: true
|
|
editable: false
|
|
jsonData:
|
|
httpMethod: POST
|
|
timeInterval: 30s
|
|
manageAlerts: false
|
|
exemplarTraceIdDestinations:
|
|
- name: trace_id
|
|
datasourceUid: tempo
|
|
|
|
- name: Loki
|
|
uid: loki
|
|
type: loki
|
|
access: proxy
|
|
orgId: 1
|
|
url: http://loki:3100
|
|
editable: false
|
|
jsonData:
|
|
timeout: 60
|
|
maxLines: 1000
|
|
# Logs -> traces: a trace_id in the log line opens Tempo.
|
|
derivedFields:
|
|
- name: TraceID
|
|
matcherType: regex
|
|
matcherRegex: '"?trace[_-]?id"?[:=]"?([a-f0-9]{16,32})'
|
|
url: '$${__value.raw}'
|
|
datasourceUid: tempo
|
|
urlDisplayLabel: View trace
|
|
|
|
- name: Tempo
|
|
uid: tempo
|
|
type: tempo
|
|
access: proxy
|
|
orgId: 1
|
|
url: http://tempo:3200
|
|
editable: false
|
|
jsonData:
|
|
httpMethod: GET
|
|
streamingEnabled:
|
|
search: true
|
|
metrics: true
|
|
# Service graph fed by the metrics_generator -> Prometheus.
|
|
serviceMap:
|
|
datasourceUid: prometheus
|
|
nodeGraph:
|
|
enabled: true
|
|
# Span -> logs by trace id rather than by service name.
|
|
#
|
|
# The service-name mapping this replaces resolved `service.name` to Loki's
|
|
# `app` label, which Alloy derives from app.kubernetes.io/name. That label
|
|
# exists for keycloak and classic-models-api but not for the six Kong
|
|
# gateways or librechat — and the gateways are precisely where the trace
|
|
# id does appear in the logs. The mapping pointed at the services where it
|
|
# could not work.
|
|
#
|
|
# A custom query on ${__trace.traceId} sidesteps the label entirely: every
|
|
# source that writes the id anywhere in the line is matched. The stream
|
|
# selector is spelled out rather than `.+` so the query stays bounded.
|
|
tracesToLogsV2:
|
|
datasourceUid: loki
|
|
spanStartTimeShift: -1h
|
|
spanEndTimeShift: 1h
|
|
filterByTraceID: false
|
|
filterBySpanID: false
|
|
customQuery: true
|
|
query: '{job=~"kong-audit|keycloak/keycloak|classic-models/classic-models-api|librechat/librechat"} |= "$${__trace.traceId}"'
|
|
# Span -> metrics against the span metrics Tempo's own metrics_generator
|
|
# feeds into Prometheus. Without explicit queries this block renders a
|
|
# bare datasource link and nothing else, which is what it did before.
|
|
# $$__tags expands to the tag mapping below, so each query is scoped to
|
|
# the span's own service.
|
|
tracesToMetrics:
|
|
datasourceUid: prometheus
|
|
spanStartTimeShift: -30m
|
|
spanEndTimeShift: 30m
|
|
tags:
|
|
- key: service.name
|
|
value: service
|
|
queries:
|
|
- name: 'Request rate'
|
|
query: 'sum by (span_name) (rate(traces_spanmetrics_calls_total{$$__tags}[$$__rate_interval]))'
|
|
- name: 'Error rate'
|
|
query: 'sum by (span_name) (rate(traces_spanmetrics_calls_total{$$__tags,status_code="STATUS_CODE_ERROR"}[$$__rate_interval]))'
|
|
- name: 'Latency p95'
|
|
query: 'histogram_quantile(0.95, sum by (span_name, le) (rate(traces_spanmetrics_latency_bucket{$$__tags}[$$__rate_interval])))'
|