From 9a889127caea916c8fc8df76beb3fb91a5198eb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Tailland?= Date: Tue, 25 Aug 2026 07:25:36 +0200 Subject: [PATCH] Send Kong traces to Tempo and gateway audit logs to Loki Point OTEL_TRACES_ENDPOINT at Tempo instead of Jaeger. The variable is a cluster-wide default, so this moves all five domains at once; Jaeger stays deployed but no longer receives anything. Tempo derives RED span metrics and the service graph from these traces on its own, and its trace ids correlate with the audit logs below. Add a Loki output to the audit collector. Kong's http-log payload is a JSON object per request and Loki has no endpoint that accepts it, so Fluent Bit is the translator between the two formats. The `domain` label comes from $TAG, which the http input derives from the request URI. That makes the path a domain posts to its identity here, so one output block covers every domain and onboarding the next one is a single AUDIT_LOG_ENDPOINT change on its side rather than another block. The file output stays on Match * so the on-disk audit trail continues for every domain, which keeps the switch reversible. Co-Authored-By: Claude Opus 5 --- .../dataplane-template-defaults.yaml | 9 ++++--- .../audit-log-collector-configmap.yaml | 24 +++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/kong-platform/dataplane-template-defaults.yaml b/kong-platform/dataplane-template-defaults.yaml index 7a61cab..e9cfbaf 100644 --- a/kong-platform/dataplane-template-defaults.yaml +++ b/kong-platform/dataplane-template-defaults.yaml @@ -18,6 +18,9 @@ data: CORS_ORIGINS: https://*.sttlab.local RATE_LIMIT_MINUTE: "300" REQUEST_MAX_BODY_SIZE: "10" # MB - # Jaeger v2 (OTLP-native), Helm-installed 2026-08-20, ns observability, - # cluster-wide — shared by every domain, not domain-specific. - OTEL_TRACES_ENDPOINT: http://jaeger.observability.svc.cluster.local:4318/v1/traces + # Tempo (OTLP-native), ns observability, cluster-wide — shared by every + # domain, not domain-specific. Replaced Jaeger on 2026-08-25: Tempo is the + # trace backend of the infra-observability stack, so its span metrics feed + # Prometheus and its traces correlate with the Loki logs from the same + # requests. Jaeger is still deployed but no longer receives anything. + OTEL_TRACES_ENDPOINT: http://tempo.observability.svc.cluster.local:4318/v1/traces diff --git a/observability/audit-log-collector-configmap.yaml b/observability/audit-log-collector-configmap.yaml index 669a20e..0718e3f 100644 --- a/observability/audit-log-collector-configmap.yaml +++ b/observability/audit-log-collector-configmap.yaml @@ -4,6 +4,15 @@ metadata: name: audit-log-collector-config namespace: observability data: + # Fluent Bit's http input derives the record tag from the request URI, so the + # path a domain's http-log plugin posts to IS its identity here: + # POST /catalog -> tag `catalog` -> Loki label domain=catalog + # Each domain therefore gets its own label without a per-domain output block, + # and onboarding one is a single ${AUDIT_LOG_ENDPOINT} change on its side. + # + # Kong's http-log payload is a JSON object per request and Loki has no + # endpoint that accepts it directly, so Fluent Bit is the translator between + # Kong's format and Loki's push API. fluent-bit.conf: | [SERVICE] Flush 1 @@ -14,9 +23,24 @@ data: Listen 0.0.0.0 Port 8888 + # On-disk audit trail, kept alongside Loki: the switch stays reversible + # for as long as this output is here. [OUTPUT] Name file Match * Path /data File audit.jsonl Format plain + + # Labels stay deliberately low-cardinality — the full Kong record (route, + # status, latencies, captured bodies) rides in the log line itself and is + # queryable with LogQL's json parser. `domain` comes from the tag, i.e. + # from the URI the domain posted to. + [OUTPUT] + Name loki + Match * + Host loki.observability.svc.cluster.local + Port 3100 + Labels job=kong-audit, domain=$TAG + Label_Keys $service['name'] + Line_Format json