From 05cf2bcce1871c9e488ab6905ecb51cd215f9cd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Tailland?= Date: Thu, 27 Aug 2026 14:12:32 +0200 Subject: [PATCH] Turn on Kong's own AI telemetry instead of improvising it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four settings, all of them off, which is why the AI gateway produced HTTP-level telemetry and nothing about the model. logging.log_payloads the prompt and the completion, in the log records and — since OTel instrumentation is on here — in the spans too logging.log_statistics model usage and token counts prometheus.ai_metrics kong_ai_llm_* : requests, tokens by type, cost, provider latency, per model and per consumer input_cost / output_cost cost per 1M tokens, which is what makes kong_ai_llm_cost_total mean anything Also removes this morning's pre-function/post-function body capture from this gateway. It was the wrong tool for the LLM route: Kong's own implementation reassembles a streamed SSE answer, while accumulating raw chunks in kong.ctx.shared does not. The five business domains keep theirs — their MCP traffic is not streamed the same way. The costs are a snapshot, checked against OpenRouter's price list for google/gemini-3.7-flash on 2026-08-27: $0.375 in, $1.875 out per 1M tokens. They are substitution variables rather than literals precisely because a stale figure produces confidently wrong money rather than an error. Note what this does NOT fix: usage is missing on streamed responses because the OpenAI-compatible API only reports it when the caller asks for stream_options.include_usage, and that is LibreChat's request to make, not Kong's. Expect token counts to stay partial until then. Co-Authored-By: Claude Opus 5 --- dev/proxy-kustomization.yaml | 7 +++ kong/ai-proxy-advanced.yaml | 19 +++++++++ kong/observability.yaml | 82 +++--------------------------------- 3 files changed, 31 insertions(+), 77 deletions(-) diff --git a/dev/proxy-kustomization.yaml b/dev/proxy-kustomization.yaml index e223697..33a2770 100644 --- a/dev/proxy-kustomization.yaml +++ b/dev/proxy-kustomization.yaml @@ -21,6 +21,13 @@ spec: OIDC_ISSUER: https://keycloak.sttlab.eu/realms/sttlab AGENT_AUDIENCE: '["llm-proxy"]' OPENROUTER_UPSTREAM_URL: https://openrouter.ai/api/v1/chat/completions + # Cost per 1M tokens for the model behind OPENROUTER_UPSTREAM_URL, used by + # ai-proxy-advanced to compute kong_ai_llm_cost_total. Checked against + # OpenRouter's price list for google/gemini-3.7-flash on 2026-08-27 + # ($0.375 in / $1.875 out). These are a snapshot, not a live rate: change + # the model or the contract and they silently become wrong money. + LLM_INPUT_COST: "0.375" + LLM_OUTPUT_COST: "1.875" OPENROUTER_EMBEDDINGS_UPSTREAM_URL: https://openrouter.ai/api/v1/embeddings VECTORDB_HOST: llm-dev-vectordb VECTORDB_PORT: "6379" diff --git a/kong/ai-proxy-advanced.yaml b/kong/ai-proxy-advanced.yaml index aea8a97..c36a483 100644 --- a/kong/ai-proxy-advanced.yaml +++ b/kong/ai-proxy-advanced.yaml @@ -10,9 +10,28 @@ config: auth: header_name: Authorization header_value: ${OPENROUTER_BEARER} + # dev only, like the rest of this gateway's audit trail: log_payloads + # puts the prompt and the completion in the log stream, and -- because + # OpenTelemetry instrumentation is on here -- in the spans as well. + # + # This is what the pre-function/post-function capture in + # observability.yaml stood in for on this route. Kong's own + # implementation reassembles a streamed SSE answer; accumulating raw + # chunks does not. + logging: + log_payloads: true + # Model usage and token counts in the log records. With this off Kong + # counted nothing of its own, which is half of why usage was missing. + log_statistics: true model: provider: openai # name deliberately unset: the client's own `model` field is passed # through to OpenRouter. options: upstream_url: ${OPENROUTER_UPSTREAM_URL} + # Cost per 1M tokens, in the provider's currency. Kong multiplies + # these by the token counts to produce kong_ai_llm_cost_total, so a + # stale figure yields confidently wrong money rather than an error. + # Re-check them whenever the model or the contract changes. + input_cost: ${LLM_INPUT_COST} + output_cost: ${LLM_OUTPUT_COST} diff --git a/kong/observability.yaml b/kong/observability.yaml index 84254fb..f857325 100644 --- a/kong/observability.yaml +++ b/kong/observability.yaml @@ -19,6 +19,11 @@ metadata: namespace: ${NAMESPACE} plugin: prometheus config: + # The reason this gateway's prometheus config differs from the shared + # template: it fronts an LLM. ai_metrics adds kong_ai_llm_* — requests, + # tokens by type, cost and provider latency, per model and per consumer. + # Without it none of that reaches Prometheus, whatever ai-proxy logs. + ai_metrics: true status_code_metrics: true latency_metrics: true bandwidth_metrics: true @@ -66,12 +71,6 @@ config: http_endpoint: ${AUDIT_LOG_ENDPOINT} method: POST content_type: application/json - # Reads what the two capture plugins below stashed in kong.ctx.shared. Both - # are nil -- and silently omitted from the record -- if those plugins are not - # applied, so this degrades to metadata-only logging rather than breaking. - custom_fields_by_lua: - request_body: "return kong.ctx.shared.audit_request_body" - response_body: "return kong.ctx.shared.audit_response_body" --- # prometheus and opentelemetry are bound control-plane-wide, matching the # shared template: they describe the gateway itself, not one service. @@ -122,74 +121,3 @@ spec: group: configuration.konghq.com kind: KongService name: ${NAME}-llm-proxy ---- -# dev only: captures the raw request/response bodies into kong.ctx.shared for -# the http-log plugin above to read back in the log phase. Same mechanism the -# five business domains use (-api/kong/mcp/audit-log-body-capture.yaml). -# Without it this gateway logged metadata only, so the prompts and completions -# going through the LLM were the one thing the audit trail did not record. -# -# The response accumulator matters more here than elsewhere: LLM answers stream -# as SSE, so body_filter runs once per chunk and the full completion only exists -# after the last one. -apiVersion: configuration.konghq.com/v1 -kind: KongPlugin -metadata: - name: ${NAME}-llm-audit-capture-request - namespace: ${NAMESPACE} -plugin: pre-function -config: - access: - - "kong.ctx.shared.audit_request_body = kong.request.get_raw_body()" ---- -apiVersion: configuration.konghq.com/v1 -kind: KongPlugin -metadata: - name: ${NAME}-llm-audit-capture-response - namespace: ${NAMESPACE} -plugin: post-function -config: - body_filter: - - | - local chunk = ngx.arg[1] - if chunk then - kong.ctx.shared.audit_response_body = (kong.ctx.shared.audit_response_body or "") .. chunk - end ---- -# Bound to the proxy service, like the audit log itself, so only real gateway -# traffic is captured. -apiVersion: configuration.konghq.com/v1alpha1 -kind: KongPluginBinding -metadata: - name: ${NAME}-llm-audit-capture-request-binding - namespace: ${NAMESPACE} -spec: - controlPlaneRef: - type: konnectNamespacedRef - konnectNamespacedRef: - name: ${CONTROL_PLANE_REF_NAME} - pluginRef: - name: ${NAME}-llm-audit-capture-request - targets: - serviceRef: - group: configuration.konghq.com - kind: KongService - name: ${NAME}-llm-proxy ---- -apiVersion: configuration.konghq.com/v1alpha1 -kind: KongPluginBinding -metadata: - name: ${NAME}-llm-audit-capture-response-binding - namespace: ${NAMESPACE} -spec: - controlPlaneRef: - type: konnectNamespacedRef - konnectNamespacedRef: - name: ${CONTROL_PLANE_REF_NAME} - pluginRef: - name: ${NAME}-llm-audit-capture-response - targets: - serviceRef: - group: configuration.konghq.com - kind: KongService - name: ${NAME}-llm-proxy