diff --git a/dev/proxy-kustomization.yaml b/dev/proxy-kustomization.yaml index e223697..33a2770 100644 --- a/dev/proxy-kustomization.yaml +++ b/dev/proxy-kustomization.yaml @@ -21,6 +21,13 @@ spec: OIDC_ISSUER: https://keycloak.sttlab.eu/realms/sttlab AGENT_AUDIENCE: '["llm-proxy"]' OPENROUTER_UPSTREAM_URL: https://openrouter.ai/api/v1/chat/completions + # Cost per 1M tokens for the model behind OPENROUTER_UPSTREAM_URL, used by + # ai-proxy-advanced to compute kong_ai_llm_cost_total. Checked against + # OpenRouter's price list for google/gemini-3.7-flash on 2026-08-27 + # ($0.375 in / $1.875 out). These are a snapshot, not a live rate: change + # the model or the contract and they silently become wrong money. + LLM_INPUT_COST: "0.375" + LLM_OUTPUT_COST: "1.875" OPENROUTER_EMBEDDINGS_UPSTREAM_URL: https://openrouter.ai/api/v1/embeddings VECTORDB_HOST: llm-dev-vectordb VECTORDB_PORT: "6379" diff --git a/kong/ai-proxy-advanced.yaml b/kong/ai-proxy-advanced.yaml index aea8a97..c36a483 100644 --- a/kong/ai-proxy-advanced.yaml +++ b/kong/ai-proxy-advanced.yaml @@ -10,9 +10,28 @@ config: auth: header_name: Authorization header_value: ${OPENROUTER_BEARER} + # dev only, like the rest of this gateway's audit trail: log_payloads + # puts the prompt and the completion in the log stream, and -- because + # OpenTelemetry instrumentation is on here -- in the spans as well. + # + # This is what the pre-function/post-function capture in + # observability.yaml stood in for on this route. Kong's own + # implementation reassembles a streamed SSE answer; accumulating raw + # chunks does not. + logging: + log_payloads: true + # Model usage and token counts in the log records. With this off Kong + # counted nothing of its own, which is half of why usage was missing. + log_statistics: true model: provider: openai # name deliberately unset: the client's own `model` field is passed # through to OpenRouter. options: upstream_url: ${OPENROUTER_UPSTREAM_URL} + # Cost per 1M tokens, in the provider's currency. Kong multiplies + # these by the token counts to produce kong_ai_llm_cost_total, so a + # stale figure yields confidently wrong money rather than an error. + # Re-check them whenever the model or the contract changes. + input_cost: ${LLM_INPUT_COST} + output_cost: ${LLM_OUTPUT_COST} diff --git a/kong/observability.yaml b/kong/observability.yaml index 84254fb..f857325 100644 --- a/kong/observability.yaml +++ b/kong/observability.yaml @@ -19,6 +19,11 @@ metadata: namespace: ${NAMESPACE} plugin: prometheus config: + # The reason this gateway's prometheus config differs from the shared + # template: it fronts an LLM. ai_metrics adds kong_ai_llm_* — requests, + # tokens by type, cost and provider latency, per model and per consumer. + # Without it none of that reaches Prometheus, whatever ai-proxy logs. + ai_metrics: true status_code_metrics: true latency_metrics: true bandwidth_metrics: true @@ -66,12 +71,6 @@ config: http_endpoint: ${AUDIT_LOG_ENDPOINT} method: POST content_type: application/json - # Reads what the two capture plugins below stashed in kong.ctx.shared. Both - # are nil -- and silently omitted from the record -- if those plugins are not - # applied, so this degrades to metadata-only logging rather than breaking. - custom_fields_by_lua: - request_body: "return kong.ctx.shared.audit_request_body" - response_body: "return kong.ctx.shared.audit_response_body" --- # prometheus and opentelemetry are bound control-plane-wide, matching the # shared template: they describe the gateway itself, not one service. @@ -122,74 +121,3 @@ spec: group: configuration.konghq.com kind: KongService name: ${NAME}-llm-proxy ---- -# dev only: captures the raw request/response bodies into kong.ctx.shared for -# the http-log plugin above to read back in the log phase. Same mechanism the -# five business domains use (-api/kong/mcp/audit-log-body-capture.yaml). -# Without it this gateway logged metadata only, so the prompts and completions -# going through the LLM were the one thing the audit trail did not record. -# -# The response accumulator matters more here than elsewhere: LLM answers stream -# as SSE, so body_filter runs once per chunk and the full completion only exists -# after the last one. -apiVersion: configuration.konghq.com/v1 -kind: KongPlugin -metadata: - name: ${NAME}-llm-audit-capture-request - namespace: ${NAMESPACE} -plugin: pre-function -config: - access: - - "kong.ctx.shared.audit_request_body = kong.request.get_raw_body()" ---- -apiVersion: configuration.konghq.com/v1 -kind: KongPlugin -metadata: - name: ${NAME}-llm-audit-capture-response - namespace: ${NAMESPACE} -plugin: post-function -config: - body_filter: - - | - local chunk = ngx.arg[1] - if chunk then - kong.ctx.shared.audit_response_body = (kong.ctx.shared.audit_response_body or "") .. chunk - end ---- -# Bound to the proxy service, like the audit log itself, so only real gateway -# traffic is captured. -apiVersion: configuration.konghq.com/v1alpha1 -kind: KongPluginBinding -metadata: - name: ${NAME}-llm-audit-capture-request-binding - namespace: ${NAMESPACE} -spec: - controlPlaneRef: - type: konnectNamespacedRef - konnectNamespacedRef: - name: ${CONTROL_PLANE_REF_NAME} - pluginRef: - name: ${NAME}-llm-audit-capture-request - targets: - serviceRef: - group: configuration.konghq.com - kind: KongService - name: ${NAME}-llm-proxy ---- -apiVersion: configuration.konghq.com/v1alpha1 -kind: KongPluginBinding -metadata: - name: ${NAME}-llm-audit-capture-response-binding - namespace: ${NAMESPACE} -spec: - controlPlaneRef: - type: konnectNamespacedRef - konnectNamespacedRef: - name: ${CONTROL_PLANE_REF_NAME} - pluginRef: - name: ${NAME}-llm-audit-capture-response - targets: - serviceRef: - group: configuration.konghq.com - kind: KongService - name: ${NAME}-llm-proxy