kong-platform-llm/kong/observability.yaml

# Observability plugins for the AI gateway.
#
# The five business domains get these from k8s-platform's
# dataplane-globalpolicy-template, but that template also carries
# openid-connect, rate-limiting and cors — all of which this gateway already
# defines its own AI-specific versions of. Applying it here would duplicate
# them, so the three observability plugins are declared locally instead,
# with the same configuration as the shared template.
#
# That parity is manual and has already broken once: the propagation block
# below was added to the shared template and not here, so this gateway kept
# starting a fresh trace per request while the five domains propagated
# correctly. Any change to the shared template's observability plugins has to
# be mirrored here by hand.
apiVersion: configuration.konghq.com/v1
kind: KongPlugin
metadata:
  name: ${NAME}-llm-prometheus
  namespace: ${NAMESPACE}
plugin: prometheus
config:
  status_code_metrics: true
  latency_metrics: true
  bandwidth_metrics: true
  upstream_health_metrics: true
---
apiVersion: configuration.konghq.com/v1
kind: KongPlugin
metadata:
  name: ${NAME}-llm-opentelemetry
  namespace: ${NAMESPACE}
plugin: opentelemetry
config:
  traces_endpoint: ${OTEL_TRACES_ENDPOINT}
  resource_attributes:
    service.name: ${NAME}
  sampling_rate: 1
  http_response_header_for_traceid: X-Trace-Id
  # Without this block every hop starts its own trace: a single end-to-end
  # transaction shows up in Tempo as a dozen unrelated traces, one per service.
  # `extract` is what makes the gateway continue the caller's trace instead of
  # opening a new one, and `inject` is what carries it to the upstream so the
  # backend can continue it in turn.
  #
  # This gateway is where LibreChat enters, so without it the chain breaks at
  # the very first hop and nothing downstream can be correlated.
  propagation:
    extract:
      - w3c
      - b3
      - jaeger
    inject:
      - w3c
    default_format: w3c
    clear: []
---
apiVersion: configuration.konghq.com/v1
kind: KongPlugin
metadata:
  name: ${NAME}-llm-audit-log
  namespace: ${NAMESPACE}
# dev only: the raw Authorization header is captured in clear text, which is
# not acceptable as-is outside dev.
plugin: http-log
config:
  http_endpoint: ${AUDIT_LOG_ENDPOINT}
  method: POST
  content_type: application/json
  # Reads what the two capture plugins below stashed in kong.ctx.shared. Both
  # are nil -- and silently omitted from the record -- if those plugins are not
  # applied, so this degrades to metadata-only logging rather than breaking.
  custom_fields_by_lua:
    request_body: "return kong.ctx.shared.audit_request_body"
    response_body: "return kong.ctx.shared.audit_response_body"
---
# prometheus and opentelemetry are bound control-plane-wide, matching the
# shared template: they describe the gateway itself, not one service.
apiVersion: configuration.konghq.com/v1alpha1
kind: KongPluginBinding
metadata:
  name: ${NAME}-llm-prometheus-binding
  namespace: ${NAMESPACE}
spec:
  controlPlaneRef:
    type: konnectNamespacedRef
    konnectNamespacedRef:
      name: ${CONTROL_PLANE_REF_NAME}
  pluginRef:
    name: ${NAME}-llm-prometheus
  scope: GlobalInControlPlane
---
apiVersion: configuration.konghq.com/v1alpha1
kind: KongPluginBinding
metadata:
  name: ${NAME}-llm-opentelemetry-binding
  namespace: ${NAMESPACE}
spec:
  controlPlaneRef:
    type: konnectNamespacedRef
    konnectNamespacedRef:
      name: ${CONTROL_PLANE_REF_NAME}
  pluginRef:
    name: ${NAME}-llm-opentelemetry
  scope: GlobalInControlPlane
---
# The audit log is bound to the proxy service, like the business domains bind
# theirs, so only real gateway traffic is captured.
apiVersion: configuration.konghq.com/v1alpha1
kind: KongPluginBinding
metadata:
  name: ${NAME}-llm-audit-log-binding
  namespace: ${NAMESPACE}
spec:
  controlPlaneRef:
    type: konnectNamespacedRef
    konnectNamespacedRef:
      name: ${CONTROL_PLANE_REF_NAME}
  pluginRef:
    name: ${NAME}-llm-audit-log
  targets:
    serviceRef:
      group: configuration.konghq.com
      kind: KongService
      name: ${NAME}-llm-proxy
---
# dev only: captures the raw request/response bodies into kong.ctx.shared for
# the http-log plugin above to read back in the log phase. Same mechanism the
# five business domains use (<domain>-api/kong/mcp/audit-log-body-capture.yaml).
# Without it this gateway logged metadata only, so the prompts and completions
# going through the LLM were the one thing the audit trail did not record.
#
# The response accumulator matters more here than elsewhere: LLM answers stream
# as SSE, so body_filter runs once per chunk and the full completion only exists
# after the last one.
apiVersion: configuration.konghq.com/v1
kind: KongPlugin
metadata:
  name: ${NAME}-llm-audit-capture-request
  namespace: ${NAMESPACE}
plugin: pre-function
config:
  access:
    - "kong.ctx.shared.audit_request_body = kong.request.get_raw_body()"
---
apiVersion: configuration.konghq.com/v1
kind: KongPlugin
metadata:
  name: ${NAME}-llm-audit-capture-response
  namespace: ${NAMESPACE}
plugin: post-function
config:
  body_filter:
    - |
      local chunk = ngx.arg[1]
      if chunk then
        kong.ctx.shared.audit_response_body = (kong.ctx.shared.audit_response_body or "") .. chunk
      end
---
# Bound to the proxy service, like the audit log itself, so only real gateway
# traffic is captured.
apiVersion: configuration.konghq.com/v1alpha1
kind: KongPluginBinding
metadata:
  name: ${NAME}-llm-audit-capture-request-binding
  namespace: ${NAMESPACE}
spec:
  controlPlaneRef:
    type: konnectNamespacedRef
    konnectNamespacedRef:
      name: ${CONTROL_PLANE_REF_NAME}
  pluginRef:
    name: ${NAME}-llm-audit-capture-request
  targets:
    serviceRef:
      group: configuration.konghq.com
      kind: KongService
      name: ${NAME}-llm-proxy
---
apiVersion: configuration.konghq.com/v1alpha1
kind: KongPluginBinding
metadata:
  name: ${NAME}-llm-audit-capture-response-binding
  namespace: ${NAMESPACE}
spec:
  controlPlaneRef:
    type: konnectNamespacedRef
    konnectNamespacedRef:
      name: ${CONTROL_PLANE_REF_NAME}
  pluginRef:
    name: ${NAME}-llm-audit-capture-response
  targets:
    serviceRef:
      group: configuration.konghq.com
      kind: KongService
      name: ${NAME}-llm-proxy