Send traces to Tempo and expose metrics to Prometheus

Point the OTLP exporter at Tempo instead of Jaeger.

Enable the built-in /metrics endpoint by setting METRICS_SECRET: without
it LibreChat answers 401 and exposes nothing. Prometheus scrapes it with a
Bearer token from a dedicated job, since annotation-based discovery cannot
carry credentials.

A Kubernetes Secret is namespaced, so the same value must also exist as
`librechat-metrics` in the observability namespace — one side verifies the
token, the other presents it. secret.example.yaml documents the pairing;
the real secret.yaml stays gitignored.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

k8s/configmap.yaml

@@ -6,14 +6,29 @@ metadata:
 data:
   HOST: "0.0.0.0"
   PORT: "3080"
-  DOMAIN_CLIENT: "https://${DOMAIN}"
+  DOMAIN_CLIENT: "https://librechat.sttlab.local"
-  DOMAIN_SERVER: "https://${DOMAIN}"
+  DOMAIN_SERVER: "https://librechat.sttlab.local"
   MEILI_HOST: "http://meilisearch:7700"
   NODE_ENV: "production"
   ALLOW_EMAIL_LOGIN: "true"
   ALLOW_REGISTRATION: "true"
-  ALLOW_SOCIAL_LOGIN: "false"
+  ALLOW_SOCIAL_LOGIN: "true"
-  ALLOW_SOCIAL_REGISTRATION: "false"
+  ALLOW_SOCIAL_REGISTRATION: "true"
+  # Keycloak SSO, additive alongside email/password login (not replacing it
+  # — ALLOW_EMAIL_LOGIN stays true). OPENID_CLIENT_SECRET/OPENID_SESSION_SECRET
+  # are in librechat-secrets (see librechat/deployment.yaml).
+  OPENID_ISSUER: "https://keycloak.sttlab.local/realms/classic-models"
+  OPENID_CLIENT_ID: "librechat"
+  OPENID_CALLBACK_URL: "/oauth/openid/callback"
+  OPENID_SCOPE: "openid profile email librechat-audience"
+  OPENID_BUTTON_LABEL: "classic-models SSO"
+  # RP-initiated logout — without this, switching Keycloak users silently
+  # reuses the existing SSO session instead of re-prompting.
+  OPENID_USE_END_SESSION_ENDPOINT: "true"
   SESSION_EXPIRY: "900000"
   REFRESH_TOKEN_EXPIRY: "604800000"
   DEBUG_LOGGING: "false"
+  OTEL_TRACING_ENABLED: "true"
+  OTEL_SERVICE_NAME: "librechat"
+  OTEL_EXPORTER_OTLP_ENDPOINT: "http://tempo.observability.svc.cluster.local:4318"
+  OTEL_TRACES_EXPORTER: "otlp"

k8s/ingress/ingress.yaml

@@ -9,17 +9,25 @@ metadata:
     nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
     nginx.ingress.kubernetes.io/proxy-buffering: "off"
     nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
-    nginx.ingress.kubernetes.io/configuration-snippet: |
+    nginx.ingress.kubernetes.io/proxy-buffer-size: "128k"
-      proxy_set_header Upgrade $http_upgrade;
+    nginx.ingress.kubernetes.io/proxy-buffers-number: "8"
-      proxy_set_header Connection "upgrade";
+    nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "256k"
+    # configuration-snippet (WebSocket Upgrade/Connection headers) removed —
+    # this cluster's ingress-nginx has snippet annotations disabled
+    # (admission webhook rejects them outright). If any LibreChat feature
+    # turns out to need raw WebSocket upgrade specifically (vs. the SSE/
+    # HTTP-1.1 streaming already covered by proxy-buffering/proxy-http-version
+    # above), this needs a different mechanism — e.g. a ConfigMap-based
+    # global nginx snippet if the cluster admin allows one, since per-Ingress
+    # snippets are blocked here.
 spec:
   ingressClassName: nginx
   tls:
     - hosts:
-        - librechat.sttlab.pc
+        - librechat.sttlab.local
       secretName: sttlab-tls
   rules:
-    - host: librechat.sttlab.pc
+    - host: librechat.sttlab.local
       http:
         paths:
           - path: /

k8s/librechat/configmap-app.yaml

@@ -5,11 +5,97 @@ metadata:
   namespace: librechat
 data:
   librechat.yaml: |
-    version: 1.3.9
+    version: 1.3.13
     mcpSettings:
       allowedDomains:
-        - "http://mcp-jira.mcp.svc.cluster.local:9000"
+        - "https://catalog-dev.sttlab.local"
+        - "https://customers-dev.sttlab.local"
+        - "https://payments-dev.sttlab.local"
+        - "https://sales-dev.sttlab.local"
+        - "https://workforce-dev.sttlab.local"
+        - "https://keycloak.sttlab.local"
     mcpServers:
-      jira:
+      catalog-mcp:
         type: streamable-http
-        url: http://mcp-jira.mcp.svc.cluster.local:9000/mcp
+        url: https://catalog-dev.sttlab.local/catalog-mcp
+        oauth:
+          authorization_url: https://keycloak.sttlab.local/realms/classic-models/protocol/openid-connect/auth
+          token_url: https://keycloak.sttlab.local/realms/classic-models/protocol/openid-connect/token
+          client_id: librechat
+          client_secret: '${LIBRECHAT_OIDC_CLIENT_SECRET}'
+          redirect_uri: https://librechat.sttlab.local/api/mcp/catalog-mcp/oauth/callback
+          scope: 'openid mcp-catalog-audience kong-token-exchange-broker-audience'
+          grant_types_supported: ['authorization_code', 'refresh_token']
+          token_endpoint_auth_methods_supported: ['client_secret_post']
+      customers-mcp:
+        type: streamable-http
+        url: https://customers-dev.sttlab.local/customers-mcp
+        oauth:
+          authorization_url: https://keycloak.sttlab.local/realms/classic-models/protocol/openid-connect/auth
+          token_url: https://keycloak.sttlab.local/realms/classic-models/protocol/openid-connect/token
+          client_id: librechat
+          client_secret: '${LIBRECHAT_OIDC_CLIENT_SECRET}'
+          redirect_uri: https://librechat.sttlab.local/api/mcp/customers-mcp/oauth/callback
+          scope: 'openid mcp-customers-audience kong-token-exchange-broker-audience'
+          grant_types_supported: ['authorization_code', 'refresh_token']
+          token_endpoint_auth_methods_supported: ['client_secret_post']
+      payments-mcp:
+        type: streamable-http
+        url: https://payments-dev.sttlab.local/payments-mcp
+        oauth:
+          authorization_url: https://keycloak.sttlab.local/realms/classic-models/protocol/openid-connect/auth
+          token_url: https://keycloak.sttlab.local/realms/classic-models/protocol/openid-connect/token
+          client_id: librechat
+          client_secret: '${LIBRECHAT_OIDC_CLIENT_SECRET}'
+          redirect_uri: https://librechat.sttlab.local/api/mcp/payments-mcp/oauth/callback
+          scope: 'openid mcp-payments-audience kong-token-exchange-broker-audience'
+          grant_types_supported: ['authorization_code', 'refresh_token']
+          token_endpoint_auth_methods_supported: ['client_secret_post']
+      sales-mcp:
+        type: streamable-http
+        url: https://sales-dev.sttlab.local/sales-mcp
+        oauth:
+          authorization_url: https://keycloak.sttlab.local/realms/classic-models/protocol/openid-connect/auth
+          token_url: https://keycloak.sttlab.local/realms/classic-models/protocol/openid-connect/token
+          client_id: librechat
+          client_secret: '${LIBRECHAT_OIDC_CLIENT_SECRET}'
+          redirect_uri: https://librechat.sttlab.local/api/mcp/sales-mcp/oauth/callback
+          scope: 'openid mcp-sales-audience kong-token-exchange-broker-audience'
+          grant_types_supported: ['authorization_code', 'refresh_token']
+          token_endpoint_auth_methods_supported: ['client_secret_post']
+      workforce-mcp:
+        type: streamable-http
+        url: https://workforce-dev.sttlab.local/workforce-mcp
+        oauth:
+          authorization_url: https://keycloak.sttlab.local/realms/classic-models/protocol/openid-connect/auth
+          token_url: https://keycloak.sttlab.local/realms/classic-models/protocol/openid-connect/token
+          client_id: librechat
+          client_secret: '${LIBRECHAT_OIDC_CLIENT_SECRET}'
+          redirect_uri: https://librechat.sttlab.local/api/mcp/workforce-mcp/oauth/callback
+          scope: 'openid mcp-workforce-audience kong-token-exchange-broker-audience'
+          grant_types_supported: ['authorization_code', 'refresh_token']
+          token_endpoint_auth_methods_supported: ['client_secret_post']
+    endpoints:
+      custom:
+        # kong-v2's platform-llm proxy (not the old kube/llms/openrouter lab)
+        # — key-auth, same shared-key limitation as before (LibreChat's
+        # custom endpoints don't support OAuth/OIDC on this outbound leg).
+        # baseURL is `/llm/v1`, not the full path: LibreChat appends
+        # `/chat/completions` itself.
+        - name: 'OpenRouter'
+          apiKey: '${PLATFORM_LLM_LIBRECHAT_APIKEY}'
+          baseURL: 'https://llm-dev.sttlab.local/llm/v1'
+          headers:
+            apikey: '${PLATFORM_LLM_LIBRECHAT_APIKEY}'
+          models:
+            default:
+              - 'google/gemini-3.7-flash'
+              - 'anthropic/claude-sonnet-5'
+              - 'openai/gpt-5.6-luna'
+              - 'deepseek/deepseek-v4-flash'
+              - 'openai/gpt-oss-120b'
+              - 'mistralai/mistral-nemo'
+            fetch: false
+          titleConvo: true
+          titleModel: 'google/gemini-3.7-flash'
+          modelDisplayLabel: 'OpenRouter'

k8s/librechat/deployment.yaml

@@ -12,7 +12,24 @@ spec:
     metadata:
       labels:
         app: librechat
+      annotations:
+        # Prometheus' dedicated `librechat` job (bearer token) targets this
+        # pod; the annotation is a selection marker, not a scrape config.
+        prometheus.io/scrape-with-auth: "true"
+        prometheus.io/port: "3080"
     spec:
+      # *.sttlab.local are lab-only hostnames (self-signed cert), not real
+      # DNS — this pod needs an explicit mapping to each Gateway's stable
+      # in-cluster ClusterIP to resolve them at all.
+      # catalog-dev: kong-v2 catalog-api Gateway (dataplane-ingress-catalog-dev-v4bkm-xqq6v), REST+MCP share this hostname now.
+      # llm-dev: kong-v2's platform-llm Gateway (dataplane-ingress-llm-dev-rlrxf-7fkb7).
+      hostAliases:
+        - ip: "192.168.194.225"
+          hostnames:
+            - "catalog-dev.sttlab.local"
+        - ip: "192.168.194.153"
+          hostnames:
+            - "llm-dev.sttlab.local"
       initContainers:
         - name: wait-for-mongodb
           image: busybox:1.36
@@ -81,13 +98,58 @@ spec:
                 secretKeyRef:
                   name: librechat-secrets
                   key: ANTHROPIC_API_KEY
+            - name: PLATFORM_LLM_LIBRECHAT_APIKEY
+              valueFrom:
+                secretKeyRef:
+                  name: librechat-secrets
+                  key: PLATFORM_LLM_LIBRECHAT_APIKEY
+            - name: LIBRECHAT_OIDC_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: librechat-secrets
+                  key: LIBRECHAT_OIDC_CLIENT_SECRET
+            # Same Keycloak client secret as LIBRECHAT_OIDC_CLIENT_SECRET
+            # above (one Keycloak client, "librechat", used both for the
+            # app's own SSO login and for each MCP server's OAuth flow) —
+            # LibreChat's OIDC-login code path reads this specific env var
+            # name, so it's duplicated rather than referenced once.
+            - name: OPENID_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: librechat-secrets
+                  key: LIBRECHAT_OIDC_CLIENT_SECRET
+            - name: OPENID_SESSION_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: librechat-secrets
+                  key: OPENID_SESSION_SECRET
+            # Enables /metrics (401 without it). Scraped by the Prometheus in
+            # the observability namespace using a Bearer token.
+            - name: METRICS_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: librechat-secrets
+                  key: METRICS_SECRET
+            # jira-mcp.sttlab.local / konggw.sttlab.local serve a self-signed
+            # cert (see hostAliases comment above) — Node rejects it by
+            # default; NODE_EXTRA_CA_CERTS trusts just this one CA, unlike
+            # NODE_TLS_REJECT_UNAUTHORIZED=0 which would disable TLS
+            # verification globally.
+            - name: NODE_EXTRA_CA_CERTS
+              value: /etc/sttlab-tls/tls.crt
+            # 512Mi was too tight for LibreChat under real use (MCP clients,
+            # OTel instrumentation) and caused repeated V8 heap-limit crashes.
+            # Cap old-space explicitly, leaving headroom under the new
+            # container limit for non-heap memory (buffers, native modules).
+            - name: NODE_OPTIONS
+              value: "--max-old-space-size=1200"
           resources:
             requests:
               cpu: 100m
-              memory: 256Mi
+              memory: 512Mi
             limits:
               cpu: 500m
-              memory: 512Mi
+              memory: 1536Mi
           livenessProbe:
             httpGet:
               path: /health
@@ -104,7 +166,16 @@ spec:
             - name: app-config
               mountPath: /app/librechat.yaml
               subPath: librechat.yaml
+            - name: sttlab-tls-cert
+              mountPath: /etc/sttlab-tls
+              readOnly: true
       volumes:
         - name: app-config
           configMap:
             name: librechat-app-config
+        - name: sttlab-tls-cert
+          secret:
+            secretName: sttlab-tls
+            items:
+              - key: tls.crt
+                path: tls.crt

k8s/secret.example.yaml

@@ -15,3 +15,16 @@ stringData:
   # Optional: fill in your AI provider API keys
   OPENAI_API_KEY: ""
   ANTHROPIC_API_KEY: ""
+  # platform-llm proxy consumer key (kong-v2/platform-llm, consumer "librechat")
+  PLATFORM_LLM_LIBRECHAT_APIKEY: ""
+  LIBRECHAT_OIDC_CLIENT_SECRET: ""
+  OPENID_SESSION_SECRET: ""
+
+  # Enables the built-in Prometheus endpoint (/metrics on port 3080).
+  # Without it LibreChat answers 401. Prometheus scrapes it with
+  # `Authorization: Bearer <METRICS_SECRET>`: the SAME value must exist in the
+  # `librechat-metrics` secret of the observability namespace
+  # (see infra-observability/k8s/prometheus/). A Kubernetes Secret is
+  # namespaced, so the value has to be duplicated — one side verifies the
+  # token, the other presents it.
+  METRICS_SECRET: "CHANGE_ME_METRICS_SECRET_MIN_32_CHARS"