commit 3937b4f36698396ed3dd0557f925cd774fceb232 Author: Stéphane Tailland Date: Tue Aug 25 07:27:38 2026 +0200 Declarative Grafana / Prometheus / Loki / Tempo observability stack Deploys the whole stack into the observability namespace of the local cluster with Kustomize alone — no helm install, no dashboard created by hand in the UI. The Makefile carries every command: deploy, dashboards, diagnostics, PVC flush, teardown. Dashboards are files, never UI objects. Each carries a stable uid, which is what keeps URLs, cross-links and alerts alive across redeployments, and references datasources by stable uid rather than by name. The dashboard ConfigMap is generated without a name-suffix hash so applying it updates it in place and Grafana reloads without a restart, while the provisioning ConfigMaps keep their hash precisely so that a change there does force one. Prometheus scrapes the kubelet /metrics/resource endpoint rather than /metrics/cadvisor: on this cluster cadvisor exposes machine_* series only and no container_* at all, which would leave every CPU and memory panel empty. Alloy ships every pod's logs to Loki through the Kubernetes API, so no /var/log mount is needed. Tempo's metrics_generator feeds span metrics and the service graph back into Prometheus, and the datasources are wired so a log line's trace_id opens the matching trace. Co-Authored-By: Claude Opus 5 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c75ac18 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.DS_Store +*.tmp +# Real secrets: only secret.example.yaml is versioned +k8s/secret.yaml +k8s/prometheus/secret.yaml diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2ac2a3a --- /dev/null +++ b/Makefile @@ -0,0 +1,356 @@ +# ============================================================================= +# infra-observability -- Grafana / Prometheus / Loki / Tempo stack (+ Alloy) +# Fully declarative: kustomize + kubectl. No imperative `helm install`. +# +# make -> help +# make deploy -> deploy everything +# make diag -> full diagnostics +# ============================================================================= + +SHELL := /bin/bash +.DEFAULT_GOAL := help +# Every target prints its own output, so commands are not echoed. +.SILENT: + +NS ?= observability +CTX ?= orbstack +KUBECTL := kubectl --context=$(CTX) +K8S := k8s +DASHBOARDS := $(K8S)/grafana/dashboards +INGRESS_IP ?= 192.168.139.101 +HOSTS := grafana.sttlab.local prometheus.sttlab.local alloy.sttlab.local + +# Components backed by a PVC (this is also the flush order) +STATEFUL := prometheus loki tempo grafana +# Restartable workloads +WORKLOADS := deployment/prometheus deployment/loki deployment/tempo deployment/grafana daemonset/alloy + +C_OK := \033[0;32m +C_WARN := \033[0;33m +C_ERR := \033[0;31m +C_HEAD := \033[1;36m +C_OFF := \033[0m + +# ----------------------------------------------------------------------------- +# Help +# ----------------------------------------------------------------------------- +.PHONY: help +help: ## Show this help + printf "$(C_HEAD)infra-observability$(C_OFF) -- namespace $(C_OK)$(NS)$(C_OFF) / context $(C_OK)$(CTX)$(C_OFF)\n\n" + awk 'BEGIN {FS = ":.*?## "} \ + /^[a-zA-Z0-9_-]+:.*?## / {printf " $(C_OK)%-24s$(C_OFF) %s\n", $$1, $$2} \ + /^##@/ {printf "\n$(C_HEAD)%s$(C_OFF)\n", substr($$0, 5)}' $(MAKEFILE_LIST) + printf "\n" + +##@ Deployment + +.PHONY: deploy +deploy: check-context secret ## Deploy the whole stack (runtimes + dashboards) + $(KUBECTL) apply -k $(K8S) + $(MAKE) --no-print-directory wait + +.PHONY: deploy-runtime +deploy-runtime: check-context secret ## Deploy runtimes only (no dashboards) + $(KUBECTL) apply -f $(K8S)/namespace.yaml -f $(K8S)/secret.yaml + $(KUBECTL) apply -k $(K8S)/prometheus + $(KUBECTL) apply -k $(K8S)/loki + $(KUBECTL) apply -k $(K8S)/tempo + $(KUBECTL) apply -k $(K8S)/alloy + $(KUBECTL) apply -k $(K8S)/grafana + $(KUBECTL) apply -k $(K8S)/ingress + $(MAKE) --no-print-directory wait + +.PHONY: prometheus loki tempo grafana alloy ingress +prometheus: check-context ## Deploy/update Prometheus only + $(KUBECTL) apply -k $(K8S)/prometheus +loki: check-context ## Deploy/update Loki only + $(KUBECTL) apply -k $(K8S)/loki +tempo: check-context ## Deploy/update Tempo only + $(KUBECTL) apply -k $(K8S)/tempo +grafana: check-context ## Deploy/update the Grafana runtime only (datasources included) + $(KUBECTL) apply -k $(K8S)/grafana +alloy: check-context ## Deploy/update Alloy only (log collection) + $(KUBECTL) apply -k $(K8S)/alloy +ingress: check-context ## Deploy/update the Ingresses and the certificate + $(KUBECTL) apply -k $(K8S)/ingress + +.PHONY: secret +secret: ## Generate k8s/secret.yaml (random admin password) if absent + if [ ! -f $(K8S)/secret.yaml ]; then \ + pw=$$(LC_ALL=C tr -dc 'A-Za-z0-9' |$$pw|" $(K8S)/secret.example.yaml > $(K8S)/secret.yaml; \ + printf "$(C_OK)secret.yaml generated$(C_OFF) -- admin password: $$pw\n"; \ + else \ + printf "secret.yaml already present (use make grafana-password to read it)\n"; \ + fi + +##@ Dashboards (declarative, stable UIDs) + +.PHONY: dashboards +dashboards: check-context dashboards-check ## Apply dashboards only (no Grafana restart) + $(KUBECTL) apply -k $(DASHBOARDS) + printf "$(C_OK)grafana-dashboards ConfigMap updated$(C_OFF) -- visible in Grafana within ~70s\n" + printf " (kubelet volume sync ~60s + provisioner poll 10s, no restart).\n" + printf " For immediate effect: $(C_WARN)make restart-grafana$(C_OFF)\n" + +.PHONY: dashboards-sync +dashboards-sync: ## Regenerate the .json list in dashboards/kustomization.yaml + cd $(DASHBOARDS) && python3 -c "import glob,re,pathlib; \ +p=pathlib.Path('kustomization.yaml'); s=p.read_text(); \ +files=sorted(glob.glob('*.json')); \ +block=''.join(' - %s\n' % f for f in files); \ +s=re.sub(r'(# >>> dashboards.*?>>>\n).*?( # <<< dashboards <<<)', lambda m: m.group(1)+block+m.group(2), s, flags=re.S); \ +p.write_text(s); print('%d dashboard(s) referenced' % len(files))" + $(MAKE) --no-print-directory dashboards-check + +.PHONY: dashboards-check +dashboards-check: ## Validate the JSON files and UID uniqueness/stability + fail=0; \ + for f in $(DASHBOARDS)/*.json; do \ + jq -e . "$$f" >/dev/null 2>&1 || { printf "$(C_ERR)invalid JSON$(C_OFF): %s\n" "$$f"; fail=1; continue; }; \ + uid=$$(jq -r '.uid // empty' "$$f"); \ + [ -n "$$uid" ] || { printf "$(C_ERR)missing uid$(C_OFF): %s\n" "$$f"; fail=1; }; \ + base=$$(basename "$$f" .json); \ + [ "$$uid" = "$$base" ] || printf "$(C_WARN)filename != uid$(C_OFF): %s (uid=%s)\n" "$$f" "$$uid"; \ + [ "$$(jq -r '.id // "null"' "$$f")" = "null" ] || printf "$(C_WARN)non-null id field$(C_OFF): %s (remove it)\n" "$$f"; \ + grep -q -- "- $$base.json" $(DASHBOARDS)/kustomization.yaml || { printf "$(C_ERR)not referenced in kustomization.yaml$(C_OFF): %s (run make dashboards-sync)\n" "$$f"; fail=1; }; \ + done; \ + dup=$$(jq -r '.uid' $(DASHBOARDS)/*.json | sort | uniq -d); \ + [ -z "$$dup" ] || { printf "$(C_ERR)duplicate UIDs$(C_OFF): %s\n" "$$dup"; fail=1; }; \ + [ $$fail -eq 0 ] && printf "$(C_OK)dashboards OK$(C_OFF) (%s)\n" "$$(ls $(DASHBOARDS)/*.json | wc -l | tr -d ' ')"; \ + exit $$fail + +.PHONY: dashboards-list +dashboards-list: ## List the dashboards actually loaded by Grafana + auth="$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_USER}' | base64 -d):$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_PASSWORD}' | base64 -d)"; \ + $(KUBECTL) -n $(NS) exec deployment/grafana -- curl -s -u "$$auth" 'http://localhost:3000/api/search?type=dash-db' \ + | jq -r '.[] | "\(.uid)\t\(.folderTitle // "General")\t\(.title)"' + +##@ Diagnostics + +.PHONY: status +status: ## Overview (pods, services, PVCs, ingresses) + $(KUBECTL) -n $(NS) get pods -o wide + echo + $(KUBECTL) -n $(NS) get svc,pvc,ingress + +.PHONY: diag +diag: ## Full diagnostics: state, PVCs, endpoints, certificate, events + printf "$(C_HEAD)== Pods ==$(C_OFF)\n"; $(KUBECTL) -n $(NS) get pods -o wide + printf "\n$(C_HEAD)== PVCs ==$(C_OFF)\n"; $(KUBECTL) -n $(NS) get pvc + printf "\n$(C_HEAD)== Endpoints ==$(C_OFF)\n"; $(KUBECTL) -n $(NS) get endpoints + printf "\n$(C_HEAD)== Ingresses ==$(C_OFF)\n"; $(KUBECTL) -n $(NS) get ingress + printf "\n$(C_HEAD)== Certificate ==$(C_OFF)\n";$(KUBECTL) -n $(NS) get certificate 2>/dev/null || echo "(cert-manager absent)" + printf "\n$(C_HEAD)== ConfigMaps ==$(C_OFF)\n";$(KUBECTL) -n $(NS) get cm + printf "\n$(C_HEAD)== Containers not ready ==$(C_OFF)\n" + $(KUBECTL) -n $(NS) get pods -o json | jq -r '.items[] | select(.status.containerStatuses != null) | .status.containerStatuses[] | select(.ready == false) | "\(.name)\trestarts=\(.restartCount)\t\(.state | keys[0]): \(.state[keys[0]].reason // "-")"' || true + printf "\n$(C_HEAD)== LibreChat scrape token ==$(C_OFF)\n" + $(MAKE) --no-print-directory check-metrics-token || true + printf "\n$(C_HEAD)== Last 15 events ==$(C_OFF)\n" + $(KUBECTL) -n $(NS) get events --sort-by=.lastTimestamp | tail -15 + +.PHONY: smoke +smoke: ## End-to-end checks: component health, datasources, dashboards, targets + # The Loki and Tempo images ship neither curl nor wget, so probes run from the + # Grafana pod, which has curl and sees the same Services as the datasources. + probe() { $(KUBECTL) -n $(NS) exec deployment/grafana -- curl -s --max-time 10 "$$1"; }; \ + auth="$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_USER}' | base64 -d):$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_PASSWORD}' | base64 -d)"; \ + printf "$(C_HEAD)== Component health ==$(C_OFF)\n"; \ + printf " %-12s %s\n" "prometheus" "$$(probe http://prometheus:9090/-/ready)"; \ + printf " %-12s %s\n" "loki" "$$(probe http://loki:3100/ready)"; \ + printf " %-12s %s\n" "tempo" "$$(probe http://tempo:3200/ready)"; \ + printf " %-12s %s\n" "grafana" "$$(probe http://localhost:3000/api/health | jq -c .)"; \ + printf " %-12s %s\n" "alloy" "$$(probe http://alloy:12345/-/ready)"; \ + printf "\n$(C_HEAD)== Grafana datasources ==$(C_OFF)\n"; \ + for uid in prometheus loki tempo; do \ + r=$$($(KUBECTL) -n $(NS) exec deployment/grafana -- curl -s -u "$$auth" -X POST "http://localhost:3000/api/datasources/uid/$$uid/health" | jq -r '"\(.status) \(.message)"'); \ + printf " %-12s %s\n" "$$uid" "$$r"; \ + done; \ + printf "\n$(C_HEAD)== Provisioned dashboards ==$(C_OFF)\n"; \ + $(KUBECTL) -n $(NS) exec deployment/grafana -- curl -s -u "$$auth" 'http://localhost:3000/api/search?type=dash-db' \ + | jq -r '.[] | " \(.uid) \(.folderTitle // "General")/\(.title)"'; \ + printf "\n$(C_HEAD)== Namespaces seen by Loki ==$(C_OFF)\n"; \ + probe 'http://loki:3100/loki/api/v1/label/namespace/values' | jq -r '" " + (.data // [] | join(", "))'; \ + printf "\n$(C_HEAD)== Prometheus targets ==$(C_OFF)\n"; \ + probe http://prometheus:9090/api/v1/targets | jq -r '.data.activeTargets | group_by(.health)[] | " \(.[0].health)\t\(length)"' + +.PHONY: targets +targets: ## List Prometheus scrape targets and their state + $(KUBECTL) -n $(NS) exec deployment/prometheus -- wget -qO- http://localhost:9090/api/v1/targets \ + | jq -r '.data.activeTargets[] | "\(.health)\t\(.labels.job)\t\(.scrapeUrl)\t\(.lastError // "")"' | sort + +.PHONY: check-metrics-token +check-metrics-token: ## Check the LibreChat token matches across both namespaces + a=$$($(KUBECTL) -n $(NS) get secret librechat-metrics -o jsonpath='{.data.token}' 2>/dev/null | base64 -d); \ + b=$$($(KUBECTL) -n librechat get secret librechat-secrets -o jsonpath='{.data.METRICS_SECRET}' 2>/dev/null | base64 -d); \ + if [ -z "$$a" ] || [ -z "$$b" ]; then \ + printf "$(C_WARN)token not found$(C_OFF) (observability/librechat-metrics=%s, librechat/librechat-secrets=%s)\n" "$${a:+present}$${a:-absent}" "$${b:+present}$${b:-absent}"; \ + elif [ "$$a" = "$$b" ]; then \ + printf "$(C_OK)LibreChat token matches$(C_OFF) across observability and librechat\n"; \ + else \ + printf "$(C_ERR)DRIFT$(C_OFF): the two Secrets hold different values.\n"; \ + printf " The librechat scrape will fall back to 401. Realign both copies.\n"; \ + exit 1; \ + fi + +.PHONY: logs-prometheus logs-loki logs-tempo logs-grafana logs-alloy +logs-prometheus: ## Follow Prometheus logs + $(KUBECTL) -n $(NS) logs -f deployment/prometheus --tail=100 +logs-loki: ## Follow Loki logs + $(KUBECTL) -n $(NS) logs -f deployment/loki --tail=100 +logs-tempo: ## Follow Tempo logs + $(KUBECTL) -n $(NS) logs -f deployment/tempo --tail=100 +logs-grafana: ## Follow Grafana logs + $(KUBECTL) -n $(NS) logs -f deployment/grafana --tail=100 +logs-alloy: ## Follow Alloy logs + $(KUBECTL) -n $(NS) logs -f daemonset/alloy --tail=100 + +.PHONY: events +events: ## Namespace events, oldest to newest + $(KUBECTL) -n $(NS) get events --sort-by=.lastTimestamp + +.PHONY: top +top: ## Pod CPU/memory usage + $(KUBECTL) -n $(NS) top pods 2>/dev/null || echo "metrics-server unavailable" + +.PHONY: urls +urls: ## Show access URLs + printf " Grafana https://grafana.sttlab.local\n" + printf " Prometheus https://prometheus.sttlab.local\n" + printf " Alloy https://alloy.sttlab.local\n" + printf "\n In-cluster: http://prometheus:9090 - http://loki:3100 - http://tempo:3200\n" + printf " OTLP : tempo.$(NS).svc.cluster.local:4317 (gRPC) / :4318 (HTTP)\n" + +.PHONY: hosts +hosts: ## Show the required /etc/hosts entries + printf "# Add to /etc/hosts:\n" + for h in $(HOSTS); do printf "$(INGRESS_IP)\t%s\n" "$$h"; done + +.PHONY: grafana-password +grafana-password: ## Show the Grafana admin credentials + printf "user : %s\n" "$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_USER}' | base64 -d)" + printf "password : %s\n" "$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_PASSWORD}' | base64 -d)" + +.PHONY: pf-grafana pf-prometheus pf-loki pf-tempo pf-alloy +pf-grafana: ## Port-forward Grafana on :3000 + $(KUBECTL) -n $(NS) port-forward svc/grafana 3000:3000 +pf-prometheus: ## Port-forward Prometheus on :9090 + $(KUBECTL) -n $(NS) port-forward svc/prometheus 9090:9090 +pf-loki: ## Port-forward Loki on :3100 + $(KUBECTL) -n $(NS) port-forward svc/loki 3100:3100 +pf-tempo: ## Port-forward Tempo on :3200 + $(KUBECTL) -n $(NS) port-forward svc/tempo 3200:3200 +pf-alloy: ## Port-forward the Alloy UI on :12345 + $(KUBECTL) -n $(NS) port-forward svc/alloy 12345:12345 + +##@ Operations + +.PHONY: wait +wait: ## Wait for every workload to become ready + for w in $(WORKLOADS); do \ + printf " %-24s " "$$w"; \ + $(KUBECTL) -n $(NS) rollout status $$w --timeout=300s 2>&1 | tail -1; \ + done + +.PHONY: restart +restart: ## Restart every workload + $(KUBECTL) -n $(NS) rollout restart $(WORKLOADS) + $(MAKE) --no-print-directory wait + +.PHONY: restart-prometheus restart-loki restart-tempo restart-grafana restart-alloy +restart-prometheus: ## Restart Prometheus + $(KUBECTL) -n $(NS) rollout restart deployment/prometheus && $(KUBECTL) -n $(NS) rollout status deployment/prometheus +restart-loki: ## Restart Loki + $(KUBECTL) -n $(NS) rollout restart deployment/loki && $(KUBECTL) -n $(NS) rollout status deployment/loki +restart-tempo: ## Restart Tempo + $(KUBECTL) -n $(NS) rollout restart deployment/tempo && $(KUBECTL) -n $(NS) rollout status deployment/tempo +restart-grafana: ## Restart Grafana (dashboards picked up immediately) + $(KUBECTL) -n $(NS) rollout restart deployment/grafana && $(KUBECTL) -n $(NS) rollout status deployment/grafana +restart-alloy: ## Restart Alloy + $(KUBECTL) -n $(NS) rollout restart daemonset/alloy && $(KUBECTL) -n $(NS) rollout status daemonset/alloy + +.PHONY: reload-prometheus +reload-prometheus: ## Hot-reload the Prometheus config (no restart) + $(KUBECTL) -n $(NS) exec deployment/prometheus -- wget -q --post-data='' -O- http://localhost:9090/-/reload \ + && printf "$(C_OK)Prometheus configuration reloaded$(C_OFF)\n" + +##@ Persistence + +.PHONY: pvc +pvc: ## PVC state and the volumes behind them + $(KUBECTL) -n $(NS) get pvc -o custom-columns='NAME:.metadata.name,STATUS:.status.phase,CAPACITY:.status.capacity.storage,SC:.spec.storageClassName,VOLUME:.spec.volumeName' + echo + $(KUBECTL) -n $(NS) get pods -o json | jq -r '.items[] | . as $$p | (.spec.volumes[]? | select(.persistentVolumeClaim) | "\($$p.metadata.name)\t-> \(.persistentVolumeClaim.claimName)")' + +.PHONY: flush-pvc +flush-pvc: ## DESTRUCTIVE: wipe all persisted data (CONFIRM=yes) + $(call confirm,Wipe ALL data (metrics, logs, traces, Grafana)) + for c in $(STATEFUL); do $(MAKE) --no-print-directory _flush COMP=$$c; done + $(MAKE) --no-print-directory wait + +.PHONY: flush-prometheus flush-loki flush-tempo flush-grafana +flush-prometheus: ## DESTRUCTIVE: wipe the Prometheus TSDB (CONFIRM=yes) + $(call confirm,Wipe the Prometheus TSDB) + $(MAKE) --no-print-directory _flush COMP=prometheus +flush-loki: ## DESTRUCTIVE: wipe the Loki logs (CONFIRM=yes) + $(call confirm,Wipe the Loki logs) + $(MAKE) --no-print-directory _flush COMP=loki +flush-tempo: ## DESTRUCTIVE: wipe the Tempo traces (CONFIRM=yes) + $(call confirm,Wipe the Tempo traces) + $(MAKE) --no-print-directory _flush COMP=tempo +flush-grafana: ## DESTRUCTIVE: wipe the Grafana database (CONFIRM=yes) + $(call confirm,Wipe the Grafana database (users and prefs; dashboards are reprovisioned)) + $(MAKE) --no-print-directory _flush COMP=grafana + +# Internal target: scale down -> delete PVC -> recreate -> scale up. +.PHONY: _flush +_flush: + printf "$(C_WARN)flushing %s$(C_OFF)\n" "$(COMP)" + $(KUBECTL) -n $(NS) scale deployment/$(COMP) --replicas=0 + $(KUBECTL) -n $(NS) wait --for=delete pod -l app=$(COMP) --timeout=120s 2>/dev/null || true + $(KUBECTL) -n $(NS) delete pvc $(COMP)-data --ignore-not-found --wait=true + $(KUBECTL) apply -k $(K8S)/$(COMP) + $(KUBECTL) -n $(NS) scale deployment/$(COMP) --replicas=1 + +##@ Validation and teardown + +.PHONY: build +build: ## Render the generated manifests to stdout + kustomize build $(K8S) + +.PHONY: diff +diff: check-context ## Diff the manifests against the cluster state + $(KUBECTL) diff -k $(K8S) || true + +.PHONY: validate +validate: dashboards-check ## Validate the manifests (build + server dry-run) + kustomize build $(K8S) >/dev/null && printf "$(C_OK)kustomize build OK$(C_OFF)\n" + $(KUBECTL) apply -k $(K8S) --dry-run=server >/dev/null && printf "$(C_OK)server dry-run OK$(C_OFF)\n" + +.PHONY: undeploy +undeploy: ## DESTRUCTIVE: delete the stack, PVCs kept (CONFIRM=yes) + $(call confirm,Delete the stack (PVCs are KEPT)) + $(KUBECTL) delete -k $(K8S) --ignore-not-found + +.PHONY: destroy +destroy: ## DESTRUCTIVE: delete the stack AND the PVCs (CONFIRM=yes) + $(call confirm,Delete the stack AND all data) + $(KUBECTL) delete -k $(K8S) --ignore-not-found + $(KUBECTL) -n $(NS) delete pvc --all --ignore-not-found + +.PHONY: check-context +check-context: + cur=$$(kubectl config current-context); \ + if [ "$$cur" != "$(CTX)" ]; then \ + printf "$(C_ERR)current context '%s' != '$(CTX)'$(C_OFF) (commands explicitly target $(CTX))\n" "$$cur"; \ + fi + +# Guard rail for destructive targets: requires CONFIRM=yes +define confirm +@if [ "$(CONFIRM)" != "yes" ]; then \ + printf "$(C_ERR)Destructive operation$(C_OFF): $(1)\n"; \ + printf "Re-run with $(C_WARN)CONFIRM=yes$(C_OFF): make $(C_WARN)$@ CONFIRM=yes$(C_OFF)\n"; \ + exit 1; \ +fi +endef diff --git a/README.md b/README.md new file mode 100644 index 0000000..1dc9800 --- /dev/null +++ b/README.md @@ -0,0 +1,281 @@ +# Infra Observability (Kubernetes / declarative) + +**Grafana 13.2 / Prometheus 3.14 / Loki 3.7 / Tempo 3.0** plus **Alloy 1.19**, +deployed fully declaratively (Kustomize) into the `observability` namespace of +the local `orbstack` cluster. + +Nothing is imperative: no `helm install`, no dashboard hand-created in the UI. +The entire desired state lives in this repository and `make deploy` reconciles it. + +## Repository layout + +```text +Makefile # Every command (make = help) +k8s/ +├── kustomization.yaml # Root Kustomize manifest +├── namespace.yaml # observability namespace +├── secret.yaml # Grafana admin credentials (GITIGNORED) +├── secret.example.yaml # Secret template +├── prometheus/ +│ ├── prometheus.yaml # Scrape config (Kubernetes SD) +│ ├── rbac.yaml # SA + discovery ClusterRole +│ ├── secret.yaml # LibreChat scrape token (GITIGNORED) +│ ├── secret.example.yaml # Token template +│ ├── pvc.yaml # Persistent TSDB (10Gi) +│ ├── deployment.yaml # Prometheus v3.14.0 +│ └── service.yaml # :9090 +├── loki/ +│ ├── loki.yaml # Monolithic config, filesystem storage +│ ├── pvc.yaml # Chunks + index (10Gi) +│ ├── deployment.yaml # Loki 3.7.6 (-target=all) +│ └── service.yaml # :3100 (HTTP) / :9096 (gRPC) +├── tempo/ +│ ├── tempo.yaml # Monolithic config + metrics-generator +│ ├── pvc.yaml # WAL + trace blocks (10Gi) +│ ├── deployment.yaml # Tempo 3.0.3 (-target=all) +│ └── service.yaml # :3200, OTLP :4317/:4318, Zipkin :9411 +├── alloy/ +│ ├── config.alloy # Pod logs -> Loki pipeline +│ ├── rbac.yaml # SA + ClusterRole (including pods/log) +│ ├── daemonset.yaml # Alloy v1.19.0 (one pod per node) +│ └── service.yaml # :12345 (UI + /metrics) +├── grafana/ +│ ├── provisioning/ +│ │ ├── datasources.yaml # Datasources with stable UIDs +│ │ └── dashboard-providers.yaml # File provider +│ ├── dashboards/ +│ │ ├── kustomization.yaml # ConfigMap WITHOUT hash (hot reload) +│ │ ├── obs-stack-health.json # uid: obs-stack-health +│ │ ├── obs-keycloak.json # uid: obs-keycloak +│ │ ├── obs-librechat.json # uid: obs-librechat +│ │ └── obs-classic-models-api.json +│ ├── pvc.yaml # Grafana SQLite database (5Gi) +│ ├── deployment.yaml # Grafana 13.2.0 +│ └── service.yaml # :3000 +└── ingress/ + ├── certificate.yaml # cert-manager Certificate (sttlab-local-ca) + └── ingress.yaml # Three NGINX Ingresses +``` + +## Getting started + +```bash +make # Help: every available target +make deploy # Deploy the whole stack +make hosts # /etc/hosts entries to add +make grafana-password # Admin credentials +make smoke # End-to-end verification +``` + +| URL | Purpose | +|---|---| +| `https://grafana.sttlab.local` | Grafana UI | +| `https://prometheus.sttlab.local` | Prometheus UI (targets, expression browser) | +| `https://alloy.sttlab.local` | Alloy UI (log pipeline state) | + +Sending traces from an in-cluster application: +`tempo.observability.svc.cluster.local:4317` (OTLP gRPC) or `:4318` (OTLP HTTP). + +## Declarative dashboards with stable UIDs + +This is the centrepiece of the repository: **a dashboard only ever exists as a +versioned JSON file**, never as an object created through the UI. + +### Rules + +1. **One `.json` file per dashboard**, under `k8s/grafana/dashboards/`. +2. **The `uid` field is stable and never changes.** It is what keeps URLs + (`/d//…`), cross-dashboard links, annotations and alerts alive across + every redeployment. By convention the filename equals the UID + (`obs-keycloak.json` -> `uid: obs-keycloak`). +3. **The `id` field must stay `null`** (or absent): it is an identifier internal + to a Grafana instance and is not portable. +4. **Datasources are referenced by stable UID**, never by name: + `"datasource": { "type": "prometheus", "uid": "prometheus" }`. + Available UIDs: `prometheus`, `loki`, `tempo`. +5. `allowUiUpdates: false`: any change made in the UI is overwritten on the next + reload. The file is the source of truth. + +### Adding a dashboard + +```bash +cp my-dashboard.json k8s/grafana/dashboards/obs-my-topic.json +# edit the JSON: "uid": "obs-my-topic", "id": null +make dashboards-sync # update the list in dashboards/kustomization.yaml +make dashboards # apply (without restarting Grafana) +``` + +`make dashboards-check` (run automatically by `dashboards` and `validate`) +verifies: valid JSON, `uid` present, `uid` unique, `id` null, file referenced in +the kustomization, filename/UID consistency. + +### How the hot reload works + +The `grafana-dashboards` ConfigMap is generated **without a name-suffix hash** +(`disableNameSuffixHash: true`), so `make dashboards` updates it *in place*: the +Grafana pod is not recreated and the file provider re-reads the directory +(`updateIntervalSeconds: 10`). + +Measured end-to-end delay: **about 70 seconds** (kubelet ConfigMap volume sync +~60s + provider poll 10s). For immediate effect: `make restart-grafana`. + +Conversely, the *provisioning* ConfigMaps (datasources, providers) **keep** their +hash suffix: those files are only read at startup, so the hash deliberately +forces a pod restart when they change. + +## Persistence + +| PVC | Component | Size | Contents | Retention | +|---|---|---|---|---| +| `prometheus-data` | Prometheus | 10Gi | TSDB | 15 d / 8 GB | +| `loki-data` | Loki | 10Gi | chunks + TSDB index | 168 h (compactor) | +| `tempo-data` | Tempo | 10Gi | WAL + trace blocks | 168 h | +| `grafana-data` | Grafana | 5Gi | SQLite database (users, prefs) | — | + +StorageClass `local-path` (the cluster default), `ReadWriteOnce`. The Deployments +use `strategy: Recreate`: with an RWO volume the old pod must release it before +the new one can start. + +Alloy's read positions live on a `hostPath` (`/var/lib/alloy`), so restarting the +DaemonSet does not re-ingest every log. + +### Wiping data + +```bash +make pvc # PVC state +make flush-loki CONFIRM=yes # wipe the logs only +make flush-prometheus CONFIRM=yes # wipe the TSDB only +make flush-pvc CONFIRM=yes # wipe EVERYTHING +``` + +The cycle is: `scale 0` -> wait for pod deletion -> `delete pvc` -> reapply the +kustomization (which recreates the PVC) -> `scale 1`. Every destructive target +requires `CONFIRM=yes`. + +Dashboards are **not** affected by `flush-grafana`: they are reprovisioned from +the ConfigMaps at startup. + +## Wiring the three signals + +- **Prometheus** discovers its targets through the Kubernetes API: apiserver, + kubelet, container resource metrics, plus any Service or Pod carrying the + `prometheus.io/scrape: "true"` annotation (`prometheus.io/port`, + `prometheus.io/path`). +- **Alloy** reads every pod's logs through the Kubernetes API (no `/var/log` + mount) and pushes them to Loki with `namespace`, `pod`, `container`, `node`, + `app`, `component` and `job` labels. +- **Tempo** receives OTLP, and its `metrics_generator` pushes span metrics and + service graphs to Prometheus through its remote-write receiver + (`--web.enable-remote-write-receiver`). +- **Grafana correlations**: Loki -> Tempo (`TraceID` derived field), + Tempo -> Loki (`tracesToLogsV2`), Tempo -> Prometheus (`serviceMap`, + `tracesToMetrics`), Prometheus exemplars -> Tempo. + +### Container resource metrics + +Prometheus scrapes the kubelet's **`/metrics/resource`** endpoint, not +`/metrics/cadvisor`. On this cluster the cadvisor endpoint answers with `machine_*` +series only and exposes no `container_*` at all, so CPU and memory panels would +stay empty. `/metrics/resource` — the source metrics-server itself reads — +provides `container_cpu_usage_seconds_total` and +`container_memory_working_set_bytes` with `namespace`/`pod`/`container` labels. + +kube-state-metrics is not deployed, so container restarts are derived from +changes in `container_start_time_seconds`. + +## Connected applications + +| Application | Traces | Metrics | Logs | Dashboard | +|---|---|---|---|---| +| `keycloak` | OTel Java agent → `tempo:4318` | native `/metrics` (management port 9000) | JSON + MDC `trace_id` | `obs-keycloak` | +| `librechat` | OTel SDK → `tempo:4318` | built-in `/metrics`, Bearer token | stdout | `obs-librechat` | +| `classic-models-api` | `opentelemetry-instrument` → `tempo:4318` | django-prometheus `/metrics` | JSON + `otelTraceID` | `obs-classic-models-api` | + +Their manifests live in the `infra-keycloak`, `infra-librechat` and +`classic-models-api` repositories; this repository only holds the platform. + +Each application dashboard carries the same four sections: an overview of RED +indicators, HTTP traffic, a runtime section specific to the stack in use, and a +**Logs** row, followed by container resources. + +The Logs row leans on Loki's `detected_level`, which is inferred at ingestion +time, so a single query works across all three log formats — Keycloak emits JSON, +LibreChat colour-coded text, and the Django API plain gunicorn access lines. No +per-application parser is needed. The volume panel is scoped to the application +container, while the two error panels cover the whole namespace so the +datastore's own failures surface alongside the application's. Where a log line +carries a `trace_id`, the Loki datasource's derived field turns it into a link +straight to the matching trace in Tempo. + +### Connecting an application + +1. **Traces** — point OTLP at Tempo: + `OTEL_EXPORTER_OTLP_ENDPOINT=http://tempo.observability.svc.cluster.local:4318` + (`:4317` for gRPC). RED span metrics and the service graph are then derived + automatically by Tempo's `metrics_generator`, with no extra instrumentation. +2. **Metrics** — annotate the **pod** (not the Service): + ```yaml + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9000" + prometheus.io/path: "/metrics" + ``` +3. **Logs** — nothing to do: Alloy collects every pod in the cluster. For Grafana + to jump from a log line to its trace, the line only needs to contain a + `trace_id` (the Loki datasource's derived field). + +### Token-protected endpoints + +Annotation-based discovery cannot carry credentials, so an authenticated target +gets its own `scrape_config` in `k8s/prometheus/prometheus.yaml`, with the token +mounted from a Secret — see the `librechat` job, which reads +`/etc/prometheus/secrets/librechat/token`. + +That token must be **identical** on both sides: `METRICS_SECRET` in the +`librechat-secrets` secret (namespace `librechat`) and the `token` key of the +`librechat-metrics` secret (namespace `observability`, file +`k8s/prometheus/secret.yaml`, gitignored). + +A Kubernetes Secret is namespaced and a pod can only mount ones from its own +namespace, so the same value has to exist twice — once for LibreChat, which +**verifies** the token, once for Prometheus, which **presents** it. The risk is +silent drift: regenerating one side without the other sends the scrape back to +401 without breaking anything else. `make check-metrics-token` (included in +`make diag`) compares both copies. + +## Diagnostics + +```bash +make status # pods, services, PVCs, ingresses +make diag # + endpoints, certificate, failing containers, events +make smoke # health of all five components, datasources, dashboards, targets +make targets # Prometheus targets and their last scrape error +make logs-loki # follow one component's logs +make top # CPU/memory usage +make pf-grafana # port-forward when the ingress is unavailable +make diff # difference between the repository and the cluster +make validate # kustomize build + server dry-run + dashboard checks +``` + +## Secrets + +`k8s/secret.yaml` and `k8s/prometheus/secret.yaml` are **gitignored**. +`make secret` generates the first from `secret.example.yaml` with a random admin +password if it does not exist (and is called automatically by `make deploy`). + +The TLS certificate is not versioned either: it is issued by the existing +`sttlab-local-ca` cert-manager ClusterIssuer (`k8s/ingress/certificate.yaml`). + +## Coexisting with what was already there + +The `observability` namespace already hosts two deployments **not managed by this +repository**, left untouched: + +- `jaeger` (installed by Helm) — it also exposes OTLP on `4317`/`4318`, but on its + own Service, so there is no conflict with Tempo. Since all three applications + now point at Tempo, Jaeger no longer receives anything; uninstall the Helm + release whenever you want a single trace backend. +- `audit-log-collector` (Fluent Bit) — writes to its own PVC. + +`make undeploy` and `make destroy` only run `kubectl delete -k k8s/`, so they +touch nothing but the resources described here. diff --git a/k8s/alloy/config.alloy b/k8s/alloy/config.alloy new file mode 100644 index 0000000..549d043 --- /dev/null +++ b/k8s/alloy/config.alloy @@ -0,0 +1,66 @@ +// Grafana Alloy -- ships the logs of every pod in the cluster to Loki. +// Reads through the Kubernetes API (no /var/log mount required). + +logging { + level = "info" + format = "logfmt" +} + +discovery.kubernetes "pods" { + role = "pod" +} + +discovery.relabel "pod_logs" { + targets = discovery.kubernetes.pods.targets + + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + target_label = "node" + } + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] + target_label = "app" + } + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_component"] + target_label = "component" + } + // job = /: the usual Loki convention. + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] + separator = "/" + target_label = "job" + } + // Only collect containers on the current node (DaemonSet). + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + regex = env("HOSTNAME_NODE") + action = "keep" + } +} + +loki.source.kubernetes "pods" { + targets = discovery.relabel.pod_logs.output + forward_to = [loki.write.default.receiver] +} + +loki.write "default" { + endpoint { + url = "http://loki:3100/loki/api/v1/push" + } + external_labels = { + cluster = "orbstack", + } +} diff --git a/k8s/alloy/daemonset.yaml b/k8s/alloy/daemonset.yaml new file mode 100644 index 0000000..1979247 --- /dev/null +++ b/k8s/alloy/daemonset.yaml @@ -0,0 +1,73 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: alloy + namespace: observability + labels: + app.kubernetes.io/name: alloy + app.kubernetes.io/component: collector + app.kubernetes.io/part-of: observability +spec: + selector: + matchLabels: + app: alloy + template: + metadata: + labels: + app: alloy + app.kubernetes.io/name: alloy + app.kubernetes.io/component: collector + app.kubernetes.io/part-of: observability + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "12345" + spec: + serviceAccountName: alloy + tolerations: + - operator: Exists + containers: + - name: alloy + image: grafana/alloy:v1.19.0 + args: + - run + - /etc/alloy/config.alloy + - --storage.path=/var/lib/alloy/data + - --server.http.listen-addr=0.0.0.0:12345 + - --disable-reporting + env: + # Used by config.alloy to tail only the pods on the current node. + - name: HOSTNAME_NODE + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - name: http + containerPort: 12345 + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: "1" + memory: 512Mi + readinessProbe: + httpGet: + path: /-/ready + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + volumeMounts: + - name: config + mountPath: /etc/alloy + - name: storage + mountPath: /var/lib/alloy + volumes: + - name: config + configMap: + name: alloy-config + - name: storage + # Read positions persisted on the node: no full log re-ingestion + # when the pod restarts. + hostPath: + path: /var/lib/alloy + type: DirectoryOrCreate diff --git a/k8s/alloy/kustomization.yaml b/k8s/alloy/kustomization.yaml new file mode 100644 index 0000000..2aa9290 --- /dev/null +++ b/k8s/alloy/kustomization.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: observability + +resources: + - rbac.yaml + - daemonset.yaml + - service.yaml + +configMapGenerator: + - name: alloy-config + files: + - config.alloy + +generatorOptions: + labels: + app.kubernetes.io/name: alloy + app.kubernetes.io/part-of: observability diff --git a/k8s/alloy/rbac.yaml b/k8s/alloy/rbac.yaml new file mode 100644 index 0000000..4c12818 --- /dev/null +++ b/k8s/alloy/rbac.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: alloy + namespace: observability + labels: + app.kubernetes.io/name: alloy + app.kubernetes.io/part-of: observability +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: observability-alloy + labels: + app.kubernetes.io/name: alloy + app.kubernetes.io/part-of: observability +rules: + - apiGroups: [""] + resources: + - nodes + - nodes/proxy + - nodes/metrics + - services + - endpoints + - pods + - namespaces + - events + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get", "list", "watch"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] + - nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: observability-alloy + labels: + app.kubernetes.io/name: alloy + app.kubernetes.io/part-of: observability +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: observability-alloy +subjects: + - kind: ServiceAccount + name: alloy + namespace: observability diff --git a/k8s/alloy/service.yaml b/k8s/alloy/service.yaml new file mode 100644 index 0000000..8e509fe --- /dev/null +++ b/k8s/alloy/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: alloy + namespace: observability + labels: + app.kubernetes.io/name: alloy + app.kubernetes.io/component: collector + app.kubernetes.io/part-of: observability +spec: + type: ClusterIP + selector: + app: alloy + ports: + - name: http + port: 12345 + targetPort: http diff --git a/k8s/grafana/dashboards/kustomization.yaml b/k8s/grafana/dashboards/kustomization.yaml new file mode 100644 index 0000000..2615de7 --- /dev/null +++ b/k8s/grafana/dashboards/kustomization.yaml @@ -0,0 +1,31 @@ +# Grafana dashboards -- declarative provisioning. +# +# Rules: +# * one .json file per dashboard, named after its UID; +# * every JSON carries a STABLE "uid" field (never regenerated): that is what +# keeps URLs, cross-links and alerts alive across redeployments; +# * `disableNameSuffixHash: true` -> the ConfigMap always keeps the same name, +# so `kubectl apply` updates it in place and Grafana reloads WITHOUT +# restarting the pod, in about 70s (see dashboard-providers.yaml). +# +# Adding a dashboard = drop the .json here and add it to `files:` below. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: observability + +configMapGenerator: + - name: grafana-dashboards + files: + # >>> dashboards (generated by `make dashboards-sync`) >>> + - obs-classic-models-api.json + - obs-keycloak.json + - obs-librechat.json + - obs-stack-health.json + # <<< dashboards <<< + +generatorOptions: + disableNameSuffixHash: true + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/component: dashboards + app.kubernetes.io/part-of: observability diff --git a/k8s/grafana/dashboards/obs-classic-models-api.json b/k8s/grafana/dashboards/obs-classic-models-api.json new file mode 100644 index 0000000..b076226 --- /dev/null +++ b/k8s/grafana/dashboards/obs-classic-models-api.json @@ -0,0 +1,1634 @@ +{ + "uid": "obs-classic-models-api", + "title": "Classic Models API — Overview", + "description": "HTTP traffic, per-view latency, MySQL access and logs for the Django API. django-prometheus metrics aggregated across gunicorn workers.", + "tags": [ + "observability", + "classic-models-api", + "django", + "application", + "metrics", + "logs" + ], + "editable": false, + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-6h", + "to": "now" + }, + "timezone": "browser", + "templating": { + "list": [] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Overview", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Requests / s", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(django_http_requests_total_by_method_total{namespace=\"classic-models\"}[$__rate_interval]))", + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "mappings": [], + "decimals": 2, + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto", + "justifyMode": "auto" + }, + "description": "Aggregated across both replicas and the 3 gunicorn workers of each (prometheus_client multiprocess mode)." + }, + { + "id": 3, + "type": "stat", + "title": "Error rate", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "(sum(rate(django_http_responses_total_by_status_total{namespace=\"classic-models\",status=~\"5..\"}[$__rate_interval])) or vector(0)) / clamp_min(sum(rate(django_http_responses_total_by_status_total{namespace=\"classic-models\"}[$__rate_interval])), 0.001)", + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "mappings": [], + "decimals": 2, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.01 + }, + { + "color": "red", + "value": 0.05 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto", + "justifyMode": "auto" + }, + "description": "5xx only: the API returns expected 401/404 that must not count as outages. `or vector(0)` is required: with no errors at all the numerator is an empty vector, and empty/x stays empty -- the panel would read \"No data\" instead of 0%. clamp_min covers the zero-traffic case." + }, + { + "id": 4, + "type": "stat", + "title": "Latency p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(django_http_requests_latency_seconds_by_view_method_bucket{namespace=\"classic-models\"}[$__rate_interval])))", + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "mappings": [], + "decimals": 3, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "red", + "value": 2 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto", + "justifyMode": "auto" + }, + "description": "Measured between django-prometheus' Before and After middleware, which bracket the whole stack." + }, + { + "id": 5, + "type": "stat", + "title": "SQL queries / s", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(django_db_execute_total{namespace=\"classic-models\"}[$__rate_interval]))", + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "mappings": [], + "decimals": 2, + "color": { + "mode": "fixed", + "fixedColor": "purple" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "id": 6, + "type": "row", + "title": "HTTP traffic", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "panels": [] + }, + { + "id": 7, + "type": "timeseries", + "title": "Responses by status class", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (class) (label_replace(rate(django_http_responses_total_by_status_total{namespace=\"classic-models\"}[$__rate_interval]), \"class\", \"$1xx\", \"status\", \"(.)..\"))", + "legendFormat": "{{class}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 70, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "linear", + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SUCCESS" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "REDIRECTION" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "CLIENT_ERROR" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SERVER_ERROR" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Request latency", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(django_http_requests_latency_seconds_by_view_method_bucket{namespace=\"classic-models\"}[$__rate_interval])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(django_http_requests_latency_seconds_by_view_method_bucket{namespace=\"classic-models\"}[$__rate_interval])))", + "legendFormat": "p95" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(django_http_requests_latency_seconds_by_view_method_bucket{namespace=\"classic-models\"}[$__rate_interval])))", + "legendFormat": "p99" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "light-blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "dark-blue" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 9, + "type": "bargauge", + "title": "Busiest views", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "topk(10, sum by (view) (rate(django_http_requests_latency_seconds_by_view_method_count{namespace=\"classic-models\"}[$__rate_interval])))", + "legendFormat": "{{view}}", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0, + "mappings": [], + "color": { + "mode": "continuous-BlPu" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "text", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 10, + "type": "timeseries", + "title": "p95 latency per view", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "topk(10, histogram_quantile(0.95, sum by (view, le) (rate(django_http_requests_latency_seconds_by_view_method_bucket{namespace=\"classic-models\"}[$__rate_interval]))))", + "legendFormat": "{{view}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Isolates the view responsible for a regression in the global p95." + }, + { + "id": 11, + "type": "timeseries", + "title": "Requests by HTTP method", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 23 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (method) (rate(django_http_requests_total_by_method_total{namespace=\"classic-models\"}[$__rate_interval]))", + "legendFormat": "{{method}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 12, + "type": "timeseries", + "title": "Response volume", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 23 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(django_http_responses_body_total_bytes_sum{namespace=\"classic-models\"}[$__rate_interval]))", + "legendFormat": "bytes/s" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": false, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Application egress, useful to spot missing pagination or an over-eager serializer." + }, + { + "id": 13, + "type": "row", + "title": "MySQL database", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "panels": [] + }, + { + "id": 14, + "type": "timeseries", + "title": "SQL queries executed", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 31 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (vendor) (rate(django_db_execute_total{namespace=\"classic-models\"}[$__rate_interval]))", + "legendFormat": "{{vendor}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 15, + "type": "timeseries", + "title": "SQL query latency", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 31 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(django_db_query_duration_seconds_bucket{namespace=\"classic-models\"}[$__rate_interval])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(django_db_query_duration_seconds_bucket{namespace=\"classic-models\"}[$__rate_interval])))", + "legendFormat": "p95" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(django_db_query_duration_seconds_bucket{namespace=\"classic-models\"}[$__rate_interval])))", + "legendFormat": "p99" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "light-blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "dark-blue" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 16, + "type": "timeseries", + "title": "New DB connections", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 31 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(django_db_new_connections_total{namespace=\"classic-models\"}[$__rate_interval]))", + "legendFormat": "connections/s" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": false, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Django opens one connection per request while CONN_MAX_AGE stays at 0. A rate close to the HTTP throughput confirms there is no connection reuse." + }, + { + "id": 17, + "type": "row", + "title": "Logs", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "panels": [] + }, + { + "id": 18, + "type": "timeseries", + "title": "Log volume by level", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (detected_level) (count_over_time({namespace=\"classic-models\",container=\"classic-models-api\"}[$__auto]))", + "legendFormat": "{{detected_level}}", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 80, + "showPoints": "never", + "barAlignment": 0, + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisLabel": "", + "axisPlacement": "auto" + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "critical" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "dark-red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "info" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "debug" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unknown" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "text" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "`detected_level` is inferred by Loki at ingestion, so one query works across every log format." + }, + { + "id": 19, + "type": "timeseries", + "title": "Errors by container (whole namespace)", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (container) (count_over_time({namespace=\"classic-models\"} | detected_level =~ \"error|critical\" [$__auto]))", + "legendFormat": "{{container}}", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 80, + "showPoints": "never", + "barAlignment": 0, + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisLabel": "", + "axisPlacement": "auto" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Namespace-wide so the datastore's failures show up next to the application's." + }, + { + "id": 20, + "type": "logs", + "title": "Recent errors and warnings", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 47 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{namespace=\"classic-models\"} | detected_level =~ \"error|critical|warn\"", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "sortOrder": "Descending", + "dedupStrategy": "none", + "enableLogDetails": true + }, + "description": "Click a line to expand it. Where the line carries a trace_id, the Loki datasource's derived field opens the matching trace in Tempo." + }, + { + "id": 21, + "type": "row", + "title": "Container resources", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 58 + }, + "panels": [] + }, + { + "id": 22, + "type": "timeseries", + "title": "CPU per container", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 59 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod, container) (rate(container_cpu_usage_seconds_total{namespace=\"classic-models\",pod=~\"classic-models-api-.*\",container!=\"\"}[$__rate_interval]))", + "legendFormat": "{{pod}}/{{container}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0, + "decimals": 3 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Cores consumed." + }, + { + "id": 23, + "type": "timeseries", + "title": "Memory per container", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 59 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod, container) (container_memory_working_set_bytes{namespace=\"classic-models\",pod=~\"classic-models-api-.*\",container!=\"\"})", + "legendFormat": "{{pod}}/{{container}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Working set: what the kernel cannot reclaim under pressure. This is the figure the OOM killer compares against the limit." + }, + { + "id": 24, + "type": "timeseries", + "title": "Restarts over the window", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 59 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod) (changes(container_start_time_seconds{namespace=\"classic-models\",pod=~\"classic-models-api-.*\",container!=\"\"}[$__range]))", + "legendFormat": "{{pod}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Container start count over the displayed window. Stays at 0 in nominal operation." + } + ] +} diff --git a/k8s/grafana/dashboards/obs-keycloak.json b/k8s/grafana/dashboards/obs-keycloak.json new file mode 100644 index 0000000..81c6f4c --- /dev/null +++ b/k8s/grafana/dashboards/obs-keycloak.json @@ -0,0 +1,1637 @@ +{ + "uid": "obs-keycloak", + "title": "Keycloak — Overview", + "description": "HTTP traffic, JVM runtime, connection pool and logs for Keycloak. Latencies come from Tempo span metrics: the Micrometer timer exposes no buckets.", + "tags": [ + "observability", + "keycloak", + "application", + "metrics", + "logs" + ], + "editable": false, + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-6h", + "to": "now" + }, + "timezone": "browser", + "templating": { + "list": [] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Overview", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Requests / s", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(http_server_requests_seconds_count{namespace=\"keycloak\"}[$__rate_interval]))", + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "mappings": [], + "decimals": 2, + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "id": 3, + "type": "stat", + "title": "Error rate", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "(sum(rate(http_server_requests_seconds_count{namespace=\"keycloak\",outcome=~\"CLIENT_ERROR|SERVER_ERROR\"}[$__rate_interval])) or vector(0)) / clamp_min(sum(rate(http_server_requests_seconds_count{namespace=\"keycloak\"}[$__rate_interval])), 0.001)", + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "mappings": [], + "decimals": 2, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.01 + }, + { + "color": "red", + "value": 0.05 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto", + "justifyMode": "auto" + }, + "description": "Share of 4xx and 5xx responses. `or vector(0)` is required: with no errors at all the numerator is an empty vector, and empty/x stays empty -- the panel would read \"No data\" instead of 0%. clamp_min covers the zero-traffic case." + }, + { + "id": 4, + "type": "stat", + "title": "Latency p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"keycloak\",span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))", + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "mappings": [], + "decimals": 3, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "red", + "value": 2 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto", + "justifyMode": "auto" + }, + "description": "Source: span metrics generated by Tempo. Keycloak's Micrometer timer exposes neither buckets nor quantiles, so no percentile is computable from /metrics." + }, + { + "id": 5, + "type": "stat", + "title": "JVM heap used", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(jvm_memory_used_bytes{namespace=\"keycloak\",area=\"heap\"})", + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "mappings": [], + "color": { + "mode": "fixed", + "fixedColor": "purple" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "id": 6, + "type": "row", + "title": "HTTP traffic", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "panels": [] + }, + { + "id": 7, + "type": "timeseries", + "title": "Requests by outcome", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (outcome) (rate(http_server_requests_seconds_count{namespace=\"keycloak\"}[$__rate_interval]))", + "legendFormat": "{{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 70, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "linear", + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SUCCESS" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "REDIRECTION" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "CLIENT_ERROR" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SERVER_ERROR" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Stacked: total height is throughput, status colours isolate the failing share." + }, + { + "id": 8, + "type": "timeseries", + "title": "Latency (Tempo span metrics)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"keycloak\",span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"keycloak\",span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))", + "legendFormat": "p95" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"keycloak\",span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))", + "legendFormat": "p99" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "light-blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "dark-blue" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "One hue from light to dark: p50/p95/p99 are ordered magnitudes of the same measure, not distinct entities." + }, + { + "id": 9, + "type": "bargauge", + "title": "Busiest endpoints", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "topk(10, sum by (uri) (rate(http_server_requests_seconds_count{namespace=\"keycloak\"}[$__rate_interval])))", + "legendFormat": "{{uri}}", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0, + "mappings": [], + "color": { + "mode": "continuous-BlPu" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "text", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Mean latency per endpoint", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "topk(10, sum by (uri) (rate(http_server_requests_seconds_sum{namespace=\"keycloak\"}[$__rate_interval])) / clamp_min(sum by (uri) (rate(http_server_requests_seconds_count{namespace=\"keycloak\"}[$__rate_interval])), 0.001))", + "legendFormat": "{{uri}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "A mean, not a percentile: that is all a bucket-less timer allows. A mean hides the tail -- for the tail, see the p99 panel." + }, + { + "id": 11, + "type": "row", + "title": "JVM runtime", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "panels": [] + }, + { + "id": 12, + "type": "timeseries", + "title": "JVM memory by area", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 24 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (area) (jvm_memory_used_bytes{namespace=\"keycloak\"})", + "legendFormat": "{{area}}" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(jvm_memory_max_bytes{namespace=\"keycloak\",area=\"heap\"} > 0)", + "legendFormat": "max heap" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 13, + "type": "timeseries", + "title": "GC pauses", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 24 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(jvm_gc_pause_seconds_sum{namespace=\"keycloak\"}[$__rate_interval])) / clamp_min(sum(rate(jvm_gc_pause_seconds_count{namespace=\"keycloak\"}[$__rate_interval])), 0.001)", + "legendFormat": "mean duration" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0, + "decimals": 4 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": false, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Mean GC pause duration. Drift here usually precedes a rise in p99 latency." + }, + { + "id": 14, + "type": "timeseries", + "title": "JVM threads", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 24 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(jvm_threads_live_threads{namespace=\"keycloak\"})", + "legendFormat": "live" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(jvm_threads_daemon_threads{namespace=\"keycloak\"})", + "legendFormat": "daemon" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(jvm_threads_peak_threads{namespace=\"keycloak\"})", + "legendFormat": "peak" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 15, + "type": "row", + "title": "PostgreSQL connection pool (Agroal)", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "panels": [] + }, + { + "id": 16, + "type": "timeseries", + "title": "Pool connections", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 33 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (datasource) (agroal_active_count{namespace=\"keycloak\"})", + "legendFormat": "active" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (datasource) (agroal_available_count{namespace=\"keycloak\"})", + "legendFormat": "available" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (datasource) (agroal_max_used_count{namespace=\"keycloak\"})", + "legendFormat": "peak used" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 17, + "type": "timeseries", + "title": "Waiting for a connection", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 33 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(agroal_awaiting_count{namespace=\"keycloak\"})", + "legendFormat": "requests waiting" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(agroal_blocking_time_average_milliseconds{namespace=\"keycloak\"})", + "legendFormat": "mean blocking time (ms)" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "A non-zero waiting count means the pool is saturated: requests queue before they even reach PostgreSQL." + }, + { + "id": 18, + "type": "row", + "title": "Logs", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "panels": [] + }, + { + "id": 19, + "type": "timeseries", + "title": "Log volume by level", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 41 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (detected_level) (count_over_time({namespace=\"keycloak\",container=\"keycloak\"}[$__auto]))", + "legendFormat": "{{detected_level}}", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 80, + "showPoints": "never", + "barAlignment": 0, + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisLabel": "", + "axisPlacement": "auto" + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "critical" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "dark-red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "info" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "debug" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unknown" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "text" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "`detected_level` is inferred by Loki at ingestion, so one query works across every log format." + }, + { + "id": 20, + "type": "timeseries", + "title": "Errors by container (whole namespace)", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 41 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (container) (count_over_time({namespace=\"keycloak\"} | detected_level =~ \"error|critical\" [$__auto]))", + "legendFormat": "{{container}}", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 80, + "showPoints": "never", + "barAlignment": 0, + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisLabel": "", + "axisPlacement": "auto" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Namespace-wide so the datastore's failures show up next to the application's." + }, + { + "id": 21, + "type": "logs", + "title": "Recent errors and warnings", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 49 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{namespace=\"keycloak\"} | detected_level =~ \"error|critical|warn\"", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "sortOrder": "Descending", + "dedupStrategy": "none", + "enableLogDetails": true + }, + "description": "Click a line to expand it. Where the line carries a trace_id, the Loki datasource's derived field opens the matching trace in Tempo." + }, + { + "id": 22, + "type": "row", + "title": "Container resources", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 60 + }, + "panels": [] + }, + { + "id": 23, + "type": "timeseries", + "title": "CPU per container", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 61 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod, container) (rate(container_cpu_usage_seconds_total{namespace=\"keycloak\",pod=~\"keycloak-.*\",container!=\"\"}[$__rate_interval]))", + "legendFormat": "{{pod}}/{{container}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0, + "decimals": 3 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Cores consumed." + }, + { + "id": 24, + "type": "timeseries", + "title": "Memory per container", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 61 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod, container) (container_memory_working_set_bytes{namespace=\"keycloak\",pod=~\"keycloak-.*\",container!=\"\"})", + "legendFormat": "{{pod}}/{{container}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Working set: what the kernel cannot reclaim under pressure. This is the figure the OOM killer compares against the limit." + }, + { + "id": 25, + "type": "timeseries", + "title": "Restarts over the window", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 61 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod) (changes(container_start_time_seconds{namespace=\"keycloak\",pod=~\"keycloak-.*\",container!=\"\"}[$__range]))", + "legendFormat": "{{pod}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Container start count over the displayed window. Stays at 0 in nominal operation." + } + ] +} diff --git a/k8s/grafana/dashboards/obs-librechat.json b/k8s/grafana/dashboards/obs-librechat.json new file mode 100644 index 0000000..f67a4dc --- /dev/null +++ b/k8s/grafana/dashboards/obs-librechat.json @@ -0,0 +1,1772 @@ +{ + "uid": "obs-librechat", + "title": "LibreChat — Overview", + "description": "HTTP traffic, Node.js runtime, MongoDB access and logs for LibreChat. Built-in /metrics endpoint, scraped with a Bearer token.", + "tags": [ + "observability", + "librechat", + "application", + "metrics", + "logs" + ], + "editable": false, + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-6h", + "to": "now" + }, + "timezone": "browser", + "templating": { + "list": [] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Overview", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Requests / s", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(http_requests_total{job=\"librechat\"}[$__rate_interval]))", + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "mappings": [], + "decimals": 2, + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "id": 3, + "type": "stat", + "title": "Error rate", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "(sum(rate(http_requests_total{job=\"librechat\",status=~\"5..\"}[$__rate_interval])) or vector(0)) / clamp_min(sum(rate(http_requests_total{job=\"librechat\"}[$__rate_interval])), 0.001)", + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "mappings": [], + "decimals": 2, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.01 + }, + { + "color": "red", + "value": 0.05 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto", + "justifyMode": "auto" + }, + "description": "5xx only: 401/403 are expected on authenticated routes and would pollute the indicator. `or vector(0)` is required: with no errors at all the numerator is an empty vector, and empty/x stays empty -- the panel would read \"No data\" instead of 0%. clamp_min covers the zero-traffic case." + }, + { + "id": 4, + "type": "stat", + "title": "Latency p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket{job=\"librechat\"}[$__rate_interval])))", + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "mappings": [], + "decimals": 3, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "red", + "value": 2 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto", + "justifyMode": "auto" + }, + "description": "Native prom-client histogram: covers every request, not just the traced ones." + }, + { + "id": 5, + "type": "stat", + "title": "Generations in flight", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(generation_jobs_in_flight{job=\"librechat\"})", + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [], + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "purple" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto", + "justifyMode": "auto" + }, + "description": "LLM completions in flight. Each holds a handle and heap for its whole duration." + }, + { + "id": 6, + "type": "row", + "title": "HTTP traffic", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "panels": [] + }, + { + "id": 7, + "type": "timeseries", + "title": "Requests by status class", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (class) (label_replace(rate(http_requests_total{job=\"librechat\"}[$__rate_interval]), \"class\", \"$1xx\", \"status\", \"(.)..\"))", + "legendFormat": "{{class}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 70, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "linear", + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SUCCESS" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "REDIRECTION" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "CLIENT_ERROR" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SERVER_ERROR" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Request latency", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(http_request_duration_seconds_bucket{job=\"librechat\"}[$__rate_interval])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket{job=\"librechat\"}[$__rate_interval])))", + "legendFormat": "p95" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_request_duration_seconds_bucket{job=\"librechat\"}[$__rate_interval])))", + "legendFormat": "p99" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "light-blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "dark-blue" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 9, + "type": "bargauge", + "title": "Busiest routes", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "topk(10, sum by (path) (rate(http_requests_total{job=\"librechat\"}[$__rate_interval])))", + "legendFormat": "{{path}}", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0, + "mappings": [], + "color": { + "mode": "continuous-BlPu" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": true, + "valueMode": "text", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Requests in flight", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(http_requests_in_flight{job=\"librechat\"})", + "legendFormat": "in flight" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": false, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Instantaneous concurrency. On a single-threaded runtime, a value that never comes back down points at a blocked event loop." + }, + { + "id": 11, + "type": "row", + "title": "Node.js runtime", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "panels": [] + }, + { + "id": 12, + "type": "timeseries", + "title": "V8 heap", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 24 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(nodejs_heap_size_used_bytes{job=\"librechat\"})", + "legendFormat": "used" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(nodejs_heap_size_total_bytes{job=\"librechat\"})", + "legendFormat": "allocated" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(process_resident_memory_bytes{job=\"librechat\"})", + "legendFormat": "process RSS" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Old-space is capped at 1200 MB by NODE_OPTIONS. Used heap flattening just below that precedes a V8 crash." + }, + { + "id": 13, + "type": "timeseries", + "title": "Event loop lag", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 24 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(nodejs_eventloop_lag_p50_seconds{job=\"librechat\"})", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(nodejs_eventloop_lag_p99_seconds{job=\"librechat\"})", + "legendFormat": "p99" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0, + "decimals": 4 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "light-blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "dark-blue" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Event loop lag: the single best saturation signal for a Node process. Past roughly 100 ms, every request suffers." + }, + { + "id": 14, + "type": "timeseries", + "title": "Time spent in GC", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 24 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (kind) (rate(nodejs_gc_duration_seconds_sum{job=\"librechat\"}[$__rate_interval]))", + "legendFormat": "{{kind}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "GC seconds per elapsed second: directly the fraction of CPU time stolen from request handling." + }, + { + "id": 15, + "type": "row", + "title": "MongoDB (Mongoose)", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "panels": [] + }, + { + "id": 16, + "type": "timeseries", + "title": "Mongo queries by operation", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 33 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "topk(10, sum by (model, operation) (rate(mongoose_queries_total{job=\"librechat\"}[$__rate_interval])))", + "legendFormat": "{{model}}.{{operation}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 17, + "type": "timeseries", + "title": "Mongo query latency", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 33 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(mongoose_query_duration_seconds_bucket{job=\"librechat\"}[$__rate_interval])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(mongoose_query_duration_seconds_bucket{job=\"librechat\"}[$__rate_interval])))", + "legendFormat": "p95" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(mongoose_query_duration_seconds_bucket{job=\"librechat\"}[$__rate_interval])))", + "legendFormat": "p99" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "light-blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "dark-blue" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 18, + "type": "timeseries", + "title": "Failed Mongo queries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 33 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (model) (rate(mongoose_queries_total{job=\"librechat\",status!=\"success\"}[$__rate_interval]))", + "legendFormat": "{{model}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Empty in nominal operation." + }, + { + "id": 19, + "type": "row", + "title": "Logs", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "panels": [] + }, + { + "id": 20, + "type": "timeseries", + "title": "Log volume by level", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 41 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (detected_level) (count_over_time({namespace=\"librechat\",container=\"librechat\"}[$__auto]))", + "legendFormat": "{{detected_level}}", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 80, + "showPoints": "never", + "barAlignment": 0, + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisLabel": "", + "axisPlacement": "auto" + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "critical" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "dark-red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "info" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "debug" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unknown" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "text" + } + } + ] + } + ] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "`detected_level` is inferred by Loki at ingestion, so one query works across every log format." + }, + { + "id": 21, + "type": "timeseries", + "title": "Errors by container (whole namespace)", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 41 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (container) (count_over_time({namespace=\"librechat\"} | detected_level =~ \"error|critical\" [$__auto]))", + "legendFormat": "{{container}}", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 80, + "showPoints": "never", + "barAlignment": 0, + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisLabel": "", + "axisPlacement": "auto" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Namespace-wide so the datastore's failures show up next to the application's." + }, + { + "id": 22, + "type": "logs", + "title": "Recent errors and warnings", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 49 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{namespace=\"librechat\"} | detected_level =~ \"error|critical|warn\"", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "sortOrder": "Descending", + "dedupStrategy": "none", + "enableLogDetails": true + }, + "description": "Click a line to expand it. Where the line carries a trace_id, the Loki datasource's derived field opens the matching trace in Tempo." + }, + { + "id": 23, + "type": "row", + "title": "Container resources", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 60 + }, + "panels": [] + }, + { + "id": 24, + "type": "timeseries", + "title": "CPU per container", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 61 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod, container) (rate(container_cpu_usage_seconds_total{namespace=\"librechat\",pod=~\"librechat-.*\",container!=\"\"}[$__rate_interval]))", + "legendFormat": "{{pod}}/{{container}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0, + "decimals": 3 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Cores consumed." + }, + { + "id": 25, + "type": "timeseries", + "title": "Memory per container", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 61 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod, container) (container_memory_working_set_bytes{namespace=\"librechat\",pod=~\"librechat-.*\",container!=\"\"})", + "legendFormat": "{{pod}}/{{container}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Working set: what the kernel cannot reclaim under pressure. This is the figure the OOM killer compares against the limit." + }, + { + "id": 26, + "type": "timeseries", + "title": "Restarts over the window", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 61 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod) (changes(container_start_time_seconds{namespace=\"librechat\",pod=~\"librechat-.*\",container!=\"\"}[$__range]))", + "legendFormat": "{{pod}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "never", + "pointSize": 8, + "gradientMode": "none", + "spanNulls": false, + "axisLabel": "", + "axisPlacement": "auto", + "lineInterpolation": "smooth" + }, + "mappings": [], + "color": { + "mode": "palette-classic" + }, + "min": 0, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Container start count over the displayed window. Stays at 0 in nominal operation." + } + ] +} diff --git a/k8s/grafana/dashboards/obs-stack-health.json b/k8s/grafana/dashboards/obs-stack-health.json new file mode 100644 index 0000000..d514823 --- /dev/null +++ b/k8s/grafana/dashboards/obs-stack-health.json @@ -0,0 +1,437 @@ +{ + "uid": "obs-stack-health", + "title": "Observability — Stack Health", + "description": "Health of the Prometheus / Loki / Tempo / Alloy stack. Reference dashboard: stable UID, declaratively provisioned.", + "tags": [ + "observability", + "stack", + "provisioned" + ], + "editable": false, + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "templating": { + "list": [ + { + "name": "namespace", + "label": "Namespace", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": "label_values(kube_pod_info, namespace)", + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { + "text": "observability", + "value": "observability" + } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Collection", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Prometheus targets UP", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(up)", + "legendFormat": "up" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + } + }, + { + "id": 3, + "type": "stat", + "title": "Prometheus targets DOWN", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(up == bool 0)", + "legendFormat": "down" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + } + }, + { + "id": 4, + "type": "stat", + "title": "Log lines ingested (/s)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(loki_distributor_lines_received_total[5m]))", + "legendFormat": "lines/s" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 1, + "color": { + "mode": "fixed", + "fixedColor": "blue" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + } + }, + { + "id": 5, + "type": "stat", + "title": "Spans ingested (/s)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(tempo_distributor_spans_received_total[5m]))", + "legendFormat": "spans/s" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 1, + "color": { + "mode": "fixed", + "fixedColor": "purple" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + } + }, + { + "id": 6, + "type": "row", + "title": "Stack resources", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 7, + "type": "timeseries", + "title": "CPU per component (observability namespace)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"observability\", container!=\"\"}[$__rate_interval]))", + "legendFormat": "{{pod}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Memory per component (observability namespace)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"observability\", container!=\"\"})", + "legendFormat": "{{pod}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 9, + "type": "row", + "title": "Logs", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 10, + "type": "logs", + "title": "Recent stack errors", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 16 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{namespace=\"observability\"} |~ \"(?i)(error|level=error|fatal)\"", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "wrapLogMessage": true, + "sortOrder": "Descending", + "dedupStrategy": "none", + "enableLogDetails": true + } + } + ] +} diff --git a/k8s/grafana/deployment.yaml b/k8s/grafana/deployment.yaml new file mode 100644 index 0000000..ea365cf --- /dev/null +++ b/k8s/grafana/deployment.yaml @@ -0,0 +1,112 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: observability + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/component: ui + app.kubernetes.io/part-of: observability +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: grafana + template: + metadata: + labels: + app: grafana + app.kubernetes.io/name: grafana + app.kubernetes.io/component: ui + app.kubernetes.io/part-of: observability + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "3000" + prometheus.io/path: "/metrics" + spec: + securityContext: + runAsUser: 472 + runAsGroup: 472 + fsGroup: 472 + containers: + - name: grafana + image: grafana/grafana:13.2.0 + ports: + - name: http + containerPort: 3000 + env: + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: observability-secrets + key: GF_SECURITY_ADMIN_USER + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: observability-secrets + key: GF_SECURITY_ADMIN_PASSWORD + - name: GF_SERVER_ROOT_URL + value: https://grafana.sttlab.local + - name: GF_PATHS_PROVISIONING + value: /etc/grafana/provisioning + # Everything is driven by the provisioning files, so creating + # dashboards or datasources from the UI is disabled. + - name: GF_USERS_ALLOW_SIGN_UP + value: "false" + - name: GF_ANALYTICS_REPORTING_ENABLED + value: "false" + - name: GF_ANALYTICS_CHECK_FOR_UPDATES + value: "false" + - name: GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES + value: "false" + - name: GF_NEWS_NEWS_FEED_ENABLED + value: "false" + - name: GF_LOG_LEVEL + value: info + - name: GF_FEATURE_TOGGLES_ENABLE + value: traceqlEditor,traceQLStreaming,correlations,lokiLogsDataplane + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + readinessProbe: + httpGet: + path: /api/health + port: http + initialDelaySeconds: 15 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /api/health + port: http + initialDelaySeconds: 60 + periodSeconds: 30 + volumeMounts: + - name: data + mountPath: /var/lib/grafana + - name: provisioning-datasources + mountPath: /etc/grafana/provisioning/datasources + - name: provisioning-dashboards + mountPath: /etc/grafana/provisioning/dashboards + - name: dashboards + mountPath: /var/lib/grafana/dashboards + volumes: + - name: data + persistentVolumeClaim: + claimName: grafana-data + - name: provisioning-datasources + configMap: + name: grafana-datasources + - name: provisioning-dashboards + configMap: + name: grafana-dashboard-providers + - name: dashboards + # Stable-named ConfigMap: updated in place by `make dashboards` and + # re-read by Grafana without a restart (~70s: kubelet sync + 10s poll). + configMap: + name: grafana-dashboards diff --git a/k8s/grafana/kustomization.yaml b/k8s/grafana/kustomization.yaml new file mode 100644 index 0000000..af15849 --- /dev/null +++ b/k8s/grafana/kustomization.yaml @@ -0,0 +1,27 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: observability + +resources: + - pvc.yaml + - deployment.yaml + - service.yaml +# Dashboards live in ./dashboards and are referenced by the root kustomization: +# `k8s/grafana` is the runtime alone, `k8s/grafana/dashboards` the content alone. +# Both are deployable independently. + +configMapGenerator: + # Name-suffix hash KEPT here: datasource and provider provisioning is only + # read at startup, so the hash deliberately forces a pod restart whenever + # these files change. + - name: grafana-datasources + files: + - datasources.yaml=provisioning/datasources.yaml + - name: grafana-dashboard-providers + files: + - dashboard-providers.yaml=provisioning/dashboard-providers.yaml + +generatorOptions: + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: observability diff --git a/k8s/grafana/provisioning/dashboard-providers.yaml b/k8s/grafana/provisioning/dashboard-providers.yaml new file mode 100644 index 0000000..77f1c34 --- /dev/null +++ b/k8s/grafana/provisioning/dashboard-providers.yaml @@ -0,0 +1,19 @@ +# File-based dashboard provider. +# The JSON files are mounted from the `grafana-dashboards` ConfigMap (generated +# without a name-suffix hash): `make dashboards` updates the ConfigMap in place +# and Grafana reloads WITHOUT restarting the pod, in about 70s +# (kubelet volume sync ~60s + updateIntervalSeconds below). +apiVersion: 1 + +providers: + - name: infra-observability + orgId: 1 + folder: Infra Observability + folderUid: infra-observability + type: file + disableDeletion: false + allowUiUpdates: false + updateIntervalSeconds: 10 + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/k8s/grafana/provisioning/datasources.yaml b/k8s/grafana/provisioning/datasources.yaml new file mode 100644 index 0000000..2337d3a --- /dev/null +++ b/k8s/grafana/provisioning/datasources.yaml @@ -0,0 +1,82 @@ +# Declarative datasource provisioning. +# The `uid` values are STABLE and are what dashboard JSON files reference: +# "datasource": { "type": "prometheus", "uid": "prometheus" } +apiVersion: 1 + +deleteDatasources: + - name: Prometheus + orgId: 1 + - name: Loki + orgId: 1 + - name: Tempo + orgId: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 + isDefault: true + editable: false + jsonData: + httpMethod: POST + timeInterval: 30s + manageAlerts: false + exemplarTraceIdDestinations: + - name: trace_id + datasourceUid: tempo + + - name: Loki + uid: loki + type: loki + access: proxy + orgId: 1 + url: http://loki:3100 + editable: false + jsonData: + timeout: 60 + maxLines: 1000 + # Logs -> traces: a trace_id in the log line opens Tempo. + derivedFields: + - name: TraceID + matcherType: regex + matcherRegex: '"?trace[_-]?id"?[:=]"?([a-f0-9]{16,32})' + url: '$${__value.raw}' + datasourceUid: tempo + urlDisplayLabel: View trace + + - name: Tempo + uid: tempo + type: tempo + access: proxy + orgId: 1 + url: http://tempo:3200 + editable: false + jsonData: + httpMethod: GET + streamingEnabled: + search: true + metrics: true + # Service graph fed by the metrics_generator -> Prometheus. + serviceMap: + datasourceUid: prometheus + nodeGraph: + enabled: true + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: -5m + spanEndTimeShift: 5m + filterByTraceID: true + filterBySpanID: false + tags: + - key: service.name + value: app + tracesToMetrics: + datasourceUid: prometheus + spanStartTimeShift: -5m + spanEndTimeShift: 5m + tags: + - key: service.name + value: service diff --git a/k8s/grafana/pvc.yaml b/k8s/grafana/pvc.yaml new file mode 100644 index 0000000..c462b70 --- /dev/null +++ b/k8s/grafana/pvc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: grafana-data + namespace: observability + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/component: storage + app.kubernetes.io/part-of: observability +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi diff --git a/k8s/grafana/service.yaml b/k8s/grafana/service.yaml new file mode 100644 index 0000000..78f24ce --- /dev/null +++ b/k8s/grafana/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: grafana + namespace: observability + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/component: ui + app.kubernetes.io/part-of: observability +spec: + type: ClusterIP + selector: + app: grafana + ports: + - name: http + port: 3000 + targetPort: http diff --git a/k8s/ingress/certificate.yaml b/k8s/ingress/certificate.yaml new file mode 100644 index 0000000..f5f3220 --- /dev/null +++ b/k8s/ingress/certificate.yaml @@ -0,0 +1,21 @@ +# Certificates issued by the `sttlab-local-ca` cert-manager ClusterIssuer +# (already present on the cluster). No private key is ever versioned. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: observability-tls + namespace: observability + labels: + app.kubernetes.io/part-of: observability +spec: + secretName: observability-tls + issuerRef: + name: sttlab-local-ca + kind: ClusterIssuer + commonName: grafana.sttlab.local + dnsNames: + - grafana.sttlab.local + - prometheus.sttlab.local + - alloy.sttlab.local + - loki.sttlab.local + - tempo.sttlab.local diff --git a/k8s/ingress/ingress.yaml b/k8s/ingress/ingress.yaml new file mode 100644 index 0000000..c28bc66 --- /dev/null +++ b/k8s/ingress/ingress.yaml @@ -0,0 +1,88 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: grafana + namespace: observability + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: observability + annotations: + nginx.ingress.kubernetes.io/proxy-body-size: "20m" + nginx.ingress.kubernetes.io/proxy-read-timeout: "600" + nginx.ingress.kubernetes.io/proxy-send-timeout: "600" + nginx.ingress.kubernetes.io/proxy-http-version: "1.1" + # Required for TraceQL streaming and Loki live tail. + nginx.ingress.kubernetes.io/proxy-buffering: "off" +spec: + ingressClassName: nginx + tls: + - hosts: + - grafana.sttlab.local + secretName: observability-tls + rules: + - host: grafana.sttlab.local + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: grafana + port: + number: 3000 +--- +# Exposed for diagnostics (targets, TSDB status, expression browser). +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: prometheus + namespace: observability + labels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/part-of: observability + annotations: + nginx.ingress.kubernetes.io/proxy-read-timeout: "600" +spec: + ingressClassName: nginx + tls: + - hosts: + - prometheus.sttlab.local + secretName: observability-tls + rules: + - host: prometheus.sttlab.local + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: prometheus + port: + number: 9090 +--- +# Alloy UI: state of the log collection pipeline components. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: alloy + namespace: observability + labels: + app.kubernetes.io/name: alloy + app.kubernetes.io/part-of: observability +spec: + ingressClassName: nginx + tls: + - hosts: + - alloy.sttlab.local + secretName: observability-tls + rules: + - host: alloy.sttlab.local + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: alloy + port: + number: 12345 diff --git a/k8s/ingress/kustomization.yaml b/k8s/ingress/kustomization.yaml new file mode 100644 index 0000000..129add5 --- /dev/null +++ b/k8s/ingress/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: observability + +resources: + - certificate.yaml + - ingress.yaml diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml new file mode 100644 index 0000000..72a2903 --- /dev/null +++ b/k8s/kustomization.yaml @@ -0,0 +1,20 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: observability + +resources: + - namespace.yaml + - secret.yaml + - prometheus + - loki + - tempo + - alloy + - grafana + - grafana/dashboards + - ingress + +labels: + - includeSelectors: false + pairs: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/part-of: observability diff --git a/k8s/loki/deployment.yaml b/k8s/loki/deployment.yaml new file mode 100644 index 0000000..e61533a --- /dev/null +++ b/k8s/loki/deployment.yaml @@ -0,0 +1,73 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: loki + namespace: observability + labels: + app.kubernetes.io/name: loki + app.kubernetes.io/component: logs + app.kubernetes.io/part-of: observability +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: loki + template: + metadata: + labels: + app: loki + app.kubernetes.io/name: loki + app.kubernetes.io/component: logs + app.kubernetes.io/part-of: observability + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "3100" + spec: + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + containers: + - name: loki + image: grafana/loki:3.7.6 + args: + - -config.file=/etc/loki/loki.yaml + - -target=all + ports: + - name: http + containerPort: 3100 + - name: grpc + containerPort: 9096 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "2" + memory: 1Gi + readinessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 60 + periodSeconds: 30 + volumeMounts: + - name: config + mountPath: /etc/loki + - name: data + mountPath: /loki + volumes: + - name: config + configMap: + name: loki-config + - name: data + persistentVolumeClaim: + claimName: loki-data diff --git a/k8s/loki/kustomization.yaml b/k8s/loki/kustomization.yaml new file mode 100644 index 0000000..13d0678 --- /dev/null +++ b/k8s/loki/kustomization.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: observability + +resources: + - pvc.yaml + - deployment.yaml + - service.yaml + +configMapGenerator: + - name: loki-config + files: + - loki.yaml + +generatorOptions: + labels: + app.kubernetes.io/name: loki + app.kubernetes.io/part-of: observability diff --git a/k8s/loki/loki.yaml b/k8s/loki/loki.yaml new file mode 100644 index 0000000..81ae290 --- /dev/null +++ b/k8s/loki/loki.yaml @@ -0,0 +1,85 @@ +# Loki 3.x in monolithic mode (-target=all), filesystem storage on a PVC. +auth_enabled: false + +server: + http_listen_address: 0.0.0.0 + http_listen_port: 3100 + grpc_listen_address: 0.0.0.0 + grpc_listen_port: 9096 + log_level: info + grpc_server_max_concurrent_streams: 1000 + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: "2024-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +storage_config: + tsdb_shipper: + active_index_directory: /loki/tsdb-index + cache_location: /loki/tsdb-cache + filesystem: + directory: /loki/chunks + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + compaction_interval: 10m + retention_delete_delay: 2h + +limits_config: + retention_period: 168h + max_query_series: 5000 + ingestion_rate_mb: 16 + ingestion_burst_size_mb: 32 + reject_old_samples: true + reject_old_samples_max_age: 168h + volume_enabled: true + metric_aggregation_enabled: true + allow_structured_metadata: true + +pattern_ingester: + enabled: true + metric_aggregation: + loki_address: localhost:3100 + +query_range: + align_queries_with_step: true + cache_results: true + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +frontend: + encoding: protobuf + +ruler: + storage: + type: local + local: + directory: /loki/rules + rule_path: /loki/rules-tmp + enable_api: true + +analytics: + reporting_enabled: false diff --git a/k8s/loki/pvc.yaml b/k8s/loki/pvc.yaml new file mode 100644 index 0000000..efd3580 --- /dev/null +++ b/k8s/loki/pvc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: loki-data + namespace: observability + labels: + app.kubernetes.io/name: loki + app.kubernetes.io/component: storage + app.kubernetes.io/part-of: observability +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/k8s/loki/service.yaml b/k8s/loki/service.yaml new file mode 100644 index 0000000..c01734f --- /dev/null +++ b/k8s/loki/service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: loki + namespace: observability + labels: + app.kubernetes.io/name: loki + app.kubernetes.io/component: logs + app.kubernetes.io/part-of: observability +spec: + type: ClusterIP + selector: + app: loki + ports: + - name: http + port: 3100 + targetPort: http + - name: grpc + port: 9096 + targetPort: grpc diff --git a/k8s/namespace.yaml b/k8s/namespace.yaml new file mode 100644 index 0000000..73716d7 --- /dev/null +++ b/k8s/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: observability + labels: + app.kubernetes.io/part-of: observability diff --git a/k8s/prometheus/deployment.yaml b/k8s/prometheus/deployment.yaml new file mode 100644 index 0000000..31b2632 --- /dev/null +++ b/k8s/prometheus/deployment.yaml @@ -0,0 +1,86 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + namespace: observability + labels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: observability +spec: + replicas: 1 + strategy: + # ReadWriteOnce PVC: the old pod must release the volume first. + type: Recreate + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + app.kubernetes.io/name: prometheus + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: observability + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + serviceAccountName: prometheus + securityContext: + runAsUser: 65534 + runAsGroup: 65534 + fsGroup: 65534 + containers: + - name: prometheus + image: prom/prometheus:v3.14.0 + args: + - --config.file=/etc/prometheus/prometheus.yaml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=15d + - --storage.tsdb.retention.size=8GB + - --web.enable-lifecycle + # Receives remote_write from Tempo's metrics_generator (span metrics + service graphs). + - --web.enable-remote-write-receiver + - --web.external-url=https://prometheus.sttlab.local + - --web.route-prefix=/ + ports: + - name: http + containerPort: 9090 + resources: + requests: + cpu: 100m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi + readinessProbe: + httpGet: + path: /-/ready + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /-/healthy + port: http + initialDelaySeconds: 30 + periodSeconds: 30 + volumeMounts: + - name: config + mountPath: /etc/prometheus + - name: data + mountPath: /prometheus + - name: librechat-metrics + mountPath: /etc/prometheus/secrets/librechat + readOnly: true + volumes: + - name: config + configMap: + name: prometheus-config + - name: librechat-metrics + secret: + secretName: librechat-metrics + - name: data + persistentVolumeClaim: + claimName: prometheus-data diff --git a/k8s/prometheus/kustomization.yaml b/k8s/prometheus/kustomization.yaml new file mode 100644 index 0000000..59d3f80 --- /dev/null +++ b/k8s/prometheus/kustomization.yaml @@ -0,0 +1,20 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: observability + +resources: + - secret.yaml + - rbac.yaml + - pvc.yaml + - deployment.yaml + - service.yaml + +configMapGenerator: + - name: prometheus-config + files: + - prometheus.yaml + +generatorOptions: + labels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/part-of: observability diff --git a/k8s/prometheus/prometheus.yaml b/k8s/prometheus/prometheus.yaml new file mode 100644 index 0000000..a3637b4 --- /dev/null +++ b/k8s/prometheus/prometheus.yaml @@ -0,0 +1,177 @@ +global: + scrape_interval: 30s + evaluation_interval: 30s + external_labels: + cluster: orbstack + source: prometheus + +scrape_configs: + # --- Control plane ------------------------------------------------------- + - job_name: kubernetes-apiservers + kubernetes_sd_configs: + - role: endpoints + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + authorization: + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: + [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: default;kubernetes;https + + # --- Kubelet ------------------------------------------------------------- + - job_name: kubernetes-nodes + kubernetes_sd_configs: + - role: node + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + authorization: + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/$1/proxy/metrics + + # --- Container resource metrics ----------------------------------------- + # /metrics/resource and NOT /metrics/cadvisor: on this cluster (OrbStack) the + # kubelet's cadvisor endpoint only exposes `machine_*` series and no + # `container_*` at all. /metrics/resource -- the source metrics-server reads -- + # provides container_cpu_usage_seconds_total and + # container_memory_working_set_bytes with namespace/pod/container labels. + - job_name: kubernetes-kubelet-resource + kubernetes_sd_configs: + - role: node + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + authorization: + credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/$1/proxy/metrics/resource + - source_labels: [__meta_kubernetes_node_name] + target_label: node + + # --- Services annotated with prometheus.io/scrape ----------------------- + - job_name: kubernetes-service-endpoints + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] + action: keep + regex: "true" + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_service_name] + target_label: service + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod + - source_labels: [__meta_kubernetes_pod_node_name] + target_label: node + + # --- Pods annotated with prometheus.io/scrape --------------------------- + - job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: "true" + # A pod yields one target per container: without this filter the init + # containers duplicate the real target. + - source_labels: [__meta_kubernetes_pod_container_init] + action: drop + regex: "true" + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod + - source_labels: [__meta_kubernetes_pod_node_name] + target_label: node + - source_labels: [__meta_kubernetes_pod_container_name] + target_label: container + - source_labels: [__meta_kubernetes_pod_phase] + action: drop + regex: (Succeeded|Failed|Pending) + + # --- LibreChat: /metrics behind a Bearer token -------------------------- + # Annotation-based discovery cannot carry credentials, so LibreChat gets its + # own job. The token comes from the `librechat-metrics` secret mounted into + # the Prometheus pod and must match METRICS_SECRET on the LibreChat side. + - job_name: librechat + kubernetes_sd_configs: + - role: pod + namespaces: + names: [librechat] + authorization: + type: Bearer + credentials_file: /etc/prometheus/secrets/librechat/token + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape_with_auth] + action: keep + regex: "true" + # A pod yields one target per container: without this filter the init + # containers duplicate the real target. + - source_labels: [__meta_kubernetes_pod_container_init] + action: drop + regex: "true" + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod + - source_labels: [__meta_kubernetes_pod_container_name] + target_label: container + - target_label: service + replacement: librechat diff --git a/k8s/prometheus/pvc.yaml b/k8s/prometheus/pvc.yaml new file mode 100644 index 0000000..c6a3d5e --- /dev/null +++ b/k8s/prometheus/pvc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: prometheus-data + namespace: observability + labels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/component: storage + app.kubernetes.io/part-of: observability +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/k8s/prometheus/rbac.yaml b/k8s/prometheus/rbac.yaml new file mode 100644 index 0000000..fbaff77 --- /dev/null +++ b/k8s/prometheus/rbac.yaml @@ -0,0 +1,51 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus + namespace: observability + labels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/part-of: observability +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: observability-prometheus + labels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/part-of: observability +rules: + - apiGroups: [""] + resources: + - nodes + - nodes/metrics + - nodes/proxy + - services + - endpoints + - pods + - namespaces + verbs: ["get", "list", "watch"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingresses"] + verbs: ["get", "list", "watch"] + - nonResourceURLs: ["/metrics", "/metrics/cadvisor"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: observability-prometheus + labels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/part-of: observability +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: observability-prometheus +subjects: + - kind: ServiceAccount + name: prometheus + namespace: observability diff --git a/k8s/prometheus/secret.example.yaml b/k8s/prometheus/secret.example.yaml new file mode 100644 index 0000000..56b23da --- /dev/null +++ b/k8s/prometheus/secret.example.yaml @@ -0,0 +1,11 @@ +# Template. The token must be IDENTICAL to the METRICS_SECRET key of the +# `librechat-secrets` secret in the librechat namespace (infra-librechat repo): +# it is the Bearer token Prometheus uses to scrape LibreChat's /metrics. +apiVersion: v1 +kind: Secret +metadata: + name: librechat-metrics + namespace: observability +type: Opaque +stringData: + token: diff --git a/k8s/prometheus/service.yaml b/k8s/prometheus/service.yaml new file mode 100644 index 0000000..6b364bc --- /dev/null +++ b/k8s/prometheus/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: prometheus + namespace: observability + labels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: observability +spec: + type: ClusterIP + selector: + app: prometheus + ports: + - name: http + port: 9090 + targetPort: http diff --git a/k8s/secret.example.yaml b/k8s/secret.example.yaml new file mode 100644 index 0000000..5cb375e --- /dev/null +++ b/k8s/secret.example.yaml @@ -0,0 +1,11 @@ +# Template: copy to secret.yaml (gitignored) and fill in the values. +# make secret +apiVersion: v1 +kind: Secret +metadata: + name: observability-secrets + namespace: observability +type: Opaque +stringData: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: diff --git a/k8s/tempo/deployment.yaml b/k8s/tempo/deployment.yaml new file mode 100644 index 0000000..146e90d --- /dev/null +++ b/k8s/tempo/deployment.yaml @@ -0,0 +1,79 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tempo + namespace: observability + labels: + app.kubernetes.io/name: tempo + app.kubernetes.io/component: traces + app.kubernetes.io/part-of: observability +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: tempo + template: + metadata: + labels: + app: tempo + app.kubernetes.io/name: tempo + app.kubernetes.io/component: traces + app.kubernetes.io/part-of: observability + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "3200" + spec: + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + containers: + - name: tempo + image: grafana/tempo:3.0.3 + args: + - -config.file=/etc/tempo/tempo.yaml + - -target=all + ports: + - name: http + containerPort: 3200 + - name: grpc + containerPort: 9095 + - name: otlp-grpc + containerPort: 4317 + - name: otlp-http + containerPort: 4318 + - name: zipkin + containerPort: 9411 + resources: + requests: + cpu: 100m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi + readinessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 60 + periodSeconds: 30 + volumeMounts: + - name: config + mountPath: /etc/tempo + - name: data + mountPath: /var/tempo + volumes: + - name: config + configMap: + name: tempo-config + - name: data + persistentVolumeClaim: + claimName: tempo-data diff --git a/k8s/tempo/kustomization.yaml b/k8s/tempo/kustomization.yaml new file mode 100644 index 0000000..0e92093 --- /dev/null +++ b/k8s/tempo/kustomization.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: observability + +resources: + - pvc.yaml + - deployment.yaml + - service.yaml + +configMapGenerator: + - name: tempo-config + files: + - tempo.yaml + +generatorOptions: + labels: + app.kubernetes.io/name: tempo + app.kubernetes.io/part-of: observability diff --git a/k8s/tempo/pvc.yaml b/k8s/tempo/pvc.yaml new file mode 100644 index 0000000..6d47f3e --- /dev/null +++ b/k8s/tempo/pvc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: tempo-data + namespace: observability + labels: + app.kubernetes.io/name: tempo + app.kubernetes.io/component: storage + app.kubernetes.io/part-of: observability +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/k8s/tempo/service.yaml b/k8s/tempo/service.yaml new file mode 100644 index 0000000..cec4953 --- /dev/null +++ b/k8s/tempo/service.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Service +metadata: + name: tempo + namespace: observability + labels: + app.kubernetes.io/name: tempo + app.kubernetes.io/component: traces + app.kubernetes.io/part-of: observability +spec: + type: ClusterIP + selector: + app: tempo + ports: + - name: http + port: 3200 + targetPort: http + - name: grpc + port: 9095 + targetPort: grpc + - name: otlp-grpc + port: 4317 + targetPort: otlp-grpc + - name: otlp-http + port: 4318 + targetPort: otlp-http + - name: zipkin + port: 9411 + targetPort: zipkin diff --git a/k8s/tempo/tempo.yaml b/k8s/tempo/tempo.yaml new file mode 100644 index 0000000..a3165ee --- /dev/null +++ b/k8s/tempo/tempo.yaml @@ -0,0 +1,64 @@ +# Tempo 3.x in monolithic mode (-target=all), local storage on a PVC. +stream_over_http_enabled: true +multitenancy_enabled: false + +server: + http_listen_address: 0.0.0.0 + http_listen_port: 3200 + grpc_listen_address: 0.0.0.0 + grpc_listen_port: 9095 + log_level: info + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + zipkin: + endpoint: 0.0.0.0:9411 + +# Generates span metrics and service graphs, pushed to Prometheus through its +# remote-write receiver (--web.enable-remote-write-receiver). +metrics_generator: + registry: + external_labels: + source: tempo + cluster: orbstack + storage: + path: /var/tempo/generator/wal + remote_write: + - url: http://prometheus:9090/api/v1/write + send_exemplars: true + +query_frontend: + search: + duration_slo: 5s + throughput_bytes_slo: 1073741824 + trace_by_id: + duration_slo: 5s + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks + +overrides: + defaults: + metrics_generator: + processors: + - service-graphs + - span-metrics + generate_native_histograms: both + # Tempo 3.x: retention is no longer a top-level `compactor:` block; it is + # driven from here (backend_scheduler/backend_worker otherwise). + compaction: + block_retention: 168h + +usage_report: + reporting_enabled: false