Declarative Grafana / Prometheus / Loki / Tempo observability stack
Deploys the whole stack into the observability namespace of the local cluster with Kustomize alone — no helm install, no dashboard created by hand in the UI. The Makefile carries every command: deploy, dashboards, diagnostics, PVC flush, teardown. Dashboards are files, never UI objects. Each carries a stable uid, which is what keeps URLs, cross-links and alerts alive across redeployments, and references datasources by stable uid rather than by name. The dashboard ConfigMap is generated without a name-suffix hash so applying it updates it in place and Grafana reloads without a restart, while the provisioning ConfigMaps keep their hash precisely so that a change there does force one. Prometheus scrapes the kubelet /metrics/resource endpoint rather than /metrics/cadvisor: on this cluster cadvisor exposes machine_* series only and no container_* at all, which would leave every CPU and memory panel empty. Alloy ships every pod's logs to Loki through the Kubernetes API, so no /var/log mount is needed. Tempo's metrics_generator feeds span metrics and the service graph back into Prometheus, and the datasources are wired so a log line's trace_id opens the matching trace. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Stéphane Tailland
@@ -0,0 +1,5 @@
|
||||
.DS_Store
|
||||
*.tmp
|
||||
# Real secrets: only secret.example.yaml is versioned
|
||||
k8s/secret.yaml
|
||||
k8s/prometheus/secret.yaml
|
||||
@@ -0,0 +1,356 @@
|
||||
# =============================================================================
|
||||
# infra-observability -- Grafana / Prometheus / Loki / Tempo stack (+ Alloy)
|
||||
# Fully declarative: kustomize + kubectl. No imperative `helm install`.
|
||||
#
|
||||
# make -> help
|
||||
# make deploy -> deploy everything
|
||||
# make diag -> full diagnostics
|
||||
# =============================================================================
|
||||
|
||||
SHELL := /bin/bash
|
||||
.DEFAULT_GOAL := help
|
||||
# Every target prints its own output, so commands are not echoed.
|
||||
.SILENT:
|
||||
|
||||
NS ?= observability
|
||||
CTX ?= orbstack
|
||||
KUBECTL := kubectl --context=$(CTX)
|
||||
K8S := k8s
|
||||
DASHBOARDS := $(K8S)/grafana/dashboards
|
||||
INGRESS_IP ?= 192.168.139.101
|
||||
HOSTS := grafana.sttlab.local prometheus.sttlab.local alloy.sttlab.local
|
||||
|
||||
# Components backed by a PVC (this is also the flush order)
|
||||
STATEFUL := prometheus loki tempo grafana
|
||||
# Restartable workloads
|
||||
WORKLOADS := deployment/prometheus deployment/loki deployment/tempo deployment/grafana daemonset/alloy
|
||||
|
||||
C_OK := \033[0;32m
|
||||
C_WARN := \033[0;33m
|
||||
C_ERR := \033[0;31m
|
||||
C_HEAD := \033[1;36m
|
||||
C_OFF := \033[0m
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Help
|
||||
# -----------------------------------------------------------------------------
|
||||
.PHONY: help
|
||||
help: ## Show this help
|
||||
printf "$(C_HEAD)infra-observability$(C_OFF) -- namespace $(C_OK)$(NS)$(C_OFF) / context $(C_OK)$(CTX)$(C_OFF)\n\n"
|
||||
awk 'BEGIN {FS = ":.*?## "} \
|
||||
/^[a-zA-Z0-9_-]+:.*?## / {printf " $(C_OK)%-24s$(C_OFF) %s\n", $$1, $$2} \
|
||||
/^##@/ {printf "\n$(C_HEAD)%s$(C_OFF)\n", substr($$0, 5)}' $(MAKEFILE_LIST)
|
||||
printf "\n"
|
||||
|
||||
##@ Deployment
|
||||
|
||||
.PHONY: deploy
|
||||
deploy: check-context secret ## Deploy the whole stack (runtimes + dashboards)
|
||||
$(KUBECTL) apply -k $(K8S)
|
||||
$(MAKE) --no-print-directory wait
|
||||
|
||||
.PHONY: deploy-runtime
|
||||
deploy-runtime: check-context secret ## Deploy runtimes only (no dashboards)
|
||||
$(KUBECTL) apply -f $(K8S)/namespace.yaml -f $(K8S)/secret.yaml
|
||||
$(KUBECTL) apply -k $(K8S)/prometheus
|
||||
$(KUBECTL) apply -k $(K8S)/loki
|
||||
$(KUBECTL) apply -k $(K8S)/tempo
|
||||
$(KUBECTL) apply -k $(K8S)/alloy
|
||||
$(KUBECTL) apply -k $(K8S)/grafana
|
||||
$(KUBECTL) apply -k $(K8S)/ingress
|
||||
$(MAKE) --no-print-directory wait
|
||||
|
||||
.PHONY: prometheus loki tempo grafana alloy ingress
|
||||
prometheus: check-context ## Deploy/update Prometheus only
|
||||
$(KUBECTL) apply -k $(K8S)/prometheus
|
||||
loki: check-context ## Deploy/update Loki only
|
||||
$(KUBECTL) apply -k $(K8S)/loki
|
||||
tempo: check-context ## Deploy/update Tempo only
|
||||
$(KUBECTL) apply -k $(K8S)/tempo
|
||||
grafana: check-context ## Deploy/update the Grafana runtime only (datasources included)
|
||||
$(KUBECTL) apply -k $(K8S)/grafana
|
||||
alloy: check-context ## Deploy/update Alloy only (log collection)
|
||||
$(KUBECTL) apply -k $(K8S)/alloy
|
||||
ingress: check-context ## Deploy/update the Ingresses and the certificate
|
||||
$(KUBECTL) apply -k $(K8S)/ingress
|
||||
|
||||
.PHONY: secret
|
||||
secret: ## Generate k8s/secret.yaml (random admin password) if absent
|
||||
if [ ! -f $(K8S)/secret.yaml ]; then \
|
||||
pw=$$(LC_ALL=C tr -dc 'A-Za-z0-9' </dev/urandom | head -c 20); \
|
||||
sed -e '1s|.*|# Generated by `make secret` -- NOT versioned (see .gitignore).|' \
|
||||
-e "s|<CHANGE_ME>|$$pw|" $(K8S)/secret.example.yaml > $(K8S)/secret.yaml; \
|
||||
printf "$(C_OK)secret.yaml generated$(C_OFF) -- admin password: $$pw\n"; \
|
||||
else \
|
||||
printf "secret.yaml already present (use make grafana-password to read it)\n"; \
|
||||
fi
|
||||
|
||||
##@ Dashboards (declarative, stable UIDs)
|
||||
|
||||
.PHONY: dashboards
|
||||
dashboards: check-context dashboards-check ## Apply dashboards only (no Grafana restart)
|
||||
$(KUBECTL) apply -k $(DASHBOARDS)
|
||||
printf "$(C_OK)grafana-dashboards ConfigMap updated$(C_OFF) -- visible in Grafana within ~70s\n"
|
||||
printf " (kubelet volume sync ~60s + provisioner poll 10s, no restart).\n"
|
||||
printf " For immediate effect: $(C_WARN)make restart-grafana$(C_OFF)\n"
|
||||
|
||||
.PHONY: dashboards-sync
|
||||
dashboards-sync: ## Regenerate the .json list in dashboards/kustomization.yaml
|
||||
cd $(DASHBOARDS) && python3 -c "import glob,re,pathlib; \
|
||||
p=pathlib.Path('kustomization.yaml'); s=p.read_text(); \
|
||||
files=sorted(glob.glob('*.json')); \
|
||||
block=''.join(' - %s\n' % f for f in files); \
|
||||
s=re.sub(r'(# >>> dashboards.*?>>>\n).*?( # <<< dashboards <<<)', lambda m: m.group(1)+block+m.group(2), s, flags=re.S); \
|
||||
p.write_text(s); print('%d dashboard(s) referenced' % len(files))"
|
||||
$(MAKE) --no-print-directory dashboards-check
|
||||
|
||||
.PHONY: dashboards-check
|
||||
dashboards-check: ## Validate the JSON files and UID uniqueness/stability
|
||||
fail=0; \
|
||||
for f in $(DASHBOARDS)/*.json; do \
|
||||
jq -e . "$$f" >/dev/null 2>&1 || { printf "$(C_ERR)invalid JSON$(C_OFF): %s\n" "$$f"; fail=1; continue; }; \
|
||||
uid=$$(jq -r '.uid // empty' "$$f"); \
|
||||
[ -n "$$uid" ] || { printf "$(C_ERR)missing uid$(C_OFF): %s\n" "$$f"; fail=1; }; \
|
||||
base=$$(basename "$$f" .json); \
|
||||
[ "$$uid" = "$$base" ] || printf "$(C_WARN)filename != uid$(C_OFF): %s (uid=%s)\n" "$$f" "$$uid"; \
|
||||
[ "$$(jq -r '.id // "null"' "$$f")" = "null" ] || printf "$(C_WARN)non-null id field$(C_OFF): %s (remove it)\n" "$$f"; \
|
||||
grep -q -- "- $$base.json" $(DASHBOARDS)/kustomization.yaml || { printf "$(C_ERR)not referenced in kustomization.yaml$(C_OFF): %s (run make dashboards-sync)\n" "$$f"; fail=1; }; \
|
||||
done; \
|
||||
dup=$$(jq -r '.uid' $(DASHBOARDS)/*.json | sort | uniq -d); \
|
||||
[ -z "$$dup" ] || { printf "$(C_ERR)duplicate UIDs$(C_OFF): %s\n" "$$dup"; fail=1; }; \
|
||||
[ $$fail -eq 0 ] && printf "$(C_OK)dashboards OK$(C_OFF) (%s)\n" "$$(ls $(DASHBOARDS)/*.json | wc -l | tr -d ' ')"; \
|
||||
exit $$fail
|
||||
|
||||
.PHONY: dashboards-list
|
||||
dashboards-list: ## List the dashboards actually loaded by Grafana
|
||||
auth="$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_USER}' | base64 -d):$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_PASSWORD}' | base64 -d)"; \
|
||||
$(KUBECTL) -n $(NS) exec deployment/grafana -- curl -s -u "$$auth" 'http://localhost:3000/api/search?type=dash-db' \
|
||||
| jq -r '.[] | "\(.uid)\t\(.folderTitle // "General")\t\(.title)"'
|
||||
|
||||
##@ Diagnostics
|
||||
|
||||
.PHONY: status
|
||||
status: ## Overview (pods, services, PVCs, ingresses)
|
||||
$(KUBECTL) -n $(NS) get pods -o wide
|
||||
echo
|
||||
$(KUBECTL) -n $(NS) get svc,pvc,ingress
|
||||
|
||||
.PHONY: diag
|
||||
diag: ## Full diagnostics: state, PVCs, endpoints, certificate, events
|
||||
printf "$(C_HEAD)== Pods ==$(C_OFF)\n"; $(KUBECTL) -n $(NS) get pods -o wide
|
||||
printf "\n$(C_HEAD)== PVCs ==$(C_OFF)\n"; $(KUBECTL) -n $(NS) get pvc
|
||||
printf "\n$(C_HEAD)== Endpoints ==$(C_OFF)\n"; $(KUBECTL) -n $(NS) get endpoints
|
||||
printf "\n$(C_HEAD)== Ingresses ==$(C_OFF)\n"; $(KUBECTL) -n $(NS) get ingress
|
||||
printf "\n$(C_HEAD)== Certificate ==$(C_OFF)\n";$(KUBECTL) -n $(NS) get certificate 2>/dev/null || echo "(cert-manager absent)"
|
||||
printf "\n$(C_HEAD)== ConfigMaps ==$(C_OFF)\n";$(KUBECTL) -n $(NS) get cm
|
||||
printf "\n$(C_HEAD)== Containers not ready ==$(C_OFF)\n"
|
||||
$(KUBECTL) -n $(NS) get pods -o json | jq -r '.items[] | select(.status.containerStatuses != null) | .status.containerStatuses[] | select(.ready == false) | "\(.name)\trestarts=\(.restartCount)\t\(.state | keys[0]): \(.state[keys[0]].reason // "-")"' || true
|
||||
printf "\n$(C_HEAD)== LibreChat scrape token ==$(C_OFF)\n"
|
||||
$(MAKE) --no-print-directory check-metrics-token || true
|
||||
printf "\n$(C_HEAD)== Last 15 events ==$(C_OFF)\n"
|
||||
$(KUBECTL) -n $(NS) get events --sort-by=.lastTimestamp | tail -15
|
||||
|
||||
.PHONY: smoke
|
||||
smoke: ## End-to-end checks: component health, datasources, dashboards, targets
|
||||
# The Loki and Tempo images ship neither curl nor wget, so probes run from the
|
||||
# Grafana pod, which has curl and sees the same Services as the datasources.
|
||||
probe() { $(KUBECTL) -n $(NS) exec deployment/grafana -- curl -s --max-time 10 "$$1"; }; \
|
||||
auth="$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_USER}' | base64 -d):$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_PASSWORD}' | base64 -d)"; \
|
||||
printf "$(C_HEAD)== Component health ==$(C_OFF)\n"; \
|
||||
printf " %-12s %s\n" "prometheus" "$$(probe http://prometheus:9090/-/ready)"; \
|
||||
printf " %-12s %s\n" "loki" "$$(probe http://loki:3100/ready)"; \
|
||||
printf " %-12s %s\n" "tempo" "$$(probe http://tempo:3200/ready)"; \
|
||||
printf " %-12s %s\n" "grafana" "$$(probe http://localhost:3000/api/health | jq -c .)"; \
|
||||
printf " %-12s %s\n" "alloy" "$$(probe http://alloy:12345/-/ready)"; \
|
||||
printf "\n$(C_HEAD)== Grafana datasources ==$(C_OFF)\n"; \
|
||||
for uid in prometheus loki tempo; do \
|
||||
r=$$($(KUBECTL) -n $(NS) exec deployment/grafana -- curl -s -u "$$auth" -X POST "http://localhost:3000/api/datasources/uid/$$uid/health" | jq -r '"\(.status) \(.message)"'); \
|
||||
printf " %-12s %s\n" "$$uid" "$$r"; \
|
||||
done; \
|
||||
printf "\n$(C_HEAD)== Provisioned dashboards ==$(C_OFF)\n"; \
|
||||
$(KUBECTL) -n $(NS) exec deployment/grafana -- curl -s -u "$$auth" 'http://localhost:3000/api/search?type=dash-db' \
|
||||
| jq -r '.[] | " \(.uid) \(.folderTitle // "General")/\(.title)"'; \
|
||||
printf "\n$(C_HEAD)== Namespaces seen by Loki ==$(C_OFF)\n"; \
|
||||
probe 'http://loki:3100/loki/api/v1/label/namespace/values' | jq -r '" " + (.data // [] | join(", "))'; \
|
||||
printf "\n$(C_HEAD)== Prometheus targets ==$(C_OFF)\n"; \
|
||||
probe http://prometheus:9090/api/v1/targets | jq -r '.data.activeTargets | group_by(.health)[] | " \(.[0].health)\t\(length)"'
|
||||
|
||||
.PHONY: targets
|
||||
targets: ## List Prometheus scrape targets and their state
|
||||
$(KUBECTL) -n $(NS) exec deployment/prometheus -- wget -qO- http://localhost:9090/api/v1/targets \
|
||||
| jq -r '.data.activeTargets[] | "\(.health)\t\(.labels.job)\t\(.scrapeUrl)\t\(.lastError // "")"' | sort
|
||||
|
||||
.PHONY: check-metrics-token
|
||||
check-metrics-token: ## Check the LibreChat token matches across both namespaces
|
||||
a=$$($(KUBECTL) -n $(NS) get secret librechat-metrics -o jsonpath='{.data.token}' 2>/dev/null | base64 -d); \
|
||||
b=$$($(KUBECTL) -n librechat get secret librechat-secrets -o jsonpath='{.data.METRICS_SECRET}' 2>/dev/null | base64 -d); \
|
||||
if [ -z "$$a" ] || [ -z "$$b" ]; then \
|
||||
printf "$(C_WARN)token not found$(C_OFF) (observability/librechat-metrics=%s, librechat/librechat-secrets=%s)\n" "$${a:+present}$${a:-absent}" "$${b:+present}$${b:-absent}"; \
|
||||
elif [ "$$a" = "$$b" ]; then \
|
||||
printf "$(C_OK)LibreChat token matches$(C_OFF) across observability and librechat\n"; \
|
||||
else \
|
||||
printf "$(C_ERR)DRIFT$(C_OFF): the two Secrets hold different values.\n"; \
|
||||
printf " The librechat scrape will fall back to 401. Realign both copies.\n"; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
.PHONY: logs-prometheus logs-loki logs-tempo logs-grafana logs-alloy
|
||||
logs-prometheus: ## Follow Prometheus logs
|
||||
$(KUBECTL) -n $(NS) logs -f deployment/prometheus --tail=100
|
||||
logs-loki: ## Follow Loki logs
|
||||
$(KUBECTL) -n $(NS) logs -f deployment/loki --tail=100
|
||||
logs-tempo: ## Follow Tempo logs
|
||||
$(KUBECTL) -n $(NS) logs -f deployment/tempo --tail=100
|
||||
logs-grafana: ## Follow Grafana logs
|
||||
$(KUBECTL) -n $(NS) logs -f deployment/grafana --tail=100
|
||||
logs-alloy: ## Follow Alloy logs
|
||||
$(KUBECTL) -n $(NS) logs -f daemonset/alloy --tail=100
|
||||
|
||||
.PHONY: events
|
||||
events: ## Namespace events, oldest to newest
|
||||
$(KUBECTL) -n $(NS) get events --sort-by=.lastTimestamp
|
||||
|
||||
.PHONY: top
|
||||
top: ## Pod CPU/memory usage
|
||||
$(KUBECTL) -n $(NS) top pods 2>/dev/null || echo "metrics-server unavailable"
|
||||
|
||||
.PHONY: urls
|
||||
urls: ## Show access URLs
|
||||
printf " Grafana https://grafana.sttlab.local\n"
|
||||
printf " Prometheus https://prometheus.sttlab.local\n"
|
||||
printf " Alloy https://alloy.sttlab.local\n"
|
||||
printf "\n In-cluster: http://prometheus:9090 - http://loki:3100 - http://tempo:3200\n"
|
||||
printf " OTLP : tempo.$(NS).svc.cluster.local:4317 (gRPC) / :4318 (HTTP)\n"
|
||||
|
||||
.PHONY: hosts
|
||||
hosts: ## Show the required /etc/hosts entries
|
||||
printf "# Add to /etc/hosts:\n"
|
||||
for h in $(HOSTS); do printf "$(INGRESS_IP)\t%s\n" "$$h"; done
|
||||
|
||||
.PHONY: grafana-password
|
||||
grafana-password: ## Show the Grafana admin credentials
|
||||
printf "user : %s\n" "$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_USER}' | base64 -d)"
|
||||
printf "password : %s\n" "$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_PASSWORD}' | base64 -d)"
|
||||
|
||||
.PHONY: pf-grafana pf-prometheus pf-loki pf-tempo pf-alloy
|
||||
pf-grafana: ## Port-forward Grafana on :3000
|
||||
$(KUBECTL) -n $(NS) port-forward svc/grafana 3000:3000
|
||||
pf-prometheus: ## Port-forward Prometheus on :9090
|
||||
$(KUBECTL) -n $(NS) port-forward svc/prometheus 9090:9090
|
||||
pf-loki: ## Port-forward Loki on :3100
|
||||
$(KUBECTL) -n $(NS) port-forward svc/loki 3100:3100
|
||||
pf-tempo: ## Port-forward Tempo on :3200
|
||||
$(KUBECTL) -n $(NS) port-forward svc/tempo 3200:3200
|
||||
pf-alloy: ## Port-forward the Alloy UI on :12345
|
||||
$(KUBECTL) -n $(NS) port-forward svc/alloy 12345:12345
|
||||
|
||||
##@ Operations
|
||||
|
||||
.PHONY: wait
|
||||
wait: ## Wait for every workload to become ready
|
||||
for w in $(WORKLOADS); do \
|
||||
printf " %-24s " "$$w"; \
|
||||
$(KUBECTL) -n $(NS) rollout status $$w --timeout=300s 2>&1 | tail -1; \
|
||||
done
|
||||
|
||||
.PHONY: restart
|
||||
restart: ## Restart every workload
|
||||
$(KUBECTL) -n $(NS) rollout restart $(WORKLOADS)
|
||||
$(MAKE) --no-print-directory wait
|
||||
|
||||
.PHONY: restart-prometheus restart-loki restart-tempo restart-grafana restart-alloy
|
||||
restart-prometheus: ## Restart Prometheus
|
||||
$(KUBECTL) -n $(NS) rollout restart deployment/prometheus && $(KUBECTL) -n $(NS) rollout status deployment/prometheus
|
||||
restart-loki: ## Restart Loki
|
||||
$(KUBECTL) -n $(NS) rollout restart deployment/loki && $(KUBECTL) -n $(NS) rollout status deployment/loki
|
||||
restart-tempo: ## Restart Tempo
|
||||
$(KUBECTL) -n $(NS) rollout restart deployment/tempo && $(KUBECTL) -n $(NS) rollout status deployment/tempo
|
||||
restart-grafana: ## Restart Grafana (dashboards picked up immediately)
|
||||
$(KUBECTL) -n $(NS) rollout restart deployment/grafana && $(KUBECTL) -n $(NS) rollout status deployment/grafana
|
||||
restart-alloy: ## Restart Alloy
|
||||
$(KUBECTL) -n $(NS) rollout restart daemonset/alloy && $(KUBECTL) -n $(NS) rollout status daemonset/alloy
|
||||
|
||||
.PHONY: reload-prometheus
|
||||
reload-prometheus: ## Hot-reload the Prometheus config (no restart)
|
||||
$(KUBECTL) -n $(NS) exec deployment/prometheus -- wget -q --post-data='' -O- http://localhost:9090/-/reload \
|
||||
&& printf "$(C_OK)Prometheus configuration reloaded$(C_OFF)\n"
|
||||
|
||||
##@ Persistence
|
||||
|
||||
.PHONY: pvc
|
||||
pvc: ## PVC state and the volumes behind them
|
||||
$(KUBECTL) -n $(NS) get pvc -o custom-columns='NAME:.metadata.name,STATUS:.status.phase,CAPACITY:.status.capacity.storage,SC:.spec.storageClassName,VOLUME:.spec.volumeName'
|
||||
echo
|
||||
$(KUBECTL) -n $(NS) get pods -o json | jq -r '.items[] | . as $$p | (.spec.volumes[]? | select(.persistentVolumeClaim) | "\($$p.metadata.name)\t-> \(.persistentVolumeClaim.claimName)")'
|
||||
|
||||
.PHONY: flush-pvc
|
||||
flush-pvc: ## DESTRUCTIVE: wipe all persisted data (CONFIRM=yes)
|
||||
$(call confirm,Wipe ALL data (metrics, logs, traces, Grafana))
|
||||
for c in $(STATEFUL); do $(MAKE) --no-print-directory _flush COMP=$$c; done
|
||||
$(MAKE) --no-print-directory wait
|
||||
|
||||
.PHONY: flush-prometheus flush-loki flush-tempo flush-grafana
|
||||
flush-prometheus: ## DESTRUCTIVE: wipe the Prometheus TSDB (CONFIRM=yes)
|
||||
$(call confirm,Wipe the Prometheus TSDB)
|
||||
$(MAKE) --no-print-directory _flush COMP=prometheus
|
||||
flush-loki: ## DESTRUCTIVE: wipe the Loki logs (CONFIRM=yes)
|
||||
$(call confirm,Wipe the Loki logs)
|
||||
$(MAKE) --no-print-directory _flush COMP=loki
|
||||
flush-tempo: ## DESTRUCTIVE: wipe the Tempo traces (CONFIRM=yes)
|
||||
$(call confirm,Wipe the Tempo traces)
|
||||
$(MAKE) --no-print-directory _flush COMP=tempo
|
||||
flush-grafana: ## DESTRUCTIVE: wipe the Grafana database (CONFIRM=yes)
|
||||
$(call confirm,Wipe the Grafana database (users and prefs; dashboards are reprovisioned))
|
||||
$(MAKE) --no-print-directory _flush COMP=grafana
|
||||
|
||||
# Internal target: scale down -> delete PVC -> recreate -> scale up.
|
||||
.PHONY: _flush
|
||||
_flush:
|
||||
printf "$(C_WARN)flushing %s$(C_OFF)\n" "$(COMP)"
|
||||
$(KUBECTL) -n $(NS) scale deployment/$(COMP) --replicas=0
|
||||
$(KUBECTL) -n $(NS) wait --for=delete pod -l app=$(COMP) --timeout=120s 2>/dev/null || true
|
||||
$(KUBECTL) -n $(NS) delete pvc $(COMP)-data --ignore-not-found --wait=true
|
||||
$(KUBECTL) apply -k $(K8S)/$(COMP)
|
||||
$(KUBECTL) -n $(NS) scale deployment/$(COMP) --replicas=1
|
||||
|
||||
##@ Validation and teardown
|
||||
|
||||
.PHONY: build
|
||||
build: ## Render the generated manifests to stdout
|
||||
kustomize build $(K8S)
|
||||
|
||||
.PHONY: diff
|
||||
diff: check-context ## Diff the manifests against the cluster state
|
||||
$(KUBECTL) diff -k $(K8S) || true
|
||||
|
||||
.PHONY: validate
|
||||
validate: dashboards-check ## Validate the manifests (build + server dry-run)
|
||||
kustomize build $(K8S) >/dev/null && printf "$(C_OK)kustomize build OK$(C_OFF)\n"
|
||||
$(KUBECTL) apply -k $(K8S) --dry-run=server >/dev/null && printf "$(C_OK)server dry-run OK$(C_OFF)\n"
|
||||
|
||||
.PHONY: undeploy
|
||||
undeploy: ## DESTRUCTIVE: delete the stack, PVCs kept (CONFIRM=yes)
|
||||
$(call confirm,Delete the stack (PVCs are KEPT))
|
||||
$(KUBECTL) delete -k $(K8S) --ignore-not-found
|
||||
|
||||
.PHONY: destroy
|
||||
destroy: ## DESTRUCTIVE: delete the stack AND the PVCs (CONFIRM=yes)
|
||||
$(call confirm,Delete the stack AND all data)
|
||||
$(KUBECTL) delete -k $(K8S) --ignore-not-found
|
||||
$(KUBECTL) -n $(NS) delete pvc --all --ignore-not-found
|
||||
|
||||
.PHONY: check-context
|
||||
check-context:
|
||||
cur=$$(kubectl config current-context); \
|
||||
if [ "$$cur" != "$(CTX)" ]; then \
|
||||
printf "$(C_ERR)current context '%s' != '$(CTX)'$(C_OFF) (commands explicitly target $(CTX))\n" "$$cur"; \
|
||||
fi
|
||||
|
||||
# Guard rail for destructive targets: requires CONFIRM=yes
|
||||
define confirm
|
||||
@if [ "$(CONFIRM)" != "yes" ]; then \
|
||||
printf "$(C_ERR)Destructive operation$(C_OFF): $(1)\n"; \
|
||||
printf "Re-run with $(C_WARN)CONFIRM=yes$(C_OFF): make $(C_WARN)$@ CONFIRM=yes$(C_OFF)\n"; \
|
||||
exit 1; \
|
||||
fi
|
||||
endef
|
||||
@@ -0,0 +1,281 @@
|
||||
# Infra Observability (Kubernetes / declarative)
|
||||
|
||||
**Grafana 13.2 / Prometheus 3.14 / Loki 3.7 / Tempo 3.0** plus **Alloy 1.19**,
|
||||
deployed fully declaratively (Kustomize) into the `observability` namespace of
|
||||
the local `orbstack` cluster.
|
||||
|
||||
Nothing is imperative: no `helm install`, no dashboard hand-created in the UI.
|
||||
The entire desired state lives in this repository and `make deploy` reconciles it.
|
||||
|
||||
## Repository layout
|
||||
|
||||
```text
|
||||
Makefile # Every command (make = help)
|
||||
k8s/
|
||||
├── kustomization.yaml # Root Kustomize manifest
|
||||
├── namespace.yaml # observability namespace
|
||||
├── secret.yaml # Grafana admin credentials (GITIGNORED)
|
||||
├── secret.example.yaml # Secret template
|
||||
├── prometheus/
|
||||
│ ├── prometheus.yaml # Scrape config (Kubernetes SD)
|
||||
│ ├── rbac.yaml # SA + discovery ClusterRole
|
||||
│ ├── secret.yaml # LibreChat scrape token (GITIGNORED)
|
||||
│ ├── secret.example.yaml # Token template
|
||||
│ ├── pvc.yaml # Persistent TSDB (10Gi)
|
||||
│ ├── deployment.yaml # Prometheus v3.14.0
|
||||
│ └── service.yaml # :9090
|
||||
├── loki/
|
||||
│ ├── loki.yaml # Monolithic config, filesystem storage
|
||||
│ ├── pvc.yaml # Chunks + index (10Gi)
|
||||
│ ├── deployment.yaml # Loki 3.7.6 (-target=all)
|
||||
│ └── service.yaml # :3100 (HTTP) / :9096 (gRPC)
|
||||
├── tempo/
|
||||
│ ├── tempo.yaml # Monolithic config + metrics-generator
|
||||
│ ├── pvc.yaml # WAL + trace blocks (10Gi)
|
||||
│ ├── deployment.yaml # Tempo 3.0.3 (-target=all)
|
||||
│ └── service.yaml # :3200, OTLP :4317/:4318, Zipkin :9411
|
||||
├── alloy/
|
||||
│ ├── config.alloy # Pod logs -> Loki pipeline
|
||||
│ ├── rbac.yaml # SA + ClusterRole (including pods/log)
|
||||
│ ├── daemonset.yaml # Alloy v1.19.0 (one pod per node)
|
||||
│ └── service.yaml # :12345 (UI + /metrics)
|
||||
├── grafana/
|
||||
│ ├── provisioning/
|
||||
│ │ ├── datasources.yaml # Datasources with stable UIDs
|
||||
│ │ └── dashboard-providers.yaml # File provider
|
||||
│ ├── dashboards/
|
||||
│ │ ├── kustomization.yaml # ConfigMap WITHOUT hash (hot reload)
|
||||
│ │ ├── obs-stack-health.json # uid: obs-stack-health
|
||||
│ │ ├── obs-keycloak.json # uid: obs-keycloak
|
||||
│ │ ├── obs-librechat.json # uid: obs-librechat
|
||||
│ │ └── obs-classic-models-api.json
|
||||
│ ├── pvc.yaml # Grafana SQLite database (5Gi)
|
||||
│ ├── deployment.yaml # Grafana 13.2.0
|
||||
│ └── service.yaml # :3000
|
||||
└── ingress/
|
||||
├── certificate.yaml # cert-manager Certificate (sttlab-local-ca)
|
||||
└── ingress.yaml # Three NGINX Ingresses
|
||||
```
|
||||
|
||||
## Getting started
|
||||
|
||||
```bash
|
||||
make # Help: every available target
|
||||
make deploy # Deploy the whole stack
|
||||
make hosts # /etc/hosts entries to add
|
||||
make grafana-password # Admin credentials
|
||||
make smoke # End-to-end verification
|
||||
```
|
||||
|
||||
| URL | Purpose |
|
||||
|---|---|
|
||||
| `https://grafana.sttlab.local` | Grafana UI |
|
||||
| `https://prometheus.sttlab.local` | Prometheus UI (targets, expression browser) |
|
||||
| `https://alloy.sttlab.local` | Alloy UI (log pipeline state) |
|
||||
|
||||
Sending traces from an in-cluster application:
|
||||
`tempo.observability.svc.cluster.local:4317` (OTLP gRPC) or `:4318` (OTLP HTTP).
|
||||
|
||||
## Declarative dashboards with stable UIDs
|
||||
|
||||
This is the centrepiece of the repository: **a dashboard only ever exists as a
|
||||
versioned JSON file**, never as an object created through the UI.
|
||||
|
||||
### Rules
|
||||
|
||||
1. **One `.json` file per dashboard**, under `k8s/grafana/dashboards/`.
|
||||
2. **The `uid` field is stable and never changes.** It is what keeps URLs
|
||||
(`/d/<uid>/…`), cross-dashboard links, annotations and alerts alive across
|
||||
every redeployment. By convention the filename equals the UID
|
||||
(`obs-keycloak.json` -> `uid: obs-keycloak`).
|
||||
3. **The `id` field must stay `null`** (or absent): it is an identifier internal
|
||||
to a Grafana instance and is not portable.
|
||||
4. **Datasources are referenced by stable UID**, never by name:
|
||||
`"datasource": { "type": "prometheus", "uid": "prometheus" }`.
|
||||
Available UIDs: `prometheus`, `loki`, `tempo`.
|
||||
5. `allowUiUpdates: false`: any change made in the UI is overwritten on the next
|
||||
reload. The file is the source of truth.
|
||||
|
||||
### Adding a dashboard
|
||||
|
||||
```bash
|
||||
cp my-dashboard.json k8s/grafana/dashboards/obs-my-topic.json
|
||||
# edit the JSON: "uid": "obs-my-topic", "id": null
|
||||
make dashboards-sync # update the list in dashboards/kustomization.yaml
|
||||
make dashboards # apply (without restarting Grafana)
|
||||
```
|
||||
|
||||
`make dashboards-check` (run automatically by `dashboards` and `validate`)
|
||||
verifies: valid JSON, `uid` present, `uid` unique, `id` null, file referenced in
|
||||
the kustomization, filename/UID consistency.
|
||||
|
||||
### How the hot reload works
|
||||
|
||||
The `grafana-dashboards` ConfigMap is generated **without a name-suffix hash**
|
||||
(`disableNameSuffixHash: true`), so `make dashboards` updates it *in place*: the
|
||||
Grafana pod is not recreated and the file provider re-reads the directory
|
||||
(`updateIntervalSeconds: 10`).
|
||||
|
||||
Measured end-to-end delay: **about 70 seconds** (kubelet ConfigMap volume sync
|
||||
~60s + provider poll 10s). For immediate effect: `make restart-grafana`.
|
||||
|
||||
Conversely, the *provisioning* ConfigMaps (datasources, providers) **keep** their
|
||||
hash suffix: those files are only read at startup, so the hash deliberately
|
||||
forces a pod restart when they change.
|
||||
|
||||
## Persistence
|
||||
|
||||
| PVC | Component | Size | Contents | Retention |
|
||||
|---|---|---|---|---|
|
||||
| `prometheus-data` | Prometheus | 10Gi | TSDB | 15 d / 8 GB |
|
||||
| `loki-data` | Loki | 10Gi | chunks + TSDB index | 168 h (compactor) |
|
||||
| `tempo-data` | Tempo | 10Gi | WAL + trace blocks | 168 h |
|
||||
| `grafana-data` | Grafana | 5Gi | SQLite database (users, prefs) | — |
|
||||
|
||||
StorageClass `local-path` (the cluster default), `ReadWriteOnce`. The Deployments
|
||||
use `strategy: Recreate`: with an RWO volume the old pod must release it before
|
||||
the new one can start.
|
||||
|
||||
Alloy's read positions live on a `hostPath` (`/var/lib/alloy`), so restarting the
|
||||
DaemonSet does not re-ingest every log.
|
||||
|
||||
### Wiping data
|
||||
|
||||
```bash
|
||||
make pvc # PVC state
|
||||
make flush-loki CONFIRM=yes # wipe the logs only
|
||||
make flush-prometheus CONFIRM=yes # wipe the TSDB only
|
||||
make flush-pvc CONFIRM=yes # wipe EVERYTHING
|
||||
```
|
||||
|
||||
The cycle is: `scale 0` -> wait for pod deletion -> `delete pvc` -> reapply the
|
||||
kustomization (which recreates the PVC) -> `scale 1`. Every destructive target
|
||||
requires `CONFIRM=yes`.
|
||||
|
||||
Dashboards are **not** affected by `flush-grafana`: they are reprovisioned from
|
||||
the ConfigMaps at startup.
|
||||
|
||||
## Wiring the three signals
|
||||
|
||||
- **Prometheus** discovers its targets through the Kubernetes API: apiserver,
|
||||
kubelet, container resource metrics, plus any Service or Pod carrying the
|
||||
`prometheus.io/scrape: "true"` annotation (`prometheus.io/port`,
|
||||
`prometheus.io/path`).
|
||||
- **Alloy** reads every pod's logs through the Kubernetes API (no `/var/log`
|
||||
mount) and pushes them to Loki with `namespace`, `pod`, `container`, `node`,
|
||||
`app`, `component` and `job` labels.
|
||||
- **Tempo** receives OTLP, and its `metrics_generator` pushes span metrics and
|
||||
service graphs to Prometheus through its remote-write receiver
|
||||
(`--web.enable-remote-write-receiver`).
|
||||
- **Grafana correlations**: Loki -> Tempo (`TraceID` derived field),
|
||||
Tempo -> Loki (`tracesToLogsV2`), Tempo -> Prometheus (`serviceMap`,
|
||||
`tracesToMetrics`), Prometheus exemplars -> Tempo.
|
||||
|
||||
### Container resource metrics
|
||||
|
||||
Prometheus scrapes the kubelet's **`/metrics/resource`** endpoint, not
|
||||
`/metrics/cadvisor`. On this cluster the cadvisor endpoint answers with `machine_*`
|
||||
series only and exposes no `container_*` at all, so CPU and memory panels would
|
||||
stay empty. `/metrics/resource` — the source metrics-server itself reads —
|
||||
provides `container_cpu_usage_seconds_total` and
|
||||
`container_memory_working_set_bytes` with `namespace`/`pod`/`container` labels.
|
||||
|
||||
kube-state-metrics is not deployed, so container restarts are derived from
|
||||
changes in `container_start_time_seconds`.
|
||||
|
||||
## Connected applications
|
||||
|
||||
| Application | Traces | Metrics | Logs | Dashboard |
|
||||
|---|---|---|---|---|
|
||||
| `keycloak` | OTel Java agent → `tempo:4318` | native `/metrics` (management port 9000) | JSON + MDC `trace_id` | `obs-keycloak` |
|
||||
| `librechat` | OTel SDK → `tempo:4318` | built-in `/metrics`, Bearer token | stdout | `obs-librechat` |
|
||||
| `classic-models-api` | `opentelemetry-instrument` → `tempo:4318` | django-prometheus `/metrics` | JSON + `otelTraceID` | `obs-classic-models-api` |
|
||||
|
||||
Their manifests live in the `infra-keycloak`, `infra-librechat` and
|
||||
`classic-models-api` repositories; this repository only holds the platform.
|
||||
|
||||
Each application dashboard carries the same four sections: an overview of RED
|
||||
indicators, HTTP traffic, a runtime section specific to the stack in use, and a
|
||||
**Logs** row, followed by container resources.
|
||||
|
||||
The Logs row leans on Loki's `detected_level`, which is inferred at ingestion
|
||||
time, so a single query works across all three log formats — Keycloak emits JSON,
|
||||
LibreChat colour-coded text, and the Django API plain gunicorn access lines. No
|
||||
per-application parser is needed. The volume panel is scoped to the application
|
||||
container, while the two error panels cover the whole namespace so the
|
||||
datastore's own failures surface alongside the application's. Where a log line
|
||||
carries a `trace_id`, the Loki datasource's derived field turns it into a link
|
||||
straight to the matching trace in Tempo.
|
||||
|
||||
### Connecting an application
|
||||
|
||||
1. **Traces** — point OTLP at Tempo:
|
||||
`OTEL_EXPORTER_OTLP_ENDPOINT=http://tempo.observability.svc.cluster.local:4318`
|
||||
(`:4317` for gRPC). RED span metrics and the service graph are then derived
|
||||
automatically by Tempo's `metrics_generator`, with no extra instrumentation.
|
||||
2. **Metrics** — annotate the **pod** (not the Service):
|
||||
```yaml
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9000"
|
||||
prometheus.io/path: "/metrics"
|
||||
```
|
||||
3. **Logs** — nothing to do: Alloy collects every pod in the cluster. For Grafana
|
||||
to jump from a log line to its trace, the line only needs to contain a
|
||||
`trace_id` (the Loki datasource's derived field).
|
||||
|
||||
### Token-protected endpoints
|
||||
|
||||
Annotation-based discovery cannot carry credentials, so an authenticated target
|
||||
gets its own `scrape_config` in `k8s/prometheus/prometheus.yaml`, with the token
|
||||
mounted from a Secret — see the `librechat` job, which reads
|
||||
`/etc/prometheus/secrets/librechat/token`.
|
||||
|
||||
That token must be **identical** on both sides: `METRICS_SECRET` in the
|
||||
`librechat-secrets` secret (namespace `librechat`) and the `token` key of the
|
||||
`librechat-metrics` secret (namespace `observability`, file
|
||||
`k8s/prometheus/secret.yaml`, gitignored).
|
||||
|
||||
A Kubernetes Secret is namespaced and a pod can only mount ones from its own
|
||||
namespace, so the same value has to exist twice — once for LibreChat, which
|
||||
**verifies** the token, once for Prometheus, which **presents** it. The risk is
|
||||
silent drift: regenerating one side without the other sends the scrape back to
|
||||
401 without breaking anything else. `make check-metrics-token` (included in
|
||||
`make diag`) compares both copies.
|
||||
|
||||
## Diagnostics
|
||||
|
||||
```bash
|
||||
make status # pods, services, PVCs, ingresses
|
||||
make diag # + endpoints, certificate, failing containers, events
|
||||
make smoke # health of all five components, datasources, dashboards, targets
|
||||
make targets # Prometheus targets and their last scrape error
|
||||
make logs-loki # follow one component's logs
|
||||
make top # CPU/memory usage
|
||||
make pf-grafana # port-forward when the ingress is unavailable
|
||||
make diff # difference between the repository and the cluster
|
||||
make validate # kustomize build + server dry-run + dashboard checks
|
||||
```
|
||||
|
||||
## Secrets
|
||||
|
||||
`k8s/secret.yaml` and `k8s/prometheus/secret.yaml` are **gitignored**.
|
||||
`make secret` generates the first from `secret.example.yaml` with a random admin
|
||||
password if it does not exist (and is called automatically by `make deploy`).
|
||||
|
||||
The TLS certificate is not versioned either: it is issued by the existing
|
||||
`sttlab-local-ca` cert-manager ClusterIssuer (`k8s/ingress/certificate.yaml`).
|
||||
|
||||
## Coexisting with what was already there
|
||||
|
||||
The `observability` namespace already hosts two deployments **not managed by this
|
||||
repository**, left untouched:
|
||||
|
||||
- `jaeger` (installed by Helm) — it also exposes OTLP on `4317`/`4318`, but on its
|
||||
own Service, so there is no conflict with Tempo. Since all three applications
|
||||
now point at Tempo, Jaeger no longer receives anything; uninstall the Helm
|
||||
release whenever you want a single trace backend.
|
||||
- `audit-log-collector` (Fluent Bit) — writes to its own PVC.
|
||||
|
||||
`make undeploy` and `make destroy` only run `kubectl delete -k k8s/`, so they
|
||||
touch nothing but the resources described here.
|
||||
@@ -0,0 +1,66 @@
|
||||
// Grafana Alloy -- ships the logs of every pod in the cluster to Loki.
|
||||
// Reads through the Kubernetes API (no /var/log mount required).
|
||||
|
||||
logging {
|
||||
level = "info"
|
||||
format = "logfmt"
|
||||
}
|
||||
|
||||
discovery.kubernetes "pods" {
|
||||
role = "pod"
|
||||
}
|
||||
|
||||
discovery.relabel "pod_logs" {
|
||||
targets = discovery.kubernetes.pods.targets
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_namespace"]
|
||||
target_label = "namespace"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_name"]
|
||||
target_label = "pod"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_container_name"]
|
||||
target_label = "container"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_node_name"]
|
||||
target_label = "node"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
|
||||
target_label = "app"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_component"]
|
||||
target_label = "component"
|
||||
}
|
||||
// job = <namespace>/<container>: the usual Loki convention.
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"]
|
||||
separator = "/"
|
||||
target_label = "job"
|
||||
}
|
||||
// Only collect containers on the current node (DaemonSet).
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_node_name"]
|
||||
regex = env("HOSTNAME_NODE")
|
||||
action = "keep"
|
||||
}
|
||||
}
|
||||
|
||||
loki.source.kubernetes "pods" {
|
||||
targets = discovery.relabel.pod_logs.output
|
||||
forward_to = [loki.write.default.receiver]
|
||||
}
|
||||
|
||||
loki.write "default" {
|
||||
endpoint {
|
||||
url = "http://loki:3100/loki/api/v1/push"
|
||||
}
|
||||
external_labels = {
|
||||
cluster = "orbstack",
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: alloy
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: alloy
|
||||
app.kubernetes.io/component: collector
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: alloy
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: alloy
|
||||
app.kubernetes.io/name: alloy
|
||||
app.kubernetes.io/component: collector
|
||||
app.kubernetes.io/part-of: observability
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "12345"
|
||||
spec:
|
||||
serviceAccountName: alloy
|
||||
tolerations:
|
||||
- operator: Exists
|
||||
containers:
|
||||
- name: alloy
|
||||
image: grafana/alloy:v1.19.0
|
||||
args:
|
||||
- run
|
||||
- /etc/alloy/config.alloy
|
||||
- --storage.path=/var/lib/alloy/data
|
||||
- --server.http.listen-addr=0.0.0.0:12345
|
||||
- --disable-reporting
|
||||
env:
|
||||
# Used by config.alloy to tail only the pods on the current node.
|
||||
- name: HOSTNAME_NODE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 12345
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 512Mi
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /-/ready
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /etc/alloy
|
||||
- name: storage
|
||||
mountPath: /var/lib/alloy
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: alloy-config
|
||||
- name: storage
|
||||
# Read positions persisted on the node: no full log re-ingestion
|
||||
# when the pod restarts.
|
||||
hostPath:
|
||||
path: /var/lib/alloy
|
||||
type: DirectoryOrCreate
|
||||
@@ -0,0 +1,18 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: observability
|
||||
|
||||
resources:
|
||||
- rbac.yaml
|
||||
- daemonset.yaml
|
||||
- service.yaml
|
||||
|
||||
configMapGenerator:
|
||||
- name: alloy-config
|
||||
files:
|
||||
- config.alloy
|
||||
|
||||
generatorOptions:
|
||||
labels:
|
||||
app.kubernetes.io/name: alloy
|
||||
app.kubernetes.io/part-of: observability
|
||||
@@ -0,0 +1,52 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: alloy
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: alloy
|
||||
app.kubernetes.io/part-of: observability
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: observability-alloy
|
||||
labels:
|
||||
app.kubernetes.io/name: alloy
|
||||
app.kubernetes.io/part-of: observability
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources:
|
||||
- nodes
|
||||
- nodes/proxy
|
||||
- nodes/metrics
|
||||
- services
|
||||
- endpoints
|
||||
- pods
|
||||
- namespaces
|
||||
- events
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/log"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["discovery.k8s.io"]
|
||||
resources: ["endpointslices"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- nonResourceURLs: ["/metrics"]
|
||||
verbs: ["get"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: observability-alloy
|
||||
labels:
|
||||
app.kubernetes.io/name: alloy
|
||||
app.kubernetes.io/part-of: observability
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: observability-alloy
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: alloy
|
||||
namespace: observability
|
||||
@@ -0,0 +1,17 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: alloy
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: alloy
|
||||
app.kubernetes.io/component: collector
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: alloy
|
||||
ports:
|
||||
- name: http
|
||||
port: 12345
|
||||
targetPort: http
|
||||
@@ -0,0 +1,31 @@
|
||||
# Grafana dashboards -- declarative provisioning.
|
||||
#
|
||||
# Rules:
|
||||
# * one .json file per dashboard, named after its UID;
|
||||
# * every JSON carries a STABLE "uid" field (never regenerated): that is what
|
||||
# keeps URLs, cross-links and alerts alive across redeployments;
|
||||
# * `disableNameSuffixHash: true` -> the ConfigMap always keeps the same name,
|
||||
# so `kubectl apply` updates it in place and Grafana reloads WITHOUT
|
||||
# restarting the pod, in about 70s (see dashboard-providers.yaml).
|
||||
#
|
||||
# Adding a dashboard = drop the .json here and add it to `files:` below.
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: observability
|
||||
|
||||
configMapGenerator:
|
||||
- name: grafana-dashboards
|
||||
files:
|
||||
# >>> dashboards (generated by `make dashboards-sync`) >>>
|
||||
- obs-classic-models-api.json
|
||||
- obs-keycloak.json
|
||||
- obs-librechat.json
|
||||
- obs-stack-health.json
|
||||
# <<< dashboards <<<
|
||||
|
||||
generatorOptions:
|
||||
disableNameSuffixHash: true
|
||||
labels:
|
||||
app.kubernetes.io/name: grafana
|
||||
app.kubernetes.io/component: dashboards
|
||||
app.kubernetes.io/part-of: observability
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,437 @@
|
||||
{
|
||||
"uid": "obs-stack-health",
|
||||
"title": "Observability — Stack Health",
|
||||
"description": "Health of the Prometheus / Loki / Tempo / Alloy stack. Reference dashboard: stable UID, declaratively provisioned.",
|
||||
"tags": [
|
||||
"observability",
|
||||
"stack",
|
||||
"provisioned"
|
||||
],
|
||||
"editable": false,
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "30s",
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timezone": "browser",
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "namespace",
|
||||
"label": "Namespace",
|
||||
"type": "query",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"query": "label_values(kube_pod_info, namespace)",
|
||||
"refresh": 2,
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"current": {
|
||||
"text": "observability",
|
||||
"value": "observability"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "row",
|
||||
"title": "Collection",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "Prometheus targets UP",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum(up)",
|
||||
"legendFormat": "up"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"textMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "stat",
|
||||
"title": "Prometheus targets DOWN",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"y": 1
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum(up == bool 0)",
|
||||
"legendFormat": "down"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"textMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "stat",
|
||||
"title": "Log lines ingested (/s)",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 6,
|
||||
"x": 12,
|
||||
"y": 1
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum(rate(loki_distributor_lines_received_total[5m]))",
|
||||
"legendFormat": "lines/s"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 1,
|
||||
"color": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "blue"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"textMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "stat",
|
||||
"title": "Spans ingested (/s)",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 6,
|
||||
"x": 18,
|
||||
"y": 1
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum(rate(tempo_distributor_spans_received_total[5m]))",
|
||||
"legendFormat": "spans/s"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 1,
|
||||
"color": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "purple"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"textMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "row",
|
||||
"title": "Stack resources",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 6
|
||||
},
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "timeseries",
|
||||
"title": "CPU per component (observability namespace)",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 7
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"observability\", container!=\"\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{pod}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"mean",
|
||||
"max"
|
||||
]
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "timeseries",
|
||||
"title": "Memory per component (observability namespace)",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 7
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"observability\", container!=\"\"})",
|
||||
"legendFormat": "{{pod}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes",
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"mean",
|
||||
"max"
|
||||
]
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "row",
|
||||
"title": "Logs",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 15
|
||||
},
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "logs",
|
||||
"title": "Recent stack errors",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 10,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "{namespace=\"observability\"} |~ \"(?i)(error|level=error|fatal)\"",
|
||||
"queryType": "range"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"showLabels": false,
|
||||
"wrapLogMessage": true,
|
||||
"sortOrder": "Descending",
|
||||
"dedupStrategy": "none",
|
||||
"enableLogDetails": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: grafana
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: grafana
|
||||
app.kubernetes.io/component: ui
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: grafana
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: grafana
|
||||
app.kubernetes.io/name: grafana
|
||||
app.kubernetes.io/component: ui
|
||||
app.kubernetes.io/part-of: observability
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "3000"
|
||||
prometheus.io/path: "/metrics"
|
||||
spec:
|
||||
securityContext:
|
||||
runAsUser: 472
|
||||
runAsGroup: 472
|
||||
fsGroup: 472
|
||||
containers:
|
||||
- name: grafana
|
||||
image: grafana/grafana:13.2.0
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 3000
|
||||
env:
|
||||
- name: GF_SECURITY_ADMIN_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: observability-secrets
|
||||
key: GF_SECURITY_ADMIN_USER
|
||||
- name: GF_SECURITY_ADMIN_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: observability-secrets
|
||||
key: GF_SECURITY_ADMIN_PASSWORD
|
||||
- name: GF_SERVER_ROOT_URL
|
||||
value: https://grafana.sttlab.local
|
||||
- name: GF_PATHS_PROVISIONING
|
||||
value: /etc/grafana/provisioning
|
||||
# Everything is driven by the provisioning files, so creating
|
||||
# dashboards or datasources from the UI is disabled.
|
||||
- name: GF_USERS_ALLOW_SIGN_UP
|
||||
value: "false"
|
||||
- name: GF_ANALYTICS_REPORTING_ENABLED
|
||||
value: "false"
|
||||
- name: GF_ANALYTICS_CHECK_FOR_UPDATES
|
||||
value: "false"
|
||||
- name: GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES
|
||||
value: "false"
|
||||
- name: GF_NEWS_NEWS_FEED_ENABLED
|
||||
value: "false"
|
||||
- name: GF_LOG_LEVEL
|
||||
value: info
|
||||
- name: GF_FEATURE_TOGGLES_ENABLE
|
||||
value: traceqlEditor,traceQLStreaming,correlations,lokiLogsDataplane
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /api/health
|
||||
port: http
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /api/health
|
||||
port: http
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 30
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /var/lib/grafana
|
||||
- name: provisioning-datasources
|
||||
mountPath: /etc/grafana/provisioning/datasources
|
||||
- name: provisioning-dashboards
|
||||
mountPath: /etc/grafana/provisioning/dashboards
|
||||
- name: dashboards
|
||||
mountPath: /var/lib/grafana/dashboards
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: grafana-data
|
||||
- name: provisioning-datasources
|
||||
configMap:
|
||||
name: grafana-datasources
|
||||
- name: provisioning-dashboards
|
||||
configMap:
|
||||
name: grafana-dashboard-providers
|
||||
- name: dashboards
|
||||
# Stable-named ConfigMap: updated in place by `make dashboards` and
|
||||
# re-read by Grafana without a restart (~70s: kubelet sync + 10s poll).
|
||||
configMap:
|
||||
name: grafana-dashboards
|
||||
@@ -0,0 +1,27 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: observability
|
||||
|
||||
resources:
|
||||
- pvc.yaml
|
||||
- deployment.yaml
|
||||
- service.yaml
|
||||
# Dashboards live in ./dashboards and are referenced by the root kustomization:
|
||||
# `k8s/grafana` is the runtime alone, `k8s/grafana/dashboards` the content alone.
|
||||
# Both are deployable independently.
|
||||
|
||||
configMapGenerator:
|
||||
# Name-suffix hash KEPT here: datasource and provider provisioning is only
|
||||
# read at startup, so the hash deliberately forces a pod restart whenever
|
||||
# these files change.
|
||||
- name: grafana-datasources
|
||||
files:
|
||||
- datasources.yaml=provisioning/datasources.yaml
|
||||
- name: grafana-dashboard-providers
|
||||
files:
|
||||
- dashboard-providers.yaml=provisioning/dashboard-providers.yaml
|
||||
|
||||
generatorOptions:
|
||||
labels:
|
||||
app.kubernetes.io/name: grafana
|
||||
app.kubernetes.io/part-of: observability
|
||||
@@ -0,0 +1,19 @@
|
||||
# File-based dashboard provider.
|
||||
# The JSON files are mounted from the `grafana-dashboards` ConfigMap (generated
|
||||
# without a name-suffix hash): `make dashboards` updates the ConfigMap in place
|
||||
# and Grafana reloads WITHOUT restarting the pod, in about 70s
|
||||
# (kubelet volume sync ~60s + updateIntervalSeconds below).
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: infra-observability
|
||||
orgId: 1
|
||||
folder: Infra Observability
|
||||
folderUid: infra-observability
|
||||
type: file
|
||||
disableDeletion: false
|
||||
allowUiUpdates: false
|
||||
updateIntervalSeconds: 10
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
foldersFromFilesStructure: false
|
||||
@@ -0,0 +1,82 @@
|
||||
# Declarative datasource provisioning.
|
||||
# The `uid` values are STABLE and are what dashboard JSON files reference:
|
||||
# "datasource": { "type": "prometheus", "uid": "prometheus" }
|
||||
apiVersion: 1
|
||||
|
||||
deleteDatasources:
|
||||
- name: Prometheus
|
||||
orgId: 1
|
||||
- name: Loki
|
||||
orgId: 1
|
||||
- name: Tempo
|
||||
orgId: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
orgId: 1
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
httpMethod: POST
|
||||
timeInterval: 30s
|
||||
manageAlerts: false
|
||||
exemplarTraceIdDestinations:
|
||||
- name: trace_id
|
||||
datasourceUid: tempo
|
||||
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
orgId: 1
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
jsonData:
|
||||
timeout: 60
|
||||
maxLines: 1000
|
||||
# Logs -> traces: a trace_id in the log line opens Tempo.
|
||||
derivedFields:
|
||||
- name: TraceID
|
||||
matcherType: regex
|
||||
matcherRegex: '"?trace[_-]?id"?[:=]"?([a-f0-9]{16,32})'
|
||||
url: '$${__value.raw}'
|
||||
datasourceUid: tempo
|
||||
urlDisplayLabel: View trace
|
||||
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
orgId: 1
|
||||
url: http://tempo:3200
|
||||
editable: false
|
||||
jsonData:
|
||||
httpMethod: GET
|
||||
streamingEnabled:
|
||||
search: true
|
||||
metrics: true
|
||||
# Service graph fed by the metrics_generator -> Prometheus.
|
||||
serviceMap:
|
||||
datasourceUid: prometheus
|
||||
nodeGraph:
|
||||
enabled: true
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
spanStartTimeShift: -5m
|
||||
spanEndTimeShift: 5m
|
||||
filterByTraceID: true
|
||||
filterBySpanID: false
|
||||
tags:
|
||||
- key: service.name
|
||||
value: app
|
||||
tracesToMetrics:
|
||||
datasourceUid: prometheus
|
||||
spanStartTimeShift: -5m
|
||||
spanEndTimeShift: 5m
|
||||
tags:
|
||||
- key: service.name
|
||||
value: service
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: grafana-data
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: grafana
|
||||
app.kubernetes.io/component: storage
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 5Gi
|
||||
@@ -0,0 +1,17 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: grafana
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: grafana
|
||||
app.kubernetes.io/component: ui
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: grafana
|
||||
ports:
|
||||
- name: http
|
||||
port: 3000
|
||||
targetPort: http
|
||||
@@ -0,0 +1,21 @@
|
||||
# Certificates issued by the `sttlab-local-ca` cert-manager ClusterIssuer
|
||||
# (already present on the cluster). No private key is ever versioned.
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: observability-tls
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
secretName: observability-tls
|
||||
issuerRef:
|
||||
name: sttlab-local-ca
|
||||
kind: ClusterIssuer
|
||||
commonName: grafana.sttlab.local
|
||||
dnsNames:
|
||||
- grafana.sttlab.local
|
||||
- prometheus.sttlab.local
|
||||
- alloy.sttlab.local
|
||||
- loki.sttlab.local
|
||||
- tempo.sttlab.local
|
||||
@@ -0,0 +1,88 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: grafana
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: grafana
|
||||
app.kubernetes.io/part-of: observability
|
||||
annotations:
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "20m"
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
|
||||
nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
|
||||
nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
|
||||
# Required for TraceQL streaming and Loki live tail.
|
||||
nginx.ingress.kubernetes.io/proxy-buffering: "off"
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
tls:
|
||||
- hosts:
|
||||
- grafana.sttlab.local
|
||||
secretName: observability-tls
|
||||
rules:
|
||||
- host: grafana.sttlab.local
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: grafana
|
||||
port:
|
||||
number: 3000
|
||||
---
|
||||
# Exposed for diagnostics (targets, TSDB status, expression browser).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: prometheus
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
app.kubernetes.io/part-of: observability
|
||||
annotations:
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
tls:
|
||||
- hosts:
|
||||
- prometheus.sttlab.local
|
||||
secretName: observability-tls
|
||||
rules:
|
||||
- host: prometheus.sttlab.local
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: prometheus
|
||||
port:
|
||||
number: 9090
|
||||
---
|
||||
# Alloy UI: state of the log collection pipeline components.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: alloy
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: alloy
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
tls:
|
||||
- hosts:
|
||||
- alloy.sttlab.local
|
||||
secretName: observability-tls
|
||||
rules:
|
||||
- host: alloy.sttlab.local
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: alloy
|
||||
port:
|
||||
number: 12345
|
||||
@@ -0,0 +1,7 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: observability
|
||||
|
||||
resources:
|
||||
- certificate.yaml
|
||||
- ingress.yaml
|
||||
@@ -0,0 +1,20 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: observability
|
||||
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- secret.yaml
|
||||
- prometheus
|
||||
- loki
|
||||
- tempo
|
||||
- alloy
|
||||
- grafana
|
||||
- grafana/dashboards
|
||||
- ingress
|
||||
|
||||
labels:
|
||||
- includeSelectors: false
|
||||
pairs:
|
||||
app.kubernetes.io/managed-by: kustomize
|
||||
app.kubernetes.io/part-of: observability
|
||||
@@ -0,0 +1,73 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: loki
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: loki
|
||||
app.kubernetes.io/component: logs
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: loki
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: loki
|
||||
app.kubernetes.io/name: loki
|
||||
app.kubernetes.io/component: logs
|
||||
app.kubernetes.io/part-of: observability
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "3100"
|
||||
spec:
|
||||
securityContext:
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
fsGroup: 10001
|
||||
containers:
|
||||
- name: loki
|
||||
image: grafana/loki:3.7.6
|
||||
args:
|
||||
- -config.file=/etc/loki/loki.yaml
|
||||
- -target=all
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 3100
|
||||
- name: grpc
|
||||
containerPort: 9096
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 1Gi
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /ready
|
||||
port: http
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /ready
|
||||
port: http
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 30
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /etc/loki
|
||||
- name: data
|
||||
mountPath: /loki
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: loki-config
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: loki-data
|
||||
@@ -0,0 +1,18 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: observability
|
||||
|
||||
resources:
|
||||
- pvc.yaml
|
||||
- deployment.yaml
|
||||
- service.yaml
|
||||
|
||||
configMapGenerator:
|
||||
- name: loki-config
|
||||
files:
|
||||
- loki.yaml
|
||||
|
||||
generatorOptions:
|
||||
labels:
|
||||
app.kubernetes.io/name: loki
|
||||
app.kubernetes.io/part-of: observability
|
||||
@@ -0,0 +1,85 @@
|
||||
# Loki 3.x in monolithic mode (-target=all), filesystem storage on a PVC.
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_address: 0.0.0.0
|
||||
http_listen_port: 3100
|
||||
grpc_listen_address: 0.0.0.0
|
||||
grpc_listen_port: 9096
|
||||
log_level: info
|
||||
grpc_server_max_concurrent_streams: 1000
|
||||
|
||||
common:
|
||||
instance_addr: 127.0.0.1
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: "2024-01-01"
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
storage_config:
|
||||
tsdb_shipper:
|
||||
active_index_directory: /loki/tsdb-index
|
||||
cache_location: /loki/tsdb-cache
|
||||
filesystem:
|
||||
directory: /loki/chunks
|
||||
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
compaction_interval: 10m
|
||||
retention_delete_delay: 2h
|
||||
|
||||
limits_config:
|
||||
retention_period: 168h
|
||||
max_query_series: 5000
|
||||
ingestion_rate_mb: 16
|
||||
ingestion_burst_size_mb: 32
|
||||
reject_old_samples: true
|
||||
reject_old_samples_max_age: 168h
|
||||
volume_enabled: true
|
||||
metric_aggregation_enabled: true
|
||||
allow_structured_metadata: true
|
||||
|
||||
pattern_ingester:
|
||||
enabled: true
|
||||
metric_aggregation:
|
||||
loki_address: localhost:3100
|
||||
|
||||
query_range:
|
||||
align_queries_with_step: true
|
||||
cache_results: true
|
||||
results_cache:
|
||||
cache:
|
||||
embedded_cache:
|
||||
enabled: true
|
||||
max_size_mb: 100
|
||||
|
||||
frontend:
|
||||
encoding: protobuf
|
||||
|
||||
ruler:
|
||||
storage:
|
||||
type: local
|
||||
local:
|
||||
directory: /loki/rules
|
||||
rule_path: /loki/rules-tmp
|
||||
enable_api: true
|
||||
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: loki-data
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: loki
|
||||
app.kubernetes.io/component: storage
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
@@ -0,0 +1,20 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: loki
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: loki
|
||||
app.kubernetes.io/component: logs
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: loki
|
||||
ports:
|
||||
- name: http
|
||||
port: 3100
|
||||
targetPort: http
|
||||
- name: grpc
|
||||
port: 9096
|
||||
targetPort: grpc
|
||||
@@ -0,0 +1,6 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: observability
|
||||
labels:
|
||||
app.kubernetes.io/part-of: observability
|
||||
@@ -0,0 +1,86 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: prometheus
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
app.kubernetes.io/component: metrics
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
# ReadWriteOnce PVC: the old pod must release the volume first.
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: prometheus
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: prometheus
|
||||
app.kubernetes.io/name: prometheus
|
||||
app.kubernetes.io/component: metrics
|
||||
app.kubernetes.io/part-of: observability
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9090"
|
||||
spec:
|
||||
serviceAccountName: prometheus
|
||||
securityContext:
|
||||
runAsUser: 65534
|
||||
runAsGroup: 65534
|
||||
fsGroup: 65534
|
||||
containers:
|
||||
- name: prometheus
|
||||
image: prom/prometheus:v3.14.0
|
||||
args:
|
||||
- --config.file=/etc/prometheus/prometheus.yaml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=15d
|
||||
- --storage.tsdb.retention.size=8GB
|
||||
- --web.enable-lifecycle
|
||||
# Receives remote_write from Tempo's metrics_generator (span metrics + service graphs).
|
||||
- --web.enable-remote-write-receiver
|
||||
- --web.external-url=https://prometheus.sttlab.local
|
||||
- --web.route-prefix=/
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 9090
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 2Gi
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /-/ready
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /-/healthy
|
||||
port: http
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /etc/prometheus
|
||||
- name: data
|
||||
mountPath: /prometheus
|
||||
- name: librechat-metrics
|
||||
mountPath: /etc/prometheus/secrets/librechat
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: prometheus-config
|
||||
- name: librechat-metrics
|
||||
secret:
|
||||
secretName: librechat-metrics
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: prometheus-data
|
||||
@@ -0,0 +1,20 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: observability
|
||||
|
||||
resources:
|
||||
- secret.yaml
|
||||
- rbac.yaml
|
||||
- pvc.yaml
|
||||
- deployment.yaml
|
||||
- service.yaml
|
||||
|
||||
configMapGenerator:
|
||||
- name: prometheus-config
|
||||
files:
|
||||
- prometheus.yaml
|
||||
|
||||
generatorOptions:
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
app.kubernetes.io/part-of: observability
|
||||
@@ -0,0 +1,177 @@
|
||||
global:
|
||||
scrape_interval: 30s
|
||||
evaluation_interval: 30s
|
||||
external_labels:
|
||||
cluster: orbstack
|
||||
source: prometheus
|
||||
|
||||
scrape_configs:
|
||||
# --- Control plane -------------------------------------------------------
|
||||
- job_name: kubernetes-apiservers
|
||||
kubernetes_sd_configs:
|
||||
- role: endpoints
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
insecure_skip_verify: true
|
||||
authorization:
|
||||
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
relabel_configs:
|
||||
- source_labels:
|
||||
[__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
|
||||
action: keep
|
||||
regex: default;kubernetes;https
|
||||
|
||||
# --- Kubelet -------------------------------------------------------------
|
||||
- job_name: kubernetes-nodes
|
||||
kubernetes_sd_configs:
|
||||
- role: node
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
insecure_skip_verify: true
|
||||
authorization:
|
||||
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
relabel_configs:
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_node_label_(.+)
|
||||
- target_label: __address__
|
||||
replacement: kubernetes.default.svc:443
|
||||
- source_labels: [__meta_kubernetes_node_name]
|
||||
regex: (.+)
|
||||
target_label: __metrics_path__
|
||||
replacement: /api/v1/nodes/$1/proxy/metrics
|
||||
|
||||
# --- Container resource metrics -----------------------------------------
|
||||
# /metrics/resource and NOT /metrics/cadvisor: on this cluster (OrbStack) the
|
||||
# kubelet's cadvisor endpoint only exposes `machine_*` series and no
|
||||
# `container_*` at all. /metrics/resource -- the source metrics-server reads --
|
||||
# provides container_cpu_usage_seconds_total and
|
||||
# container_memory_working_set_bytes with namespace/pod/container labels.
|
||||
- job_name: kubernetes-kubelet-resource
|
||||
kubernetes_sd_configs:
|
||||
- role: node
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
insecure_skip_verify: true
|
||||
authorization:
|
||||
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
relabel_configs:
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_node_label_(.+)
|
||||
- target_label: __address__
|
||||
replacement: kubernetes.default.svc:443
|
||||
- source_labels: [__meta_kubernetes_node_name]
|
||||
regex: (.+)
|
||||
target_label: __metrics_path__
|
||||
replacement: /api/v1/nodes/$1/proxy/metrics/resource
|
||||
- source_labels: [__meta_kubernetes_node_name]
|
||||
target_label: node
|
||||
|
||||
# --- Services annotated with prometheus.io/scrape -----------------------
|
||||
- job_name: kubernetes-service-endpoints
|
||||
kubernetes_sd_configs:
|
||||
- role: endpoints
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
|
||||
action: keep
|
||||
regex: "true"
|
||||
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
|
||||
action: replace
|
||||
target_label: __scheme__
|
||||
regex: (https?)
|
||||
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
|
||||
action: replace
|
||||
target_label: __metrics_path__
|
||||
regex: (.+)
|
||||
- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
|
||||
action: replace
|
||||
target_label: __address__
|
||||
regex: ([^:]+)(?::\d+)?;(\d+)
|
||||
replacement: $1:$2
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_service_label_(.+)
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
target_label: namespace
|
||||
- source_labels: [__meta_kubernetes_service_name]
|
||||
target_label: service
|
||||
- source_labels: [__meta_kubernetes_pod_name]
|
||||
target_label: pod
|
||||
- source_labels: [__meta_kubernetes_pod_node_name]
|
||||
target_label: node
|
||||
|
||||
# --- Pods annotated with prometheus.io/scrape ---------------------------
|
||||
- job_name: kubernetes-pods
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
|
||||
action: keep
|
||||
regex: "true"
|
||||
# A pod yields one target per container: without this filter the init
|
||||
# containers duplicate the real target.
|
||||
- source_labels: [__meta_kubernetes_pod_container_init]
|
||||
action: drop
|
||||
regex: "true"
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
|
||||
action: replace
|
||||
target_label: __scheme__
|
||||
regex: (https?)
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
|
||||
action: replace
|
||||
target_label: __metrics_path__
|
||||
regex: (.+)
|
||||
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
|
||||
action: replace
|
||||
target_label: __address__
|
||||
regex: ([^:]+)(?::\d+)?;(\d+)
|
||||
replacement: $1:$2
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_pod_label_(.+)
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
target_label: namespace
|
||||
- source_labels: [__meta_kubernetes_pod_name]
|
||||
target_label: pod
|
||||
- source_labels: [__meta_kubernetes_pod_node_name]
|
||||
target_label: node
|
||||
- source_labels: [__meta_kubernetes_pod_container_name]
|
||||
target_label: container
|
||||
- source_labels: [__meta_kubernetes_pod_phase]
|
||||
action: drop
|
||||
regex: (Succeeded|Failed|Pending)
|
||||
|
||||
# --- LibreChat: /metrics behind a Bearer token --------------------------
|
||||
# Annotation-based discovery cannot carry credentials, so LibreChat gets its
|
||||
# own job. The token comes from the `librechat-metrics` secret mounted into
|
||||
# the Prometheus pod and must match METRICS_SECRET on the LibreChat side.
|
||||
- job_name: librechat
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
namespaces:
|
||||
names: [librechat]
|
||||
authorization:
|
||||
type: Bearer
|
||||
credentials_file: /etc/prometheus/secrets/librechat/token
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape_with_auth]
|
||||
action: keep
|
||||
regex: "true"
|
||||
# A pod yields one target per container: without this filter the init
|
||||
# containers duplicate the real target.
|
||||
- source_labels: [__meta_kubernetes_pod_container_init]
|
||||
action: drop
|
||||
regex: "true"
|
||||
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
|
||||
action: replace
|
||||
target_label: __address__
|
||||
regex: ([^:]+)(?::\d+)?;(\d+)
|
||||
replacement: $1:$2
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
target_label: namespace
|
||||
- source_labels: [__meta_kubernetes_pod_name]
|
||||
target_label: pod
|
||||
- source_labels: [__meta_kubernetes_pod_container_name]
|
||||
target_label: container
|
||||
- target_label: service
|
||||
replacement: librechat
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: prometheus-data
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
app.kubernetes.io/component: storage
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
@@ -0,0 +1,51 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: prometheus
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
app.kubernetes.io/part-of: observability
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: observability-prometheus
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
app.kubernetes.io/part-of: observability
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources:
|
||||
- nodes
|
||||
- nodes/metrics
|
||||
- nodes/proxy
|
||||
- services
|
||||
- endpoints
|
||||
- pods
|
||||
- namespaces
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["discovery.k8s.io"]
|
||||
resources: ["endpointslices"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["networking.k8s.io"]
|
||||
resources: ["ingresses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- nonResourceURLs: ["/metrics", "/metrics/cadvisor"]
|
||||
verbs: ["get"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: observability-prometheus
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
app.kubernetes.io/part-of: observability
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: observability-prometheus
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: prometheus
|
||||
namespace: observability
|
||||
@@ -0,0 +1,11 @@
|
||||
# Template. The token must be IDENTICAL to the METRICS_SECRET key of the
|
||||
# `librechat-secrets` secret in the librechat namespace (infra-librechat repo):
|
||||
# it is the Bearer token Prometheus uses to scrape LibreChat's /metrics.
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: librechat-metrics
|
||||
namespace: observability
|
||||
type: Opaque
|
||||
stringData:
|
||||
token: <CHANGE_ME>
|
||||
@@ -0,0 +1,17 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: prometheus
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
app.kubernetes.io/component: metrics
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: prometheus
|
||||
ports:
|
||||
- name: http
|
||||
port: 9090
|
||||
targetPort: http
|
||||
@@ -0,0 +1,11 @@
|
||||
# Template: copy to secret.yaml (gitignored) and fill in the values.
|
||||
# make secret
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: observability-secrets
|
||||
namespace: observability
|
||||
type: Opaque
|
||||
stringData:
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD: <CHANGE_ME>
|
||||
@@ -0,0 +1,79 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: tempo
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: tempo
|
||||
app.kubernetes.io/component: traces
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: tempo
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: tempo
|
||||
app.kubernetes.io/name: tempo
|
||||
app.kubernetes.io/component: traces
|
||||
app.kubernetes.io/part-of: observability
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "3200"
|
||||
spec:
|
||||
securityContext:
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
fsGroup: 10001
|
||||
containers:
|
||||
- name: tempo
|
||||
image: grafana/tempo:3.0.3
|
||||
args:
|
||||
- -config.file=/etc/tempo/tempo.yaml
|
||||
- -target=all
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 3200
|
||||
- name: grpc
|
||||
containerPort: 9095
|
||||
- name: otlp-grpc
|
||||
containerPort: 4317
|
||||
- name: otlp-http
|
||||
containerPort: 4318
|
||||
- name: zipkin
|
||||
containerPort: 9411
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 2Gi
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /ready
|
||||
port: http
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /ready
|
||||
port: http
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 30
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /etc/tempo
|
||||
- name: data
|
||||
mountPath: /var/tempo
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: tempo-config
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: tempo-data
|
||||
@@ -0,0 +1,18 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: observability
|
||||
|
||||
resources:
|
||||
- pvc.yaml
|
||||
- deployment.yaml
|
||||
- service.yaml
|
||||
|
||||
configMapGenerator:
|
||||
- name: tempo-config
|
||||
files:
|
||||
- tempo.yaml
|
||||
|
||||
generatorOptions:
|
||||
labels:
|
||||
app.kubernetes.io/name: tempo
|
||||
app.kubernetes.io/part-of: observability
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: tempo-data
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: tempo
|
||||
app.kubernetes.io/component: storage
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
@@ -0,0 +1,29 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: tempo
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: tempo
|
||||
app.kubernetes.io/component: traces
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: tempo
|
||||
ports:
|
||||
- name: http
|
||||
port: 3200
|
||||
targetPort: http
|
||||
- name: grpc
|
||||
port: 9095
|
||||
targetPort: grpc
|
||||
- name: otlp-grpc
|
||||
port: 4317
|
||||
targetPort: otlp-grpc
|
||||
- name: otlp-http
|
||||
port: 4318
|
||||
targetPort: otlp-http
|
||||
- name: zipkin
|
||||
port: 9411
|
||||
targetPort: zipkin
|
||||
@@ -0,0 +1,64 @@
|
||||
# Tempo 3.x in monolithic mode (-target=all), local storage on a PVC.
|
||||
stream_over_http_enabled: true
|
||||
multitenancy_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_address: 0.0.0.0
|
||||
http_listen_port: 3200
|
||||
grpc_listen_address: 0.0.0.0
|
||||
grpc_listen_port: 9095
|
||||
log_level: info
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
zipkin:
|
||||
endpoint: 0.0.0.0:9411
|
||||
|
||||
# Generates span metrics and service graphs, pushed to Prometheus through its
|
||||
# remote-write receiver (--web.enable-remote-write-receiver).
|
||||
metrics_generator:
|
||||
registry:
|
||||
external_labels:
|
||||
source: tempo
|
||||
cluster: orbstack
|
||||
storage:
|
||||
path: /var/tempo/generator/wal
|
||||
remote_write:
|
||||
- url: http://prometheus:9090/api/v1/write
|
||||
send_exemplars: true
|
||||
|
||||
query_frontend:
|
||||
search:
|
||||
duration_slo: 5s
|
||||
throughput_bytes_slo: 1073741824
|
||||
trace_by_id:
|
||||
duration_slo: 5s
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
|
||||
overrides:
|
||||
defaults:
|
||||
metrics_generator:
|
||||
processors:
|
||||
- service-graphs
|
||||
- span-metrics
|
||||
generate_native_histograms: both
|
||||
# Tempo 3.x: retention is no longer a top-level `compactor:` block; it is
|
||||
# driven from here (backend_scheduler/backend_worker otherwise).
|
||||
compaction:
|
||||
block_retention: 168h
|
||||
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
Reference in New Issue
Block a user