Declarative Grafana / Prometheus / Loki / Tempo observability stack
Deploys the whole stack into the observability namespace of the local cluster with Kustomize alone — no helm install, no dashboard created by hand in the UI. The Makefile carries every command: deploy, dashboards, diagnostics, PVC flush, teardown. Dashboards are files, never UI objects. Each carries a stable uid, which is what keeps URLs, cross-links and alerts alive across redeployments, and references datasources by stable uid rather than by name. The dashboard ConfigMap is generated without a name-suffix hash so applying it updates it in place and Grafana reloads without a restart, while the provisioning ConfigMaps keep their hash precisely so that a change there does force one. Prometheus scrapes the kubelet /metrics/resource endpoint rather than /metrics/cadvisor: on this cluster cadvisor exposes machine_* series only and no container_* at all, which would leave every CPU and memory panel empty. Alloy ships every pod's logs to Loki through the Kubernetes API, so no /var/log mount is needed. Tempo's metrics_generator feeds span metrics and the service graph back into Prometheus, and the datasources are wired so a log line's trace_id opens the matching trace. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Stéphane Tailland
@@ -0,0 +1,5 @@
|
|||||||
|
.DS_Store
|
||||||
|
*.tmp
|
||||||
|
# Real secrets: only secret.example.yaml is versioned
|
||||||
|
k8s/secret.yaml
|
||||||
|
k8s/prometheus/secret.yaml
|
||||||
@@ -0,0 +1,356 @@
|
|||||||
|
# =============================================================================
|
||||||
|
# infra-observability -- Grafana / Prometheus / Loki / Tempo stack (+ Alloy)
|
||||||
|
# Fully declarative: kustomize + kubectl. No imperative `helm install`.
|
||||||
|
#
|
||||||
|
# make -> help
|
||||||
|
# make deploy -> deploy everything
|
||||||
|
# make diag -> full diagnostics
|
||||||
|
# =============================================================================
|
||||||
|
|
||||||
|
SHELL := /bin/bash
|
||||||
|
.DEFAULT_GOAL := help
|
||||||
|
# Every target prints its own output, so commands are not echoed.
|
||||||
|
.SILENT:
|
||||||
|
|
||||||
|
NS ?= observability
|
||||||
|
CTX ?= orbstack
|
||||||
|
KUBECTL := kubectl --context=$(CTX)
|
||||||
|
K8S := k8s
|
||||||
|
DASHBOARDS := $(K8S)/grafana/dashboards
|
||||||
|
INGRESS_IP ?= 192.168.139.101
|
||||||
|
HOSTS := grafana.sttlab.local prometheus.sttlab.local alloy.sttlab.local
|
||||||
|
|
||||||
|
# Components backed by a PVC (this is also the flush order)
|
||||||
|
STATEFUL := prometheus loki tempo grafana
|
||||||
|
# Restartable workloads
|
||||||
|
WORKLOADS := deployment/prometheus deployment/loki deployment/tempo deployment/grafana daemonset/alloy
|
||||||
|
|
||||||
|
C_OK := \033[0;32m
|
||||||
|
C_WARN := \033[0;33m
|
||||||
|
C_ERR := \033[0;31m
|
||||||
|
C_HEAD := \033[1;36m
|
||||||
|
C_OFF := \033[0m
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
|
# Help
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
|
.PHONY: help
|
||||||
|
help: ## Show this help
|
||||||
|
printf "$(C_HEAD)infra-observability$(C_OFF) -- namespace $(C_OK)$(NS)$(C_OFF) / context $(C_OK)$(CTX)$(C_OFF)\n\n"
|
||||||
|
awk 'BEGIN {FS = ":.*?## "} \
|
||||||
|
/^[a-zA-Z0-9_-]+:.*?## / {printf " $(C_OK)%-24s$(C_OFF) %s\n", $$1, $$2} \
|
||||||
|
/^##@/ {printf "\n$(C_HEAD)%s$(C_OFF)\n", substr($$0, 5)}' $(MAKEFILE_LIST)
|
||||||
|
printf "\n"
|
||||||
|
|
||||||
|
##@ Deployment
|
||||||
|
|
||||||
|
.PHONY: deploy
|
||||||
|
deploy: check-context secret ## Deploy the whole stack (runtimes + dashboards)
|
||||||
|
$(KUBECTL) apply -k $(K8S)
|
||||||
|
$(MAKE) --no-print-directory wait
|
||||||
|
|
||||||
|
.PHONY: deploy-runtime
|
||||||
|
deploy-runtime: check-context secret ## Deploy runtimes only (no dashboards)
|
||||||
|
$(KUBECTL) apply -f $(K8S)/namespace.yaml -f $(K8S)/secret.yaml
|
||||||
|
$(KUBECTL) apply -k $(K8S)/prometheus
|
||||||
|
$(KUBECTL) apply -k $(K8S)/loki
|
||||||
|
$(KUBECTL) apply -k $(K8S)/tempo
|
||||||
|
$(KUBECTL) apply -k $(K8S)/alloy
|
||||||
|
$(KUBECTL) apply -k $(K8S)/grafana
|
||||||
|
$(KUBECTL) apply -k $(K8S)/ingress
|
||||||
|
$(MAKE) --no-print-directory wait
|
||||||
|
|
||||||
|
.PHONY: prometheus loki tempo grafana alloy ingress
|
||||||
|
prometheus: check-context ## Deploy/update Prometheus only
|
||||||
|
$(KUBECTL) apply -k $(K8S)/prometheus
|
||||||
|
loki: check-context ## Deploy/update Loki only
|
||||||
|
$(KUBECTL) apply -k $(K8S)/loki
|
||||||
|
tempo: check-context ## Deploy/update Tempo only
|
||||||
|
$(KUBECTL) apply -k $(K8S)/tempo
|
||||||
|
grafana: check-context ## Deploy/update the Grafana runtime only (datasources included)
|
||||||
|
$(KUBECTL) apply -k $(K8S)/grafana
|
||||||
|
alloy: check-context ## Deploy/update Alloy only (log collection)
|
||||||
|
$(KUBECTL) apply -k $(K8S)/alloy
|
||||||
|
ingress: check-context ## Deploy/update the Ingresses and the certificate
|
||||||
|
$(KUBECTL) apply -k $(K8S)/ingress
|
||||||
|
|
||||||
|
.PHONY: secret
|
||||||
|
secret: ## Generate k8s/secret.yaml (random admin password) if absent
|
||||||
|
if [ ! -f $(K8S)/secret.yaml ]; then \
|
||||||
|
pw=$$(LC_ALL=C tr -dc 'A-Za-z0-9' </dev/urandom | head -c 20); \
|
||||||
|
sed -e '1s|.*|# Generated by `make secret` -- NOT versioned (see .gitignore).|' \
|
||||||
|
-e "s|<CHANGE_ME>|$$pw|" $(K8S)/secret.example.yaml > $(K8S)/secret.yaml; \
|
||||||
|
printf "$(C_OK)secret.yaml generated$(C_OFF) -- admin password: $$pw\n"; \
|
||||||
|
else \
|
||||||
|
printf "secret.yaml already present (use make grafana-password to read it)\n"; \
|
||||||
|
fi
|
||||||
|
|
||||||
|
##@ Dashboards (declarative, stable UIDs)
|
||||||
|
|
||||||
|
.PHONY: dashboards
|
||||||
|
dashboards: check-context dashboards-check ## Apply dashboards only (no Grafana restart)
|
||||||
|
$(KUBECTL) apply -k $(DASHBOARDS)
|
||||||
|
printf "$(C_OK)grafana-dashboards ConfigMap updated$(C_OFF) -- visible in Grafana within ~70s\n"
|
||||||
|
printf " (kubelet volume sync ~60s + provisioner poll 10s, no restart).\n"
|
||||||
|
printf " For immediate effect: $(C_WARN)make restart-grafana$(C_OFF)\n"
|
||||||
|
|
||||||
|
.PHONY: dashboards-sync
|
||||||
|
dashboards-sync: ## Regenerate the .json list in dashboards/kustomization.yaml
|
||||||
|
cd $(DASHBOARDS) && python3 -c "import glob,re,pathlib; \
|
||||||
|
p=pathlib.Path('kustomization.yaml'); s=p.read_text(); \
|
||||||
|
files=sorted(glob.glob('*.json')); \
|
||||||
|
block=''.join(' - %s\n' % f for f in files); \
|
||||||
|
s=re.sub(r'(# >>> dashboards.*?>>>\n).*?( # <<< dashboards <<<)', lambda m: m.group(1)+block+m.group(2), s, flags=re.S); \
|
||||||
|
p.write_text(s); print('%d dashboard(s) referenced' % len(files))"
|
||||||
|
$(MAKE) --no-print-directory dashboards-check
|
||||||
|
|
||||||
|
.PHONY: dashboards-check
|
||||||
|
dashboards-check: ## Validate the JSON files and UID uniqueness/stability
|
||||||
|
fail=0; \
|
||||||
|
for f in $(DASHBOARDS)/*.json; do \
|
||||||
|
jq -e . "$$f" >/dev/null 2>&1 || { printf "$(C_ERR)invalid JSON$(C_OFF): %s\n" "$$f"; fail=1; continue; }; \
|
||||||
|
uid=$$(jq -r '.uid // empty' "$$f"); \
|
||||||
|
[ -n "$$uid" ] || { printf "$(C_ERR)missing uid$(C_OFF): %s\n" "$$f"; fail=1; }; \
|
||||||
|
base=$$(basename "$$f" .json); \
|
||||||
|
[ "$$uid" = "$$base" ] || printf "$(C_WARN)filename != uid$(C_OFF): %s (uid=%s)\n" "$$f" "$$uid"; \
|
||||||
|
[ "$$(jq -r '.id // "null"' "$$f")" = "null" ] || printf "$(C_WARN)non-null id field$(C_OFF): %s (remove it)\n" "$$f"; \
|
||||||
|
grep -q -- "- $$base.json" $(DASHBOARDS)/kustomization.yaml || { printf "$(C_ERR)not referenced in kustomization.yaml$(C_OFF): %s (run make dashboards-sync)\n" "$$f"; fail=1; }; \
|
||||||
|
done; \
|
||||||
|
dup=$$(jq -r '.uid' $(DASHBOARDS)/*.json | sort | uniq -d); \
|
||||||
|
[ -z "$$dup" ] || { printf "$(C_ERR)duplicate UIDs$(C_OFF): %s\n" "$$dup"; fail=1; }; \
|
||||||
|
[ $$fail -eq 0 ] && printf "$(C_OK)dashboards OK$(C_OFF) (%s)\n" "$$(ls $(DASHBOARDS)/*.json | wc -l | tr -d ' ')"; \
|
||||||
|
exit $$fail
|
||||||
|
|
||||||
|
.PHONY: dashboards-list
|
||||||
|
dashboards-list: ## List the dashboards actually loaded by Grafana
|
||||||
|
auth="$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_USER}' | base64 -d):$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_PASSWORD}' | base64 -d)"; \
|
||||||
|
$(KUBECTL) -n $(NS) exec deployment/grafana -- curl -s -u "$$auth" 'http://localhost:3000/api/search?type=dash-db' \
|
||||||
|
| jq -r '.[] | "\(.uid)\t\(.folderTitle // "General")\t\(.title)"'
|
||||||
|
|
||||||
|
##@ Diagnostics
|
||||||
|
|
||||||
|
.PHONY: status
|
||||||
|
status: ## Overview (pods, services, PVCs, ingresses)
|
||||||
|
$(KUBECTL) -n $(NS) get pods -o wide
|
||||||
|
echo
|
||||||
|
$(KUBECTL) -n $(NS) get svc,pvc,ingress
|
||||||
|
|
||||||
|
.PHONY: diag
|
||||||
|
diag: ## Full diagnostics: state, PVCs, endpoints, certificate, events
|
||||||
|
printf "$(C_HEAD)== Pods ==$(C_OFF)\n"; $(KUBECTL) -n $(NS) get pods -o wide
|
||||||
|
printf "\n$(C_HEAD)== PVCs ==$(C_OFF)\n"; $(KUBECTL) -n $(NS) get pvc
|
||||||
|
printf "\n$(C_HEAD)== Endpoints ==$(C_OFF)\n"; $(KUBECTL) -n $(NS) get endpoints
|
||||||
|
printf "\n$(C_HEAD)== Ingresses ==$(C_OFF)\n"; $(KUBECTL) -n $(NS) get ingress
|
||||||
|
printf "\n$(C_HEAD)== Certificate ==$(C_OFF)\n";$(KUBECTL) -n $(NS) get certificate 2>/dev/null || echo "(cert-manager absent)"
|
||||||
|
printf "\n$(C_HEAD)== ConfigMaps ==$(C_OFF)\n";$(KUBECTL) -n $(NS) get cm
|
||||||
|
printf "\n$(C_HEAD)== Containers not ready ==$(C_OFF)\n"
|
||||||
|
$(KUBECTL) -n $(NS) get pods -o json | jq -r '.items[] | select(.status.containerStatuses != null) | .status.containerStatuses[] | select(.ready == false) | "\(.name)\trestarts=\(.restartCount)\t\(.state | keys[0]): \(.state[keys[0]].reason // "-")"' || true
|
||||||
|
printf "\n$(C_HEAD)== LibreChat scrape token ==$(C_OFF)\n"
|
||||||
|
$(MAKE) --no-print-directory check-metrics-token || true
|
||||||
|
printf "\n$(C_HEAD)== Last 15 events ==$(C_OFF)\n"
|
||||||
|
$(KUBECTL) -n $(NS) get events --sort-by=.lastTimestamp | tail -15
|
||||||
|
|
||||||
|
.PHONY: smoke
|
||||||
|
smoke: ## End-to-end checks: component health, datasources, dashboards, targets
|
||||||
|
# The Loki and Tempo images ship neither curl nor wget, so probes run from the
|
||||||
|
# Grafana pod, which has curl and sees the same Services as the datasources.
|
||||||
|
probe() { $(KUBECTL) -n $(NS) exec deployment/grafana -- curl -s --max-time 10 "$$1"; }; \
|
||||||
|
auth="$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_USER}' | base64 -d):$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_PASSWORD}' | base64 -d)"; \
|
||||||
|
printf "$(C_HEAD)== Component health ==$(C_OFF)\n"; \
|
||||||
|
printf " %-12s %s\n" "prometheus" "$$(probe http://prometheus:9090/-/ready)"; \
|
||||||
|
printf " %-12s %s\n" "loki" "$$(probe http://loki:3100/ready)"; \
|
||||||
|
printf " %-12s %s\n" "tempo" "$$(probe http://tempo:3200/ready)"; \
|
||||||
|
printf " %-12s %s\n" "grafana" "$$(probe http://localhost:3000/api/health | jq -c .)"; \
|
||||||
|
printf " %-12s %s\n" "alloy" "$$(probe http://alloy:12345/-/ready)"; \
|
||||||
|
printf "\n$(C_HEAD)== Grafana datasources ==$(C_OFF)\n"; \
|
||||||
|
for uid in prometheus loki tempo; do \
|
||||||
|
r=$$($(KUBECTL) -n $(NS) exec deployment/grafana -- curl -s -u "$$auth" -X POST "http://localhost:3000/api/datasources/uid/$$uid/health" | jq -r '"\(.status) \(.message)"'); \
|
||||||
|
printf " %-12s %s\n" "$$uid" "$$r"; \
|
||||||
|
done; \
|
||||||
|
printf "\n$(C_HEAD)== Provisioned dashboards ==$(C_OFF)\n"; \
|
||||||
|
$(KUBECTL) -n $(NS) exec deployment/grafana -- curl -s -u "$$auth" 'http://localhost:3000/api/search?type=dash-db' \
|
||||||
|
| jq -r '.[] | " \(.uid) \(.folderTitle // "General")/\(.title)"'; \
|
||||||
|
printf "\n$(C_HEAD)== Namespaces seen by Loki ==$(C_OFF)\n"; \
|
||||||
|
probe 'http://loki:3100/loki/api/v1/label/namespace/values' | jq -r '" " + (.data // [] | join(", "))'; \
|
||||||
|
printf "\n$(C_HEAD)== Prometheus targets ==$(C_OFF)\n"; \
|
||||||
|
probe http://prometheus:9090/api/v1/targets | jq -r '.data.activeTargets | group_by(.health)[] | " \(.[0].health)\t\(length)"'
|
||||||
|
|
||||||
|
.PHONY: targets
|
||||||
|
targets: ## List Prometheus scrape targets and their state
|
||||||
|
$(KUBECTL) -n $(NS) exec deployment/prometheus -- wget -qO- http://localhost:9090/api/v1/targets \
|
||||||
|
| jq -r '.data.activeTargets[] | "\(.health)\t\(.labels.job)\t\(.scrapeUrl)\t\(.lastError // "")"' | sort
|
||||||
|
|
||||||
|
.PHONY: check-metrics-token
|
||||||
|
check-metrics-token: ## Check the LibreChat token matches across both namespaces
|
||||||
|
a=$$($(KUBECTL) -n $(NS) get secret librechat-metrics -o jsonpath='{.data.token}' 2>/dev/null | base64 -d); \
|
||||||
|
b=$$($(KUBECTL) -n librechat get secret librechat-secrets -o jsonpath='{.data.METRICS_SECRET}' 2>/dev/null | base64 -d); \
|
||||||
|
if [ -z "$$a" ] || [ -z "$$b" ]; then \
|
||||||
|
printf "$(C_WARN)token not found$(C_OFF) (observability/librechat-metrics=%s, librechat/librechat-secrets=%s)\n" "$${a:+present}$${a:-absent}" "$${b:+present}$${b:-absent}"; \
|
||||||
|
elif [ "$$a" = "$$b" ]; then \
|
||||||
|
printf "$(C_OK)LibreChat token matches$(C_OFF) across observability and librechat\n"; \
|
||||||
|
else \
|
||||||
|
printf "$(C_ERR)DRIFT$(C_OFF): the two Secrets hold different values.\n"; \
|
||||||
|
printf " The librechat scrape will fall back to 401. Realign both copies.\n"; \
|
||||||
|
exit 1; \
|
||||||
|
fi
|
||||||
|
|
||||||
|
.PHONY: logs-prometheus logs-loki logs-tempo logs-grafana logs-alloy
|
||||||
|
logs-prometheus: ## Follow Prometheus logs
|
||||||
|
$(KUBECTL) -n $(NS) logs -f deployment/prometheus --tail=100
|
||||||
|
logs-loki: ## Follow Loki logs
|
||||||
|
$(KUBECTL) -n $(NS) logs -f deployment/loki --tail=100
|
||||||
|
logs-tempo: ## Follow Tempo logs
|
||||||
|
$(KUBECTL) -n $(NS) logs -f deployment/tempo --tail=100
|
||||||
|
logs-grafana: ## Follow Grafana logs
|
||||||
|
$(KUBECTL) -n $(NS) logs -f deployment/grafana --tail=100
|
||||||
|
logs-alloy: ## Follow Alloy logs
|
||||||
|
$(KUBECTL) -n $(NS) logs -f daemonset/alloy --tail=100
|
||||||
|
|
||||||
|
.PHONY: events
|
||||||
|
events: ## Namespace events, oldest to newest
|
||||||
|
$(KUBECTL) -n $(NS) get events --sort-by=.lastTimestamp
|
||||||
|
|
||||||
|
.PHONY: top
|
||||||
|
top: ## Pod CPU/memory usage
|
||||||
|
$(KUBECTL) -n $(NS) top pods 2>/dev/null || echo "metrics-server unavailable"
|
||||||
|
|
||||||
|
.PHONY: urls
|
||||||
|
urls: ## Show access URLs
|
||||||
|
printf " Grafana https://grafana.sttlab.local\n"
|
||||||
|
printf " Prometheus https://prometheus.sttlab.local\n"
|
||||||
|
printf " Alloy https://alloy.sttlab.local\n"
|
||||||
|
printf "\n In-cluster: http://prometheus:9090 - http://loki:3100 - http://tempo:3200\n"
|
||||||
|
printf " OTLP : tempo.$(NS).svc.cluster.local:4317 (gRPC) / :4318 (HTTP)\n"
|
||||||
|
|
||||||
|
.PHONY: hosts
|
||||||
|
hosts: ## Show the required /etc/hosts entries
|
||||||
|
printf "# Add to /etc/hosts:\n"
|
||||||
|
for h in $(HOSTS); do printf "$(INGRESS_IP)\t%s\n" "$$h"; done
|
||||||
|
|
||||||
|
.PHONY: grafana-password
|
||||||
|
grafana-password: ## Show the Grafana admin credentials
|
||||||
|
printf "user : %s\n" "$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_USER}' | base64 -d)"
|
||||||
|
printf "password : %s\n" "$$($(KUBECTL) -n $(NS) get secret observability-secrets -o jsonpath='{.data.GF_SECURITY_ADMIN_PASSWORD}' | base64 -d)"
|
||||||
|
|
||||||
|
.PHONY: pf-grafana pf-prometheus pf-loki pf-tempo pf-alloy
|
||||||
|
pf-grafana: ## Port-forward Grafana on :3000
|
||||||
|
$(KUBECTL) -n $(NS) port-forward svc/grafana 3000:3000
|
||||||
|
pf-prometheus: ## Port-forward Prometheus on :9090
|
||||||
|
$(KUBECTL) -n $(NS) port-forward svc/prometheus 9090:9090
|
||||||
|
pf-loki: ## Port-forward Loki on :3100
|
||||||
|
$(KUBECTL) -n $(NS) port-forward svc/loki 3100:3100
|
||||||
|
pf-tempo: ## Port-forward Tempo on :3200
|
||||||
|
$(KUBECTL) -n $(NS) port-forward svc/tempo 3200:3200
|
||||||
|
pf-alloy: ## Port-forward the Alloy UI on :12345
|
||||||
|
$(KUBECTL) -n $(NS) port-forward svc/alloy 12345:12345
|
||||||
|
|
||||||
|
##@ Operations
|
||||||
|
|
||||||
|
.PHONY: wait
|
||||||
|
wait: ## Wait for every workload to become ready
|
||||||
|
for w in $(WORKLOADS); do \
|
||||||
|
printf " %-24s " "$$w"; \
|
||||||
|
$(KUBECTL) -n $(NS) rollout status $$w --timeout=300s 2>&1 | tail -1; \
|
||||||
|
done
|
||||||
|
|
||||||
|
.PHONY: restart
|
||||||
|
restart: ## Restart every workload
|
||||||
|
$(KUBECTL) -n $(NS) rollout restart $(WORKLOADS)
|
||||||
|
$(MAKE) --no-print-directory wait
|
||||||
|
|
||||||
|
.PHONY: restart-prometheus restart-loki restart-tempo restart-grafana restart-alloy
|
||||||
|
restart-prometheus: ## Restart Prometheus
|
||||||
|
$(KUBECTL) -n $(NS) rollout restart deployment/prometheus && $(KUBECTL) -n $(NS) rollout status deployment/prometheus
|
||||||
|
restart-loki: ## Restart Loki
|
||||||
|
$(KUBECTL) -n $(NS) rollout restart deployment/loki && $(KUBECTL) -n $(NS) rollout status deployment/loki
|
||||||
|
restart-tempo: ## Restart Tempo
|
||||||
|
$(KUBECTL) -n $(NS) rollout restart deployment/tempo && $(KUBECTL) -n $(NS) rollout status deployment/tempo
|
||||||
|
restart-grafana: ## Restart Grafana (dashboards picked up immediately)
|
||||||
|
$(KUBECTL) -n $(NS) rollout restart deployment/grafana && $(KUBECTL) -n $(NS) rollout status deployment/grafana
|
||||||
|
restart-alloy: ## Restart Alloy
|
||||||
|
$(KUBECTL) -n $(NS) rollout restart daemonset/alloy && $(KUBECTL) -n $(NS) rollout status daemonset/alloy
|
||||||
|
|
||||||
|
.PHONY: reload-prometheus
|
||||||
|
reload-prometheus: ## Hot-reload the Prometheus config (no restart)
|
||||||
|
$(KUBECTL) -n $(NS) exec deployment/prometheus -- wget -q --post-data='' -O- http://localhost:9090/-/reload \
|
||||||
|
&& printf "$(C_OK)Prometheus configuration reloaded$(C_OFF)\n"
|
||||||
|
|
||||||
|
##@ Persistence
|
||||||
|
|
||||||
|
.PHONY: pvc
|
||||||
|
pvc: ## PVC state and the volumes behind them
|
||||||
|
$(KUBECTL) -n $(NS) get pvc -o custom-columns='NAME:.metadata.name,STATUS:.status.phase,CAPACITY:.status.capacity.storage,SC:.spec.storageClassName,VOLUME:.spec.volumeName'
|
||||||
|
echo
|
||||||
|
$(KUBECTL) -n $(NS) get pods -o json | jq -r '.items[] | . as $$p | (.spec.volumes[]? | select(.persistentVolumeClaim) | "\($$p.metadata.name)\t-> \(.persistentVolumeClaim.claimName)")'
|
||||||
|
|
||||||
|
.PHONY: flush-pvc
|
||||||
|
flush-pvc: ## DESTRUCTIVE: wipe all persisted data (CONFIRM=yes)
|
||||||
|
$(call confirm,Wipe ALL data (metrics, logs, traces, Grafana))
|
||||||
|
for c in $(STATEFUL); do $(MAKE) --no-print-directory _flush COMP=$$c; done
|
||||||
|
$(MAKE) --no-print-directory wait
|
||||||
|
|
||||||
|
.PHONY: flush-prometheus flush-loki flush-tempo flush-grafana
|
||||||
|
flush-prometheus: ## DESTRUCTIVE: wipe the Prometheus TSDB (CONFIRM=yes)
|
||||||
|
$(call confirm,Wipe the Prometheus TSDB)
|
||||||
|
$(MAKE) --no-print-directory _flush COMP=prometheus
|
||||||
|
flush-loki: ## DESTRUCTIVE: wipe the Loki logs (CONFIRM=yes)
|
||||||
|
$(call confirm,Wipe the Loki logs)
|
||||||
|
$(MAKE) --no-print-directory _flush COMP=loki
|
||||||
|
flush-tempo: ## DESTRUCTIVE: wipe the Tempo traces (CONFIRM=yes)
|
||||||
|
$(call confirm,Wipe the Tempo traces)
|
||||||
|
$(MAKE) --no-print-directory _flush COMP=tempo
|
||||||
|
flush-grafana: ## DESTRUCTIVE: wipe the Grafana database (CONFIRM=yes)
|
||||||
|
$(call confirm,Wipe the Grafana database (users and prefs; dashboards are reprovisioned))
|
||||||
|
$(MAKE) --no-print-directory _flush COMP=grafana
|
||||||
|
|
||||||
|
# Internal target: scale down -> delete PVC -> recreate -> scale up.
|
||||||
|
.PHONY: _flush
|
||||||
|
_flush:
|
||||||
|
printf "$(C_WARN)flushing %s$(C_OFF)\n" "$(COMP)"
|
||||||
|
$(KUBECTL) -n $(NS) scale deployment/$(COMP) --replicas=0
|
||||||
|
$(KUBECTL) -n $(NS) wait --for=delete pod -l app=$(COMP) --timeout=120s 2>/dev/null || true
|
||||||
|
$(KUBECTL) -n $(NS) delete pvc $(COMP)-data --ignore-not-found --wait=true
|
||||||
|
$(KUBECTL) apply -k $(K8S)/$(COMP)
|
||||||
|
$(KUBECTL) -n $(NS) scale deployment/$(COMP) --replicas=1
|
||||||
|
|
||||||
|
##@ Validation and teardown
|
||||||
|
|
||||||
|
.PHONY: build
|
||||||
|
build: ## Render the generated manifests to stdout
|
||||||
|
kustomize build $(K8S)
|
||||||
|
|
||||||
|
.PHONY: diff
|
||||||
|
diff: check-context ## Diff the manifests against the cluster state
|
||||||
|
$(KUBECTL) diff -k $(K8S) || true
|
||||||
|
|
||||||
|
.PHONY: validate
|
||||||
|
validate: dashboards-check ## Validate the manifests (build + server dry-run)
|
||||||
|
kustomize build $(K8S) >/dev/null && printf "$(C_OK)kustomize build OK$(C_OFF)\n"
|
||||||
|
$(KUBECTL) apply -k $(K8S) --dry-run=server >/dev/null && printf "$(C_OK)server dry-run OK$(C_OFF)\n"
|
||||||
|
|
||||||
|
.PHONY: undeploy
|
||||||
|
undeploy: ## DESTRUCTIVE: delete the stack, PVCs kept (CONFIRM=yes)
|
||||||
|
$(call confirm,Delete the stack (PVCs are KEPT))
|
||||||
|
$(KUBECTL) delete -k $(K8S) --ignore-not-found
|
||||||
|
|
||||||
|
.PHONY: destroy
|
||||||
|
destroy: ## DESTRUCTIVE: delete the stack AND the PVCs (CONFIRM=yes)
|
||||||
|
$(call confirm,Delete the stack AND all data)
|
||||||
|
$(KUBECTL) delete -k $(K8S) --ignore-not-found
|
||||||
|
$(KUBECTL) -n $(NS) delete pvc --all --ignore-not-found
|
||||||
|
|
||||||
|
.PHONY: check-context
|
||||||
|
check-context:
|
||||||
|
cur=$$(kubectl config current-context); \
|
||||||
|
if [ "$$cur" != "$(CTX)" ]; then \
|
||||||
|
printf "$(C_ERR)current context '%s' != '$(CTX)'$(C_OFF) (commands explicitly target $(CTX))\n" "$$cur"; \
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Guard rail for destructive targets: requires CONFIRM=yes
|
||||||
|
define confirm
|
||||||
|
@if [ "$(CONFIRM)" != "yes" ]; then \
|
||||||
|
printf "$(C_ERR)Destructive operation$(C_OFF): $(1)\n"; \
|
||||||
|
printf "Re-run with $(C_WARN)CONFIRM=yes$(C_OFF): make $(C_WARN)$@ CONFIRM=yes$(C_OFF)\n"; \
|
||||||
|
exit 1; \
|
||||||
|
fi
|
||||||
|
endef
|
||||||
@@ -0,0 +1,281 @@
|
|||||||
|
# Infra Observability (Kubernetes / declarative)
|
||||||
|
|
||||||
|
**Grafana 13.2 / Prometheus 3.14 / Loki 3.7 / Tempo 3.0** plus **Alloy 1.19**,
|
||||||
|
deployed fully declaratively (Kustomize) into the `observability` namespace of
|
||||||
|
the local `orbstack` cluster.
|
||||||
|
|
||||||
|
Nothing is imperative: no `helm install`, no dashboard hand-created in the UI.
|
||||||
|
The entire desired state lives in this repository and `make deploy` reconciles it.
|
||||||
|
|
||||||
|
## Repository layout
|
||||||
|
|
||||||
|
```text
|
||||||
|
Makefile # Every command (make = help)
|
||||||
|
k8s/
|
||||||
|
├── kustomization.yaml # Root Kustomize manifest
|
||||||
|
├── namespace.yaml # observability namespace
|
||||||
|
├── secret.yaml # Grafana admin credentials (GITIGNORED)
|
||||||
|
├── secret.example.yaml # Secret template
|
||||||
|
├── prometheus/
|
||||||
|
│ ├── prometheus.yaml # Scrape config (Kubernetes SD)
|
||||||
|
│ ├── rbac.yaml # SA + discovery ClusterRole
|
||||||
|
│ ├── secret.yaml # LibreChat scrape token (GITIGNORED)
|
||||||
|
│ ├── secret.example.yaml # Token template
|
||||||
|
│ ├── pvc.yaml # Persistent TSDB (10Gi)
|
||||||
|
│ ├── deployment.yaml # Prometheus v3.14.0
|
||||||
|
│ └── service.yaml # :9090
|
||||||
|
├── loki/
|
||||||
|
│ ├── loki.yaml # Monolithic config, filesystem storage
|
||||||
|
│ ├── pvc.yaml # Chunks + index (10Gi)
|
||||||
|
│ ├── deployment.yaml # Loki 3.7.6 (-target=all)
|
||||||
|
│ └── service.yaml # :3100 (HTTP) / :9096 (gRPC)
|
||||||
|
├── tempo/
|
||||||
|
│ ├── tempo.yaml # Monolithic config + metrics-generator
|
||||||
|
│ ├── pvc.yaml # WAL + trace blocks (10Gi)
|
||||||
|
│ ├── deployment.yaml # Tempo 3.0.3 (-target=all)
|
||||||
|
│ └── service.yaml # :3200, OTLP :4317/:4318, Zipkin :9411
|
||||||
|
├── alloy/
|
||||||
|
│ ├── config.alloy # Pod logs -> Loki pipeline
|
||||||
|
│ ├── rbac.yaml # SA + ClusterRole (including pods/log)
|
||||||
|
│ ├── daemonset.yaml # Alloy v1.19.0 (one pod per node)
|
||||||
|
│ └── service.yaml # :12345 (UI + /metrics)
|
||||||
|
├── grafana/
|
||||||
|
│ ├── provisioning/
|
||||||
|
│ │ ├── datasources.yaml # Datasources with stable UIDs
|
||||||
|
│ │ └── dashboard-providers.yaml # File provider
|
||||||
|
│ ├── dashboards/
|
||||||
|
│ │ ├── kustomization.yaml # ConfigMap WITHOUT hash (hot reload)
|
||||||
|
│ │ ├── obs-stack-health.json # uid: obs-stack-health
|
||||||
|
│ │ ├── obs-keycloak.json # uid: obs-keycloak
|
||||||
|
│ │ ├── obs-librechat.json # uid: obs-librechat
|
||||||
|
│ │ └── obs-classic-models-api.json
|
||||||
|
│ ├── pvc.yaml # Grafana SQLite database (5Gi)
|
||||||
|
│ ├── deployment.yaml # Grafana 13.2.0
|
||||||
|
│ └── service.yaml # :3000
|
||||||
|
└── ingress/
|
||||||
|
├── certificate.yaml # cert-manager Certificate (sttlab-local-ca)
|
||||||
|
└── ingress.yaml # Three NGINX Ingresses
|
||||||
|
```
|
||||||
|
|
||||||
|
## Getting started
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make # Help: every available target
|
||||||
|
make deploy # Deploy the whole stack
|
||||||
|
make hosts # /etc/hosts entries to add
|
||||||
|
make grafana-password # Admin credentials
|
||||||
|
make smoke # End-to-end verification
|
||||||
|
```
|
||||||
|
|
||||||
|
| URL | Purpose |
|
||||||
|
|---|---|
|
||||||
|
| `https://grafana.sttlab.local` | Grafana UI |
|
||||||
|
| `https://prometheus.sttlab.local` | Prometheus UI (targets, expression browser) |
|
||||||
|
| `https://alloy.sttlab.local` | Alloy UI (log pipeline state) |
|
||||||
|
|
||||||
|
Sending traces from an in-cluster application:
|
||||||
|
`tempo.observability.svc.cluster.local:4317` (OTLP gRPC) or `:4318` (OTLP HTTP).
|
||||||
|
|
||||||
|
## Declarative dashboards with stable UIDs
|
||||||
|
|
||||||
|
This is the centrepiece of the repository: **a dashboard only ever exists as a
|
||||||
|
versioned JSON file**, never as an object created through the UI.
|
||||||
|
|
||||||
|
### Rules
|
||||||
|
|
||||||
|
1. **One `.json` file per dashboard**, under `k8s/grafana/dashboards/`.
|
||||||
|
2. **The `uid` field is stable and never changes.** It is what keeps URLs
|
||||||
|
(`/d/<uid>/…`), cross-dashboard links, annotations and alerts alive across
|
||||||
|
every redeployment. By convention the filename equals the UID
|
||||||
|
(`obs-keycloak.json` -> `uid: obs-keycloak`).
|
||||||
|
3. **The `id` field must stay `null`** (or absent): it is an identifier internal
|
||||||
|
to a Grafana instance and is not portable.
|
||||||
|
4. **Datasources are referenced by stable UID**, never by name:
|
||||||
|
`"datasource": { "type": "prometheus", "uid": "prometheus" }`.
|
||||||
|
Available UIDs: `prometheus`, `loki`, `tempo`.
|
||||||
|
5. `allowUiUpdates: false`: any change made in the UI is overwritten on the next
|
||||||
|
reload. The file is the source of truth.
|
||||||
|
|
||||||
|
### Adding a dashboard
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cp my-dashboard.json k8s/grafana/dashboards/obs-my-topic.json
|
||||||
|
# edit the JSON: "uid": "obs-my-topic", "id": null
|
||||||
|
make dashboards-sync # update the list in dashboards/kustomization.yaml
|
||||||
|
make dashboards # apply (without restarting Grafana)
|
||||||
|
```
|
||||||
|
|
||||||
|
`make dashboards-check` (run automatically by `dashboards` and `validate`)
|
||||||
|
verifies: valid JSON, `uid` present, `uid` unique, `id` null, file referenced in
|
||||||
|
the kustomization, filename/UID consistency.
|
||||||
|
|
||||||
|
### How the hot reload works
|
||||||
|
|
||||||
|
The `grafana-dashboards` ConfigMap is generated **without a name-suffix hash**
|
||||||
|
(`disableNameSuffixHash: true`), so `make dashboards` updates it *in place*: the
|
||||||
|
Grafana pod is not recreated and the file provider re-reads the directory
|
||||||
|
(`updateIntervalSeconds: 10`).
|
||||||
|
|
||||||
|
Measured end-to-end delay: **about 70 seconds** (kubelet ConfigMap volume sync
|
||||||
|
~60s + provider poll 10s). For immediate effect: `make restart-grafana`.
|
||||||
|
|
||||||
|
Conversely, the *provisioning* ConfigMaps (datasources, providers) **keep** their
|
||||||
|
hash suffix: those files are only read at startup, so the hash deliberately
|
||||||
|
forces a pod restart when they change.
|
||||||
|
|
||||||
|
## Persistence
|
||||||
|
|
||||||
|
| PVC | Component | Size | Contents | Retention |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| `prometheus-data` | Prometheus | 10Gi | TSDB | 15 d / 8 GB |
|
||||||
|
| `loki-data` | Loki | 10Gi | chunks + TSDB index | 168 h (compactor) |
|
||||||
|
| `tempo-data` | Tempo | 10Gi | WAL + trace blocks | 168 h |
|
||||||
|
| `grafana-data` | Grafana | 5Gi | SQLite database (users, prefs) | — |
|
||||||
|
|
||||||
|
StorageClass `local-path` (the cluster default), `ReadWriteOnce`. The Deployments
|
||||||
|
use `strategy: Recreate`: with an RWO volume the old pod must release it before
|
||||||
|
the new one can start.
|
||||||
|
|
||||||
|
Alloy's read positions live on a `hostPath` (`/var/lib/alloy`), so restarting the
|
||||||
|
DaemonSet does not re-ingest every log.
|
||||||
|
|
||||||
|
### Wiping data
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make pvc # PVC state
|
||||||
|
make flush-loki CONFIRM=yes # wipe the logs only
|
||||||
|
make flush-prometheus CONFIRM=yes # wipe the TSDB only
|
||||||
|
make flush-pvc CONFIRM=yes # wipe EVERYTHING
|
||||||
|
```
|
||||||
|
|
||||||
|
The cycle is: `scale 0` -> wait for pod deletion -> `delete pvc` -> reapply the
|
||||||
|
kustomization (which recreates the PVC) -> `scale 1`. Every destructive target
|
||||||
|
requires `CONFIRM=yes`.
|
||||||
|
|
||||||
|
Dashboards are **not** affected by `flush-grafana`: they are reprovisioned from
|
||||||
|
the ConfigMaps at startup.
|
||||||
|
|
||||||
|
## Wiring the three signals
|
||||||
|
|
||||||
|
- **Prometheus** discovers its targets through the Kubernetes API: apiserver,
|
||||||
|
kubelet, container resource metrics, plus any Service or Pod carrying the
|
||||||
|
`prometheus.io/scrape: "true"` annotation (`prometheus.io/port`,
|
||||||
|
`prometheus.io/path`).
|
||||||
|
- **Alloy** reads every pod's logs through the Kubernetes API (no `/var/log`
|
||||||
|
mount) and pushes them to Loki with `namespace`, `pod`, `container`, `node`,
|
||||||
|
`app`, `component` and `job` labels.
|
||||||
|
- **Tempo** receives OTLP, and its `metrics_generator` pushes span metrics and
|
||||||
|
service graphs to Prometheus through its remote-write receiver
|
||||||
|
(`--web.enable-remote-write-receiver`).
|
||||||
|
- **Grafana correlations**: Loki -> Tempo (`TraceID` derived field),
|
||||||
|
Tempo -> Loki (`tracesToLogsV2`), Tempo -> Prometheus (`serviceMap`,
|
||||||
|
`tracesToMetrics`), Prometheus exemplars -> Tempo.
|
||||||
|
|
||||||
|
### Container resource metrics
|
||||||
|
|
||||||
|
Prometheus scrapes the kubelet's **`/metrics/resource`** endpoint, not
|
||||||
|
`/metrics/cadvisor`. On this cluster the cadvisor endpoint answers with `machine_*`
|
||||||
|
series only and exposes no `container_*` at all, so CPU and memory panels would
|
||||||
|
stay empty. `/metrics/resource` — the source metrics-server itself reads —
|
||||||
|
provides `container_cpu_usage_seconds_total` and
|
||||||
|
`container_memory_working_set_bytes` with `namespace`/`pod`/`container` labels.
|
||||||
|
|
||||||
|
kube-state-metrics is not deployed, so container restarts are derived from
|
||||||
|
changes in `container_start_time_seconds`.
|
||||||
|
|
||||||
|
## Connected applications
|
||||||
|
|
||||||
|
| Application | Traces | Metrics | Logs | Dashboard |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| `keycloak` | OTel Java agent → `tempo:4318` | native `/metrics` (management port 9000) | JSON + MDC `trace_id` | `obs-keycloak` |
|
||||||
|
| `librechat` | OTel SDK → `tempo:4318` | built-in `/metrics`, Bearer token | stdout | `obs-librechat` |
|
||||||
|
| `classic-models-api` | `opentelemetry-instrument` → `tempo:4318` | django-prometheus `/metrics` | JSON + `otelTraceID` | `obs-classic-models-api` |
|
||||||
|
|
||||||
|
Their manifests live in the `infra-keycloak`, `infra-librechat` and
|
||||||
|
`classic-models-api` repositories; this repository only holds the platform.
|
||||||
|
|
||||||
|
Each application dashboard carries the same four sections: an overview of RED
|
||||||
|
indicators, HTTP traffic, a runtime section specific to the stack in use, and a
|
||||||
|
**Logs** row, followed by container resources.
|
||||||
|
|
||||||
|
The Logs row leans on Loki's `detected_level`, which is inferred at ingestion
|
||||||
|
time, so a single query works across all three log formats — Keycloak emits JSON,
|
||||||
|
LibreChat colour-coded text, and the Django API plain gunicorn access lines. No
|
||||||
|
per-application parser is needed. The volume panel is scoped to the application
|
||||||
|
container, while the two error panels cover the whole namespace so the
|
||||||
|
datastore's own failures surface alongside the application's. Where a log line
|
||||||
|
carries a `trace_id`, the Loki datasource's derived field turns it into a link
|
||||||
|
straight to the matching trace in Tempo.
|
||||||
|
|
||||||
|
### Connecting an application
|
||||||
|
|
||||||
|
1. **Traces** — point OTLP at Tempo:
|
||||||
|
`OTEL_EXPORTER_OTLP_ENDPOINT=http://tempo.observability.svc.cluster.local:4318`
|
||||||
|
(`:4317` for gRPC). RED span metrics and the service graph are then derived
|
||||||
|
automatically by Tempo's `metrics_generator`, with no extra instrumentation.
|
||||||
|
2. **Metrics** — annotate the **pod** (not the Service):
|
||||||
|
```yaml
|
||||||
|
annotations:
|
||||||
|
prometheus.io/scrape: "true"
|
||||||
|
prometheus.io/port: "9000"
|
||||||
|
prometheus.io/path: "/metrics"
|
||||||
|
```
|
||||||
|
3. **Logs** — nothing to do: Alloy collects every pod in the cluster. For Grafana
|
||||||
|
to jump from a log line to its trace, the line only needs to contain a
|
||||||
|
`trace_id` (the Loki datasource's derived field).
|
||||||
|
|
||||||
|
### Token-protected endpoints
|
||||||
|
|
||||||
|
Annotation-based discovery cannot carry credentials, so an authenticated target
|
||||||
|
gets its own `scrape_config` in `k8s/prometheus/prometheus.yaml`, with the token
|
||||||
|
mounted from a Secret — see the `librechat` job, which reads
|
||||||
|
`/etc/prometheus/secrets/librechat/token`.
|
||||||
|
|
||||||
|
That token must be **identical** on both sides: `METRICS_SECRET` in the
|
||||||
|
`librechat-secrets` secret (namespace `librechat`) and the `token` key of the
|
||||||
|
`librechat-metrics` secret (namespace `observability`, file
|
||||||
|
`k8s/prometheus/secret.yaml`, gitignored).
|
||||||
|
|
||||||
|
A Kubernetes Secret is namespaced and a pod can only mount ones from its own
|
||||||
|
namespace, so the same value has to exist twice — once for LibreChat, which
|
||||||
|
**verifies** the token, once for Prometheus, which **presents** it. The risk is
|
||||||
|
silent drift: regenerating one side without the other sends the scrape back to
|
||||||
|
401 without breaking anything else. `make check-metrics-token` (included in
|
||||||
|
`make diag`) compares both copies.
|
||||||
|
|
||||||
|
## Diagnostics
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make status # pods, services, PVCs, ingresses
|
||||||
|
make diag # + endpoints, certificate, failing containers, events
|
||||||
|
make smoke # health of all five components, datasources, dashboards, targets
|
||||||
|
make targets # Prometheus targets and their last scrape error
|
||||||
|
make logs-loki # follow one component's logs
|
||||||
|
make top # CPU/memory usage
|
||||||
|
make pf-grafana # port-forward when the ingress is unavailable
|
||||||
|
make diff # difference between the repository and the cluster
|
||||||
|
make validate # kustomize build + server dry-run + dashboard checks
|
||||||
|
```
|
||||||
|
|
||||||
|
## Secrets
|
||||||
|
|
||||||
|
`k8s/secret.yaml` and `k8s/prometheus/secret.yaml` are **gitignored**.
|
||||||
|
`make secret` generates the first from `secret.example.yaml` with a random admin
|
||||||
|
password if it does not exist (and is called automatically by `make deploy`).
|
||||||
|
|
||||||
|
The TLS certificate is not versioned either: it is issued by the existing
|
||||||
|
`sttlab-local-ca` cert-manager ClusterIssuer (`k8s/ingress/certificate.yaml`).
|
||||||
|
|
||||||
|
## Coexisting with what was already there
|
||||||
|
|
||||||
|
The `observability` namespace already hosts two deployments **not managed by this
|
||||||
|
repository**, left untouched:
|
||||||
|
|
||||||
|
- `jaeger` (installed by Helm) — it also exposes OTLP on `4317`/`4318`, but on its
|
||||||
|
own Service, so there is no conflict with Tempo. Since all three applications
|
||||||
|
now point at Tempo, Jaeger no longer receives anything; uninstall the Helm
|
||||||
|
release whenever you want a single trace backend.
|
||||||
|
- `audit-log-collector` (Fluent Bit) — writes to its own PVC.
|
||||||
|
|
||||||
|
`make undeploy` and `make destroy` only run `kubectl delete -k k8s/`, so they
|
||||||
|
touch nothing but the resources described here.
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
// Grafana Alloy -- ships the logs of every pod in the cluster to Loki.
|
||||||
|
// Reads through the Kubernetes API (no /var/log mount required).
|
||||||
|
|
||||||
|
logging {
|
||||||
|
level = "info"
|
||||||
|
format = "logfmt"
|
||||||
|
}
|
||||||
|
|
||||||
|
discovery.kubernetes "pods" {
|
||||||
|
role = "pod"
|
||||||
|
}
|
||||||
|
|
||||||
|
discovery.relabel "pod_logs" {
|
||||||
|
targets = discovery.kubernetes.pods.targets
|
||||||
|
|
||||||
|
rule {
|
||||||
|
source_labels = ["__meta_kubernetes_namespace"]
|
||||||
|
target_label = "namespace"
|
||||||
|
}
|
||||||
|
rule {
|
||||||
|
source_labels = ["__meta_kubernetes_pod_name"]
|
||||||
|
target_label = "pod"
|
||||||
|
}
|
||||||
|
rule {
|
||||||
|
source_labels = ["__meta_kubernetes_pod_container_name"]
|
||||||
|
target_label = "container"
|
||||||
|
}
|
||||||
|
rule {
|
||||||
|
source_labels = ["__meta_kubernetes_pod_node_name"]
|
||||||
|
target_label = "node"
|
||||||
|
}
|
||||||
|
rule {
|
||||||
|
source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
|
||||||
|
target_label = "app"
|
||||||
|
}
|
||||||
|
rule {
|
||||||
|
source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_component"]
|
||||||
|
target_label = "component"
|
||||||
|
}
|
||||||
|
// job = <namespace>/<container>: the usual Loki convention.
|
||||||
|
rule {
|
||||||
|
source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"]
|
||||||
|
separator = "/"
|
||||||
|
target_label = "job"
|
||||||
|
}
|
||||||
|
// Only collect containers on the current node (DaemonSet).
|
||||||
|
rule {
|
||||||
|
source_labels = ["__meta_kubernetes_pod_node_name"]
|
||||||
|
regex = env("HOSTNAME_NODE")
|
||||||
|
action = "keep"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
loki.source.kubernetes "pods" {
|
||||||
|
targets = discovery.relabel.pod_logs.output
|
||||||
|
forward_to = [loki.write.default.receiver]
|
||||||
|
}
|
||||||
|
|
||||||
|
loki.write "default" {
|
||||||
|
endpoint {
|
||||||
|
url = "http://loki:3100/loki/api/v1/push"
|
||||||
|
}
|
||||||
|
external_labels = {
|
||||||
|
cluster = "orbstack",
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: DaemonSet
|
||||||
|
metadata:
|
||||||
|
name: alloy
|
||||||
|
namespace: observability
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: alloy
|
||||||
|
app.kubernetes.io/component: collector
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: alloy
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: alloy
|
||||||
|
app.kubernetes.io/name: alloy
|
||||||
|
app.kubernetes.io/component: collector
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
annotations:
|
||||||
|
prometheus.io/scrape: "true"
|
||||||
|
prometheus.io/port: "12345"
|
||||||
|
spec:
|
||||||
|
serviceAccountName: alloy
|
||||||
|
tolerations:
|
||||||
|
- operator: Exists
|
||||||
|
containers:
|
||||||
|
- name: alloy
|
||||||
|
image: grafana/alloy:v1.19.0
|
||||||
|
args:
|
||||||
|
- run
|
||||||
|
- /etc/alloy/config.alloy
|
||||||
|
- --storage.path=/var/lib/alloy/data
|
||||||
|
- --server.http.listen-addr=0.0.0.0:12345
|
||||||
|
- --disable-reporting
|
||||||
|
env:
|
||||||
|
# Used by config.alloy to tail only the pods on the current node.
|
||||||
|
- name: HOSTNAME_NODE
|
||||||
|
valueFrom:
|
||||||
|
fieldRef:
|
||||||
|
fieldPath: spec.nodeName
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 12345
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 50m
|
||||||
|
memory: 128Mi
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 512Mi
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /-/ready
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 10
|
||||||
|
periodSeconds: 10
|
||||||
|
volumeMounts:
|
||||||
|
- name: config
|
||||||
|
mountPath: /etc/alloy
|
||||||
|
- name: storage
|
||||||
|
mountPath: /var/lib/alloy
|
||||||
|
volumes:
|
||||||
|
- name: config
|
||||||
|
configMap:
|
||||||
|
name: alloy-config
|
||||||
|
- name: storage
|
||||||
|
# Read positions persisted on the node: no full log re-ingestion
|
||||||
|
# when the pod restarts.
|
||||||
|
hostPath:
|
||||||
|
path: /var/lib/alloy
|
||||||
|
type: DirectoryOrCreate
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
namespace: observability
|
||||||
|
|
||||||
|
resources:
|
||||||
|
- rbac.yaml
|
||||||
|
- daemonset.yaml
|
||||||
|
- service.yaml
|
||||||
|
|
||||||
|
configMapGenerator:
|
||||||
|
- name: alloy-config
|
||||||
|
files:
|
||||||
|
- config.alloy
|
||||||
|
|
||||||
|
generatorOptions:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: alloy
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: alloy
|
||||||
|
namespace: observability
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: alloy
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: observability-alloy
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: alloy
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
rules:
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources:
|
||||||
|
- nodes
|
||||||
|
- nodes/proxy
|
||||||
|
- nodes/metrics
|
||||||
|
- services
|
||||||
|
- endpoints
|
||||||
|
- pods
|
||||||
|
- namespaces
|
||||||
|
- events
|
||||||
|
verbs: ["get", "list", "watch"]
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["pods/log"]
|
||||||
|
verbs: ["get", "list", "watch"]
|
||||||
|
- apiGroups: ["discovery.k8s.io"]
|
||||||
|
resources: ["endpointslices"]
|
||||||
|
verbs: ["get", "list", "watch"]
|
||||||
|
- nonResourceURLs: ["/metrics"]
|
||||||
|
verbs: ["get"]
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
metadata:
|
||||||
|
name: observability-alloy
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: alloy
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: ClusterRole
|
||||||
|
name: observability-alloy
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: alloy
|
||||||
|
namespace: observability
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: alloy
|
||||||
|
namespace: observability
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: alloy
|
||||||
|
app.kubernetes.io/component: collector
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
selector:
|
||||||
|
app: alloy
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 12345
|
||||||
|
targetPort: http
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
# Grafana dashboards -- declarative provisioning.
|
||||||
|
#
|
||||||
|
# Rules:
|
||||||
|
# * one .json file per dashboard, named after its UID;
|
||||||
|
# * every JSON carries a STABLE "uid" field (never regenerated): that is what
|
||||||
|
# keeps URLs, cross-links and alerts alive across redeployments;
|
||||||
|
# * `disableNameSuffixHash: true` -> the ConfigMap always keeps the same name,
|
||||||
|
# so `kubectl apply` updates it in place and Grafana reloads WITHOUT
|
||||||
|
# restarting the pod, in about 70s (see dashboard-providers.yaml).
|
||||||
|
#
|
||||||
|
# Adding a dashboard = drop the .json here and add it to `files:` below.
|
||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
namespace: observability
|
||||||
|
|
||||||
|
configMapGenerator:
|
||||||
|
- name: grafana-dashboards
|
||||||
|
files:
|
||||||
|
# >>> dashboards (generated by `make dashboards-sync`) >>>
|
||||||
|
- obs-classic-models-api.json
|
||||||
|
- obs-keycloak.json
|
||||||
|
- obs-librechat.json
|
||||||
|
- obs-stack-health.json
|
||||||
|
# <<< dashboards <<<
|
||||||
|
|
||||||
|
generatorOptions:
|
||||||
|
disableNameSuffixHash: true
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: grafana
|
||||||
|
app.kubernetes.io/component: dashboards
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,437 @@
|
|||||||
|
{
|
||||||
|
"uid": "obs-stack-health",
|
||||||
|
"title": "Observability — Stack Health",
|
||||||
|
"description": "Health of the Prometheus / Loki / Tempo / Alloy stack. Reference dashboard: stable UID, declaratively provisioned.",
|
||||||
|
"tags": [
|
||||||
|
"observability",
|
||||||
|
"stack",
|
||||||
|
"provisioned"
|
||||||
|
],
|
||||||
|
"editable": false,
|
||||||
|
"schemaVersion": 39,
|
||||||
|
"version": 1,
|
||||||
|
"refresh": "30s",
|
||||||
|
"time": {
|
||||||
|
"from": "now-1h",
|
||||||
|
"to": "now"
|
||||||
|
},
|
||||||
|
"timezone": "browser",
|
||||||
|
"templating": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"name": "namespace",
|
||||||
|
"label": "Namespace",
|
||||||
|
"type": "query",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"query": "label_values(kube_pod_info, namespace)",
|
||||||
|
"refresh": 2,
|
||||||
|
"includeAll": true,
|
||||||
|
"multi": true,
|
||||||
|
"current": {
|
||||||
|
"text": "observability",
|
||||||
|
"value": "observability"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"type": "row",
|
||||||
|
"title": "Collection",
|
||||||
|
"gridPos": {
|
||||||
|
"h": 1,
|
||||||
|
"w": 24,
|
||||||
|
"x": 0,
|
||||||
|
"y": 0
|
||||||
|
},
|
||||||
|
"collapsed": false,
|
||||||
|
"panels": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Prometheus targets UP",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 5,
|
||||||
|
"w": 6,
|
||||||
|
"x": 0,
|
||||||
|
"y": 1
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"expr": "sum(up)",
|
||||||
|
"legendFormat": "up"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"color": {
|
||||||
|
"mode": "thresholds"
|
||||||
|
},
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"color": "red",
|
||||||
|
"value": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"color": "green",
|
||||||
|
"value": 1
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"reduceOptions": {
|
||||||
|
"calcs": [
|
||||||
|
"lastNotNull"
|
||||||
|
],
|
||||||
|
"fields": "",
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "area",
|
||||||
|
"textMode": "auto"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 3,
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Prometheus targets DOWN",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 5,
|
||||||
|
"w": 6,
|
||||||
|
"x": 6,
|
||||||
|
"y": 1
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"expr": "sum(up == bool 0)",
|
||||||
|
"legendFormat": "down"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"color": {
|
||||||
|
"mode": "thresholds"
|
||||||
|
},
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"color": "green",
|
||||||
|
"value": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"color": "red",
|
||||||
|
"value": 1
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"reduceOptions": {
|
||||||
|
"calcs": [
|
||||||
|
"lastNotNull"
|
||||||
|
],
|
||||||
|
"fields": "",
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "none",
|
||||||
|
"textMode": "auto"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 4,
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Log lines ingested (/s)",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 5,
|
||||||
|
"w": 6,
|
||||||
|
"x": 12,
|
||||||
|
"y": 1
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"expr": "sum(rate(loki_distributor_lines_received_total[5m]))",
|
||||||
|
"legendFormat": "lines/s"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 1,
|
||||||
|
"color": {
|
||||||
|
"mode": "fixed",
|
||||||
|
"fixedColor": "blue"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"reduceOptions": {
|
||||||
|
"calcs": [
|
||||||
|
"lastNotNull"
|
||||||
|
],
|
||||||
|
"fields": "",
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "area",
|
||||||
|
"textMode": "auto"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 5,
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Spans ingested (/s)",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 5,
|
||||||
|
"w": 6,
|
||||||
|
"x": 18,
|
||||||
|
"y": 1
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"expr": "sum(rate(tempo_distributor_spans_received_total[5m]))",
|
||||||
|
"legendFormat": "spans/s"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 1,
|
||||||
|
"color": {
|
||||||
|
"mode": "fixed",
|
||||||
|
"fixedColor": "purple"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"reduceOptions": {
|
||||||
|
"calcs": [
|
||||||
|
"lastNotNull"
|
||||||
|
],
|
||||||
|
"fields": "",
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "area",
|
||||||
|
"textMode": "auto"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 6,
|
||||||
|
"type": "row",
|
||||||
|
"title": "Stack resources",
|
||||||
|
"gridPos": {
|
||||||
|
"h": 1,
|
||||||
|
"w": 24,
|
||||||
|
"x": 0,
|
||||||
|
"y": 6
|
||||||
|
},
|
||||||
|
"collapsed": false,
|
||||||
|
"panels": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 7,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "CPU per component (observability namespace)",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 0,
|
||||||
|
"y": 7
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"observability\", container!=\"\"}[$__rate_interval]))",
|
||||||
|
"legendFormat": "{{pod}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"custom": {
|
||||||
|
"drawStyle": "line",
|
||||||
|
"lineWidth": 1,
|
||||||
|
"fillOpacity": 10,
|
||||||
|
"showPoints": "never"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "table",
|
||||||
|
"placement": "bottom",
|
||||||
|
"calcs": [
|
||||||
|
"mean",
|
||||||
|
"max"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "desc"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 8,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Memory per component (observability namespace)",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 12,
|
||||||
|
"y": 7
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"observability\", container!=\"\"})",
|
||||||
|
"legendFormat": "{{pod}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "bytes",
|
||||||
|
"custom": {
|
||||||
|
"drawStyle": "line",
|
||||||
|
"lineWidth": 1,
|
||||||
|
"fillOpacity": 10,
|
||||||
|
"showPoints": "never"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "table",
|
||||||
|
"placement": "bottom",
|
||||||
|
"calcs": [
|
||||||
|
"mean",
|
||||||
|
"max"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "desc"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 9,
|
||||||
|
"type": "row",
|
||||||
|
"title": "Logs",
|
||||||
|
"gridPos": {
|
||||||
|
"h": 1,
|
||||||
|
"w": 24,
|
||||||
|
"x": 0,
|
||||||
|
"y": 15
|
||||||
|
},
|
||||||
|
"collapsed": false,
|
||||||
|
"panels": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 10,
|
||||||
|
"type": "logs",
|
||||||
|
"title": "Recent stack errors",
|
||||||
|
"datasource": {
|
||||||
|
"type": "loki",
|
||||||
|
"uid": "loki"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 10,
|
||||||
|
"w": 24,
|
||||||
|
"x": 0,
|
||||||
|
"y": 16
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "loki",
|
||||||
|
"uid": "loki"
|
||||||
|
},
|
||||||
|
"expr": "{namespace=\"observability\"} |~ \"(?i)(error|level=error|fatal)\"",
|
||||||
|
"queryType": "range"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"showTime": true,
|
||||||
|
"showLabels": false,
|
||||||
|
"wrapLogMessage": true,
|
||||||
|
"sortOrder": "Descending",
|
||||||
|
"dedupStrategy": "none",
|
||||||
|
"enableLogDetails": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,112 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: grafana
|
||||||
|
namespace: observability
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: grafana
|
||||||
|
app.kubernetes.io/component: ui
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
strategy:
|
||||||
|
type: Recreate
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: grafana
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: grafana
|
||||||
|
app.kubernetes.io/name: grafana
|
||||||
|
app.kubernetes.io/component: ui
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
annotations:
|
||||||
|
prometheus.io/scrape: "true"
|
||||||
|
prometheus.io/port: "3000"
|
||||||
|
prometheus.io/path: "/metrics"
|
||||||
|
spec:
|
||||||
|
securityContext:
|
||||||
|
runAsUser: 472
|
||||||
|
runAsGroup: 472
|
||||||
|
fsGroup: 472
|
||||||
|
containers:
|
||||||
|
- name: grafana
|
||||||
|
image: grafana/grafana:13.2.0
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 3000
|
||||||
|
env:
|
||||||
|
- name: GF_SECURITY_ADMIN_USER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: observability-secrets
|
||||||
|
key: GF_SECURITY_ADMIN_USER
|
||||||
|
- name: GF_SECURITY_ADMIN_PASSWORD
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: observability-secrets
|
||||||
|
key: GF_SECURITY_ADMIN_PASSWORD
|
||||||
|
- name: GF_SERVER_ROOT_URL
|
||||||
|
value: https://grafana.sttlab.local
|
||||||
|
- name: GF_PATHS_PROVISIONING
|
||||||
|
value: /etc/grafana/provisioning
|
||||||
|
# Everything is driven by the provisioning files, so creating
|
||||||
|
# dashboards or datasources from the UI is disabled.
|
||||||
|
- name: GF_USERS_ALLOW_SIGN_UP
|
||||||
|
value: "false"
|
||||||
|
- name: GF_ANALYTICS_REPORTING_ENABLED
|
||||||
|
value: "false"
|
||||||
|
- name: GF_ANALYTICS_CHECK_FOR_UPDATES
|
||||||
|
value: "false"
|
||||||
|
- name: GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES
|
||||||
|
value: "false"
|
||||||
|
- name: GF_NEWS_NEWS_FEED_ENABLED
|
||||||
|
value: "false"
|
||||||
|
- name: GF_LOG_LEVEL
|
||||||
|
value: info
|
||||||
|
- name: GF_FEATURE_TOGGLES_ENABLE
|
||||||
|
value: traceqlEditor,traceQLStreaming,correlations,lokiLogsDataplane
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 256Mi
|
||||||
|
limits:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 1Gi
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /api/health
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 15
|
||||||
|
periodSeconds: 10
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /api/health
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 30
|
||||||
|
volumeMounts:
|
||||||
|
- name: data
|
||||||
|
mountPath: /var/lib/grafana
|
||||||
|
- name: provisioning-datasources
|
||||||
|
mountPath: /etc/grafana/provisioning/datasources
|
||||||
|
- name: provisioning-dashboards
|
||||||
|
mountPath: /etc/grafana/provisioning/dashboards
|
||||||
|
- name: dashboards
|
||||||
|
mountPath: /var/lib/grafana/dashboards
|
||||||
|
volumes:
|
||||||
|
- name: data
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: grafana-data
|
||||||
|
- name: provisioning-datasources
|
||||||
|
configMap:
|
||||||
|
name: grafana-datasources
|
||||||
|
- name: provisioning-dashboards
|
||||||
|
configMap:
|
||||||
|
name: grafana-dashboard-providers
|
||||||
|
- name: dashboards
|
||||||
|
# Stable-named ConfigMap: updated in place by `make dashboards` and
|
||||||
|
# re-read by Grafana without a restart (~70s: kubelet sync + 10s poll).
|
||||||
|
configMap:
|
||||||
|
name: grafana-dashboards
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
namespace: observability
|
||||||
|
|
||||||
|
resources:
|
||||||
|
- pvc.yaml
|
||||||
|
- deployment.yaml
|
||||||
|
- service.yaml
|
||||||
|
# Dashboards live in ./dashboards and are referenced by the root kustomization:
|
||||||
|
# `k8s/grafana` is the runtime alone, `k8s/grafana/dashboards` the content alone.
|
||||||
|
# Both are deployable independently.
|
||||||
|
|
||||||
|
configMapGenerator:
|
||||||
|
# Name-suffix hash KEPT here: datasource and provider provisioning is only
|
||||||
|
# read at startup, so the hash deliberately forces a pod restart whenever
|
||||||
|
# these files change.
|
||||||
|
- name: grafana-datasources
|
||||||
|
files:
|
||||||
|
- datasources.yaml=provisioning/datasources.yaml
|
||||||
|
- name: grafana-dashboard-providers
|
||||||
|
files:
|
||||||
|
- dashboard-providers.yaml=provisioning/dashboard-providers.yaml
|
||||||
|
|
||||||
|
generatorOptions:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: grafana
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
# File-based dashboard provider.
|
||||||
|
# The JSON files are mounted from the `grafana-dashboards` ConfigMap (generated
|
||||||
|
# without a name-suffix hash): `make dashboards` updates the ConfigMap in place
|
||||||
|
# and Grafana reloads WITHOUT restarting the pod, in about 70s
|
||||||
|
# (kubelet volume sync ~60s + updateIntervalSeconds below).
|
||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
providers:
|
||||||
|
- name: infra-observability
|
||||||
|
orgId: 1
|
||||||
|
folder: Infra Observability
|
||||||
|
folderUid: infra-observability
|
||||||
|
type: file
|
||||||
|
disableDeletion: false
|
||||||
|
allowUiUpdates: false
|
||||||
|
updateIntervalSeconds: 10
|
||||||
|
options:
|
||||||
|
path: /var/lib/grafana/dashboards
|
||||||
|
foldersFromFilesStructure: false
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
# Declarative datasource provisioning.
|
||||||
|
# The `uid` values are STABLE and are what dashboard JSON files reference:
|
||||||
|
# "datasource": { "type": "prometheus", "uid": "prometheus" }
|
||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
deleteDatasources:
|
||||||
|
- name: Prometheus
|
||||||
|
orgId: 1
|
||||||
|
- name: Loki
|
||||||
|
orgId: 1
|
||||||
|
- name: Tempo
|
||||||
|
orgId: 1
|
||||||
|
|
||||||
|
datasources:
|
||||||
|
- name: Prometheus
|
||||||
|
uid: prometheus
|
||||||
|
type: prometheus
|
||||||
|
access: proxy
|
||||||
|
orgId: 1
|
||||||
|
url: http://prometheus:9090
|
||||||
|
isDefault: true
|
||||||
|
editable: false
|
||||||
|
jsonData:
|
||||||
|
httpMethod: POST
|
||||||
|
timeInterval: 30s
|
||||||
|
manageAlerts: false
|
||||||
|
exemplarTraceIdDestinations:
|
||||||
|
- name: trace_id
|
||||||
|
datasourceUid: tempo
|
||||||
|
|
||||||
|
- name: Loki
|
||||||
|
uid: loki
|
||||||
|
type: loki
|
||||||
|
access: proxy
|
||||||
|
orgId: 1
|
||||||
|
url: http://loki:3100
|
||||||
|
editable: false
|
||||||
|
jsonData:
|
||||||
|
timeout: 60
|
||||||
|
maxLines: 1000
|
||||||
|
# Logs -> traces: a trace_id in the log line opens Tempo.
|
||||||
|
derivedFields:
|
||||||
|
- name: TraceID
|
||||||
|
matcherType: regex
|
||||||
|
matcherRegex: '"?trace[_-]?id"?[:=]"?([a-f0-9]{16,32})'
|
||||||
|
url: '$${__value.raw}'
|
||||||
|
datasourceUid: tempo
|
||||||
|
urlDisplayLabel: View trace
|
||||||
|
|
||||||
|
- name: Tempo
|
||||||
|
uid: tempo
|
||||||
|
type: tempo
|
||||||
|
access: proxy
|
||||||
|
orgId: 1
|
||||||
|
url: http://tempo:3200
|
||||||
|
editable: false
|
||||||
|
jsonData:
|
||||||
|
httpMethod: GET
|
||||||
|
streamingEnabled:
|
||||||
|
search: true
|
||||||
|
metrics: true
|
||||||
|
# Service graph fed by the metrics_generator -> Prometheus.
|
||||||
|
serviceMap:
|
||||||
|
datasourceUid: prometheus
|
||||||
|
nodeGraph:
|
||||||
|
enabled: true
|
||||||
|
tracesToLogsV2:
|
||||||
|
datasourceUid: loki
|
||||||
|
spanStartTimeShift: -5m
|
||||||
|
spanEndTimeShift: 5m
|
||||||
|
filterByTraceID: true
|
||||||
|
filterBySpanID: false
|
||||||
|
tags:
|
||||||
|
- key: service.name
|
||||||
|
value: app
|
||||||
|
tracesToMetrics:
|
||||||
|
datasourceUid: prometheus
|
||||||
|
spanStartTimeShift: -5m
|
||||||
|
spanEndTimeShift: 5m
|
||||||
|
tags:
|
||||||
|
- key: service.name
|
||||||
|
value: service
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolumeClaim
|
||||||
|
metadata:
|
||||||
|
name: grafana-data
|
||||||
|
namespace: observability
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: grafana
|
||||||
|
app.kubernetes.io/component: storage
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 5Gi
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: grafana
|
||||||
|
namespace: observability
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: grafana
|
||||||
|
app.kubernetes.io/component: ui
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
selector:
|
||||||
|
app: grafana
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 3000
|
||||||
|
targetPort: http
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
# Certificates issued by the `sttlab-local-ca` cert-manager ClusterIssuer
|
||||||
|
# (already present on the cluster). No private key is ever versioned.
|
||||||
|
apiVersion: cert-manager.io/v1
|
||||||
|
kind: Certificate
|
||||||
|
metadata:
|
||||||
|
name: observability-tls
|
||||||
|
namespace: observability
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
spec:
|
||||||
|
secretName: observability-tls
|
||||||
|
issuerRef:
|
||||||
|
name: sttlab-local-ca
|
||||||
|
kind: ClusterIssuer
|
||||||
|
commonName: grafana.sttlab.local
|
||||||
|
dnsNames:
|
||||||
|
- grafana.sttlab.local
|
||||||
|
- prometheus.sttlab.local
|
||||||
|
- alloy.sttlab.local
|
||||||
|
- loki.sttlab.local
|
||||||
|
- tempo.sttlab.local
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: Ingress
|
||||||
|
metadata:
|
||||||
|
name: grafana
|
||||||
|
namespace: observability
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: grafana
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
annotations:
|
||||||
|
nginx.ingress.kubernetes.io/proxy-body-size: "20m"
|
||||||
|
nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
|
||||||
|
nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
|
||||||
|
nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
|
||||||
|
# Required for TraceQL streaming and Loki live tail.
|
||||||
|
nginx.ingress.kubernetes.io/proxy-buffering: "off"
|
||||||
|
spec:
|
||||||
|
ingressClassName: nginx
|
||||||
|
tls:
|
||||||
|
- hosts:
|
||||||
|
- grafana.sttlab.local
|
||||||
|
secretName: observability-tls
|
||||||
|
rules:
|
||||||
|
- host: grafana.sttlab.local
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
backend:
|
||||||
|
service:
|
||||||
|
name: grafana
|
||||||
|
port:
|
||||||
|
number: 3000
|
||||||
|
---
|
||||||
|
# Exposed for diagnostics (targets, TSDB status, expression browser).
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: Ingress
|
||||||
|
metadata:
|
||||||
|
name: prometheus
|
||||||
|
namespace: observability
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: prometheus
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
annotations:
|
||||||
|
nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
|
||||||
|
spec:
|
||||||
|
ingressClassName: nginx
|
||||||
|
tls:
|
||||||
|
- hosts:
|
||||||
|
- prometheus.sttlab.local
|
||||||
|
secretName: observability-tls
|
||||||
|
rules:
|
||||||
|
- host: prometheus.sttlab.local
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
backend:
|
||||||
|
service:
|
||||||
|
name: prometheus
|
||||||
|
port:
|
||||||
|
number: 9090
|
||||||
|
---
|
||||||
|
# Alloy UI: state of the log collection pipeline components.
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: Ingress
|
||||||
|
metadata:
|
||||||
|
name: alloy
|
||||||
|
namespace: observability
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: alloy
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
spec:
|
||||||
|
ingressClassName: nginx
|
||||||
|
tls:
|
||||||
|
- hosts:
|
||||||
|
- alloy.sttlab.local
|
||||||
|
secretName: observability-tls
|
||||||
|
rules:
|
||||||
|
- host: alloy.sttlab.local
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
backend:
|
||||||
|
service:
|
||||||
|
name: alloy
|
||||||
|
port:
|
||||||
|
number: 12345
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
namespace: observability
|
||||||
|
|
||||||
|
resources:
|
||||||
|
- certificate.yaml
|
||||||
|
- ingress.yaml
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
namespace: observability
|
||||||
|
|
||||||
|
resources:
|
||||||
|
- namespace.yaml
|
||||||
|
- secret.yaml
|
||||||
|
- prometheus
|
||||||
|
- loki
|
||||||
|
- tempo
|
||||||
|
- alloy
|
||||||
|
- grafana
|
||||||
|
- grafana/dashboards
|
||||||
|
- ingress
|
||||||
|
|
||||||
|
labels:
|
||||||
|
- includeSelectors: false
|
||||||
|
pairs:
|
||||||
|
app.kubernetes.io/managed-by: kustomize
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: loki
|
||||||
|
namespace: observability
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: loki
|
||||||
|
app.kubernetes.io/component: logs
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
strategy:
|
||||||
|
type: Recreate
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: loki
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: loki
|
||||||
|
app.kubernetes.io/name: loki
|
||||||
|
app.kubernetes.io/component: logs
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
annotations:
|
||||||
|
prometheus.io/scrape: "true"
|
||||||
|
prometheus.io/port: "3100"
|
||||||
|
spec:
|
||||||
|
securityContext:
|
||||||
|
runAsUser: 10001
|
||||||
|
runAsGroup: 10001
|
||||||
|
fsGroup: 10001
|
||||||
|
containers:
|
||||||
|
- name: loki
|
||||||
|
image: grafana/loki:3.7.6
|
||||||
|
args:
|
||||||
|
- -config.file=/etc/loki/loki.yaml
|
||||||
|
- -target=all
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 3100
|
||||||
|
- name: grpc
|
||||||
|
containerPort: 9096
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 256Mi
|
||||||
|
limits:
|
||||||
|
cpu: "2"
|
||||||
|
memory: 1Gi
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /ready
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 30
|
||||||
|
periodSeconds: 10
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /ready
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 30
|
||||||
|
volumeMounts:
|
||||||
|
- name: config
|
||||||
|
mountPath: /etc/loki
|
||||||
|
- name: data
|
||||||
|
mountPath: /loki
|
||||||
|
volumes:
|
||||||
|
- name: config
|
||||||
|
configMap:
|
||||||
|
name: loki-config
|
||||||
|
- name: data
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: loki-data
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
namespace: observability
|
||||||
|
|
||||||
|
resources:
|
||||||
|
- pvc.yaml
|
||||||
|
- deployment.yaml
|
||||||
|
- service.yaml
|
||||||
|
|
||||||
|
configMapGenerator:
|
||||||
|
- name: loki-config
|
||||||
|
files:
|
||||||
|
- loki.yaml
|
||||||
|
|
||||||
|
generatorOptions:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: loki
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
# Loki 3.x in monolithic mode (-target=all), filesystem storage on a PVC.
|
||||||
|
auth_enabled: false
|
||||||
|
|
||||||
|
server:
|
||||||
|
http_listen_address: 0.0.0.0
|
||||||
|
http_listen_port: 3100
|
||||||
|
grpc_listen_address: 0.0.0.0
|
||||||
|
grpc_listen_port: 9096
|
||||||
|
log_level: info
|
||||||
|
grpc_server_max_concurrent_streams: 1000
|
||||||
|
|
||||||
|
common:
|
||||||
|
instance_addr: 127.0.0.1
|
||||||
|
path_prefix: /loki
|
||||||
|
storage:
|
||||||
|
filesystem:
|
||||||
|
chunks_directory: /loki/chunks
|
||||||
|
rules_directory: /loki/rules
|
||||||
|
replication_factor: 1
|
||||||
|
ring:
|
||||||
|
kvstore:
|
||||||
|
store: inmemory
|
||||||
|
|
||||||
|
schema_config:
|
||||||
|
configs:
|
||||||
|
- from: "2024-01-01"
|
||||||
|
store: tsdb
|
||||||
|
object_store: filesystem
|
||||||
|
schema: v13
|
||||||
|
index:
|
||||||
|
prefix: index_
|
||||||
|
period: 24h
|
||||||
|
|
||||||
|
storage_config:
|
||||||
|
tsdb_shipper:
|
||||||
|
active_index_directory: /loki/tsdb-index
|
||||||
|
cache_location: /loki/tsdb-cache
|
||||||
|
filesystem:
|
||||||
|
directory: /loki/chunks
|
||||||
|
|
||||||
|
compactor:
|
||||||
|
working_directory: /loki/compactor
|
||||||
|
retention_enabled: true
|
||||||
|
delete_request_store: filesystem
|
||||||
|
compaction_interval: 10m
|
||||||
|
retention_delete_delay: 2h
|
||||||
|
|
||||||
|
limits_config:
|
||||||
|
retention_period: 168h
|
||||||
|
max_query_series: 5000
|
||||||
|
ingestion_rate_mb: 16
|
||||||
|
ingestion_burst_size_mb: 32
|
||||||
|
reject_old_samples: true
|
||||||
|
reject_old_samples_max_age: 168h
|
||||||
|
volume_enabled: true
|
||||||
|
metric_aggregation_enabled: true
|
||||||
|
allow_structured_metadata: true
|
||||||
|
|
||||||
|
pattern_ingester:
|
||||||
|
enabled: true
|
||||||
|
metric_aggregation:
|
||||||
|
loki_address: localhost:3100
|
||||||
|
|
||||||
|
query_range:
|
||||||
|
align_queries_with_step: true
|
||||||
|
cache_results: true
|
||||||
|
results_cache:
|
||||||
|
cache:
|
||||||
|
embedded_cache:
|
||||||
|
enabled: true
|
||||||
|
max_size_mb: 100
|
||||||
|
|
||||||
|
frontend:
|
||||||
|
encoding: protobuf
|
||||||
|
|
||||||
|
ruler:
|
||||||
|
storage:
|
||||||
|
type: local
|
||||||
|
local:
|
||||||
|
directory: /loki/rules
|
||||||
|
rule_path: /loki/rules-tmp
|
||||||
|
enable_api: true
|
||||||
|
|
||||||
|
analytics:
|
||||||
|
reporting_enabled: false
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolumeClaim
|
||||||
|
metadata:
|
||||||
|
name: loki-data
|
||||||
|
namespace: observability
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: loki
|
||||||
|
app.kubernetes.io/component: storage
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 10Gi
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: loki
|
||||||
|
namespace: observability
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: loki
|
||||||
|
app.kubernetes.io/component: logs
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
selector:
|
||||||
|
app: loki
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 3100
|
||||||
|
targetPort: http
|
||||||
|
- name: grpc
|
||||||
|
port: 9096
|
||||||
|
targetPort: grpc
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: observability
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: prometheus
|
||||||
|
namespace: observability
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: prometheus
|
||||||
|
app.kubernetes.io/component: metrics
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
strategy:
|
||||||
|
# ReadWriteOnce PVC: the old pod must release the volume first.
|
||||||
|
type: Recreate
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: prometheus
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: prometheus
|
||||||
|
app.kubernetes.io/name: prometheus
|
||||||
|
app.kubernetes.io/component: metrics
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
annotations:
|
||||||
|
prometheus.io/scrape: "true"
|
||||||
|
prometheus.io/port: "9090"
|
||||||
|
spec:
|
||||||
|
serviceAccountName: prometheus
|
||||||
|
securityContext:
|
||||||
|
runAsUser: 65534
|
||||||
|
runAsGroup: 65534
|
||||||
|
fsGroup: 65534
|
||||||
|
containers:
|
||||||
|
- name: prometheus
|
||||||
|
image: prom/prometheus:v3.14.0
|
||||||
|
args:
|
||||||
|
- --config.file=/etc/prometheus/prometheus.yaml
|
||||||
|
- --storage.tsdb.path=/prometheus
|
||||||
|
- --storage.tsdb.retention.time=15d
|
||||||
|
- --storage.tsdb.retention.size=8GB
|
||||||
|
- --web.enable-lifecycle
|
||||||
|
# Receives remote_write from Tempo's metrics_generator (span metrics + service graphs).
|
||||||
|
- --web.enable-remote-write-receiver
|
||||||
|
- --web.external-url=https://prometheus.sttlab.local
|
||||||
|
- --web.route-prefix=/
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 9090
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 512Mi
|
||||||
|
limits:
|
||||||
|
cpu: "2"
|
||||||
|
memory: 2Gi
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /-/ready
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 10
|
||||||
|
periodSeconds: 10
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /-/healthy
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 30
|
||||||
|
periodSeconds: 30
|
||||||
|
volumeMounts:
|
||||||
|
- name: config
|
||||||
|
mountPath: /etc/prometheus
|
||||||
|
- name: data
|
||||||
|
mountPath: /prometheus
|
||||||
|
- name: librechat-metrics
|
||||||
|
mountPath: /etc/prometheus/secrets/librechat
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: config
|
||||||
|
configMap:
|
||||||
|
name: prometheus-config
|
||||||
|
- name: librechat-metrics
|
||||||
|
secret:
|
||||||
|
secretName: librechat-metrics
|
||||||
|
- name: data
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: prometheus-data
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
namespace: observability
|
||||||
|
|
||||||
|
resources:
|
||||||
|
- secret.yaml
|
||||||
|
- rbac.yaml
|
||||||
|
- pvc.yaml
|
||||||
|
- deployment.yaml
|
||||||
|
- service.yaml
|
||||||
|
|
||||||
|
configMapGenerator:
|
||||||
|
- name: prometheus-config
|
||||||
|
files:
|
||||||
|
- prometheus.yaml
|
||||||
|
|
||||||
|
generatorOptions:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: prometheus
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
@@ -0,0 +1,177 @@
|
|||||||
|
global:
|
||||||
|
scrape_interval: 30s
|
||||||
|
evaluation_interval: 30s
|
||||||
|
external_labels:
|
||||||
|
cluster: orbstack
|
||||||
|
source: prometheus
|
||||||
|
|
||||||
|
scrape_configs:
|
||||||
|
# --- Control plane -------------------------------------------------------
|
||||||
|
- job_name: kubernetes-apiservers
|
||||||
|
kubernetes_sd_configs:
|
||||||
|
- role: endpoints
|
||||||
|
scheme: https
|
||||||
|
tls_config:
|
||||||
|
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||||
|
insecure_skip_verify: true
|
||||||
|
authorization:
|
||||||
|
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||||
|
relabel_configs:
|
||||||
|
- source_labels:
|
||||||
|
[__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
|
||||||
|
action: keep
|
||||||
|
regex: default;kubernetes;https
|
||||||
|
|
||||||
|
# --- Kubelet -------------------------------------------------------------
|
||||||
|
- job_name: kubernetes-nodes
|
||||||
|
kubernetes_sd_configs:
|
||||||
|
- role: node
|
||||||
|
scheme: https
|
||||||
|
tls_config:
|
||||||
|
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||||
|
insecure_skip_verify: true
|
||||||
|
authorization:
|
||||||
|
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||||
|
relabel_configs:
|
||||||
|
- action: labelmap
|
||||||
|
regex: __meta_kubernetes_node_label_(.+)
|
||||||
|
- target_label: __address__
|
||||||
|
replacement: kubernetes.default.svc:443
|
||||||
|
- source_labels: [__meta_kubernetes_node_name]
|
||||||
|
regex: (.+)
|
||||||
|
target_label: __metrics_path__
|
||||||
|
replacement: /api/v1/nodes/$1/proxy/metrics
|
||||||
|
|
||||||
|
# --- Container resource metrics -----------------------------------------
|
||||||
|
# /metrics/resource and NOT /metrics/cadvisor: on this cluster (OrbStack) the
|
||||||
|
# kubelet's cadvisor endpoint only exposes `machine_*` series and no
|
||||||
|
# `container_*` at all. /metrics/resource -- the source metrics-server reads --
|
||||||
|
# provides container_cpu_usage_seconds_total and
|
||||||
|
# container_memory_working_set_bytes with namespace/pod/container labels.
|
||||||
|
- job_name: kubernetes-kubelet-resource
|
||||||
|
kubernetes_sd_configs:
|
||||||
|
- role: node
|
||||||
|
scheme: https
|
||||||
|
tls_config:
|
||||||
|
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||||
|
insecure_skip_verify: true
|
||||||
|
authorization:
|
||||||
|
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||||
|
relabel_configs:
|
||||||
|
- action: labelmap
|
||||||
|
regex: __meta_kubernetes_node_label_(.+)
|
||||||
|
- target_label: __address__
|
||||||
|
replacement: kubernetes.default.svc:443
|
||||||
|
- source_labels: [__meta_kubernetes_node_name]
|
||||||
|
regex: (.+)
|
||||||
|
target_label: __metrics_path__
|
||||||
|
replacement: /api/v1/nodes/$1/proxy/metrics/resource
|
||||||
|
- source_labels: [__meta_kubernetes_node_name]
|
||||||
|
target_label: node
|
||||||
|
|
||||||
|
# --- Services annotated with prometheus.io/scrape -----------------------
|
||||||
|
- job_name: kubernetes-service-endpoints
|
||||||
|
kubernetes_sd_configs:
|
||||||
|
- role: endpoints
|
||||||
|
relabel_configs:
|
||||||
|
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
|
||||||
|
action: keep
|
||||||
|
regex: "true"
|
||||||
|
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
|
||||||
|
action: replace
|
||||||
|
target_label: __scheme__
|
||||||
|
regex: (https?)
|
||||||
|
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
|
||||||
|
action: replace
|
||||||
|
target_label: __metrics_path__
|
||||||
|
regex: (.+)
|
||||||
|
- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
|
||||||
|
action: replace
|
||||||
|
target_label: __address__
|
||||||
|
regex: ([^:]+)(?::\d+)?;(\d+)
|
||||||
|
replacement: $1:$2
|
||||||
|
- action: labelmap
|
||||||
|
regex: __meta_kubernetes_service_label_(.+)
|
||||||
|
- source_labels: [__meta_kubernetes_namespace]
|
||||||
|
target_label: namespace
|
||||||
|
- source_labels: [__meta_kubernetes_service_name]
|
||||||
|
target_label: service
|
||||||
|
- source_labels: [__meta_kubernetes_pod_name]
|
||||||
|
target_label: pod
|
||||||
|
- source_labels: [__meta_kubernetes_pod_node_name]
|
||||||
|
target_label: node
|
||||||
|
|
||||||
|
# --- Pods annotated with prometheus.io/scrape ---------------------------
|
||||||
|
- job_name: kubernetes-pods
|
||||||
|
kubernetes_sd_configs:
|
||||||
|
- role: pod
|
||||||
|
relabel_configs:
|
||||||
|
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
|
||||||
|
action: keep
|
||||||
|
regex: "true"
|
||||||
|
# A pod yields one target per container: without this filter the init
|
||||||
|
# containers duplicate the real target.
|
||||||
|
- source_labels: [__meta_kubernetes_pod_container_init]
|
||||||
|
action: drop
|
||||||
|
regex: "true"
|
||||||
|
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
|
||||||
|
action: replace
|
||||||
|
target_label: __scheme__
|
||||||
|
regex: (https?)
|
||||||
|
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
|
||||||
|
action: replace
|
||||||
|
target_label: __metrics_path__
|
||||||
|
regex: (.+)
|
||||||
|
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
|
||||||
|
action: replace
|
||||||
|
target_label: __address__
|
||||||
|
regex: ([^:]+)(?::\d+)?;(\d+)
|
||||||
|
replacement: $1:$2
|
||||||
|
- action: labelmap
|
||||||
|
regex: __meta_kubernetes_pod_label_(.+)
|
||||||
|
- source_labels: [__meta_kubernetes_namespace]
|
||||||
|
target_label: namespace
|
||||||
|
- source_labels: [__meta_kubernetes_pod_name]
|
||||||
|
target_label: pod
|
||||||
|
- source_labels: [__meta_kubernetes_pod_node_name]
|
||||||
|
target_label: node
|
||||||
|
- source_labels: [__meta_kubernetes_pod_container_name]
|
||||||
|
target_label: container
|
||||||
|
- source_labels: [__meta_kubernetes_pod_phase]
|
||||||
|
action: drop
|
||||||
|
regex: (Succeeded|Failed|Pending)
|
||||||
|
|
||||||
|
# --- LibreChat: /metrics behind a Bearer token --------------------------
|
||||||
|
# Annotation-based discovery cannot carry credentials, so LibreChat gets its
|
||||||
|
# own job. The token comes from the `librechat-metrics` secret mounted into
|
||||||
|
# the Prometheus pod and must match METRICS_SECRET on the LibreChat side.
|
||||||
|
- job_name: librechat
|
||||||
|
kubernetes_sd_configs:
|
||||||
|
- role: pod
|
||||||
|
namespaces:
|
||||||
|
names: [librechat]
|
||||||
|
authorization:
|
||||||
|
type: Bearer
|
||||||
|
credentials_file: /etc/prometheus/secrets/librechat/token
|
||||||
|
relabel_configs:
|
||||||
|
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape_with_auth]
|
||||||
|
action: keep
|
||||||
|
regex: "true"
|
||||||
|
# A pod yields one target per container: without this filter the init
|
||||||
|
# containers duplicate the real target.
|
||||||
|
- source_labels: [__meta_kubernetes_pod_container_init]
|
||||||
|
action: drop
|
||||||
|
regex: "true"
|
||||||
|
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
|
||||||
|
action: replace
|
||||||
|
target_label: __address__
|
||||||
|
regex: ([^:]+)(?::\d+)?;(\d+)
|
||||||
|
replacement: $1:$2
|
||||||
|
- source_labels: [__meta_kubernetes_namespace]
|
||||||
|
target_label: namespace
|
||||||
|
- source_labels: [__meta_kubernetes_pod_name]
|
||||||
|
target_label: pod
|
||||||
|
- source_labels: [__meta_kubernetes_pod_container_name]
|
||||||
|
target_label: container
|
||||||
|
- target_label: service
|
||||||
|
replacement: librechat
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolumeClaim
|
||||||
|
metadata:
|
||||||
|
name: prometheus-data
|
||||||
|
namespace: observability
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: prometheus
|
||||||
|
app.kubernetes.io/component: storage
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 10Gi
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: prometheus
|
||||||
|
namespace: observability
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: prometheus
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: observability-prometheus
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: prometheus
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
rules:
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources:
|
||||||
|
- nodes
|
||||||
|
- nodes/metrics
|
||||||
|
- nodes/proxy
|
||||||
|
- services
|
||||||
|
- endpoints
|
||||||
|
- pods
|
||||||
|
- namespaces
|
||||||
|
verbs: ["get", "list", "watch"]
|
||||||
|
- apiGroups: ["discovery.k8s.io"]
|
||||||
|
resources: ["endpointslices"]
|
||||||
|
verbs: ["get", "list", "watch"]
|
||||||
|
- apiGroups: ["networking.k8s.io"]
|
||||||
|
resources: ["ingresses"]
|
||||||
|
verbs: ["get", "list", "watch"]
|
||||||
|
- nonResourceURLs: ["/metrics", "/metrics/cadvisor"]
|
||||||
|
verbs: ["get"]
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
metadata:
|
||||||
|
name: observability-prometheus
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: prometheus
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: ClusterRole
|
||||||
|
name: observability-prometheus
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: prometheus
|
||||||
|
namespace: observability
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
# Template. The token must be IDENTICAL to the METRICS_SECRET key of the
|
||||||
|
# `librechat-secrets` secret in the librechat namespace (infra-librechat repo):
|
||||||
|
# it is the Bearer token Prometheus uses to scrape LibreChat's /metrics.
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Secret
|
||||||
|
metadata:
|
||||||
|
name: librechat-metrics
|
||||||
|
namespace: observability
|
||||||
|
type: Opaque
|
||||||
|
stringData:
|
||||||
|
token: <CHANGE_ME>
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: prometheus
|
||||||
|
namespace: observability
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: prometheus
|
||||||
|
app.kubernetes.io/component: metrics
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
selector:
|
||||||
|
app: prometheus
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 9090
|
||||||
|
targetPort: http
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
# Template: copy to secret.yaml (gitignored) and fill in the values.
|
||||||
|
# make secret
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Secret
|
||||||
|
metadata:
|
||||||
|
name: observability-secrets
|
||||||
|
namespace: observability
|
||||||
|
type: Opaque
|
||||||
|
stringData:
|
||||||
|
GF_SECURITY_ADMIN_USER: admin
|
||||||
|
GF_SECURITY_ADMIN_PASSWORD: <CHANGE_ME>
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: tempo
|
||||||
|
namespace: observability
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: tempo
|
||||||
|
app.kubernetes.io/component: traces
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
strategy:
|
||||||
|
type: Recreate
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: tempo
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: tempo
|
||||||
|
app.kubernetes.io/name: tempo
|
||||||
|
app.kubernetes.io/component: traces
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
annotations:
|
||||||
|
prometheus.io/scrape: "true"
|
||||||
|
prometheus.io/port: "3200"
|
||||||
|
spec:
|
||||||
|
securityContext:
|
||||||
|
runAsUser: 10001
|
||||||
|
runAsGroup: 10001
|
||||||
|
fsGroup: 10001
|
||||||
|
containers:
|
||||||
|
- name: tempo
|
||||||
|
image: grafana/tempo:3.0.3
|
||||||
|
args:
|
||||||
|
- -config.file=/etc/tempo/tempo.yaml
|
||||||
|
- -target=all
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 3200
|
||||||
|
- name: grpc
|
||||||
|
containerPort: 9095
|
||||||
|
- name: otlp-grpc
|
||||||
|
containerPort: 4317
|
||||||
|
- name: otlp-http
|
||||||
|
containerPort: 4318
|
||||||
|
- name: zipkin
|
||||||
|
containerPort: 9411
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 512Mi
|
||||||
|
limits:
|
||||||
|
cpu: "2"
|
||||||
|
memory: 2Gi
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /ready
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 30
|
||||||
|
periodSeconds: 10
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /ready
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 30
|
||||||
|
volumeMounts:
|
||||||
|
- name: config
|
||||||
|
mountPath: /etc/tempo
|
||||||
|
- name: data
|
||||||
|
mountPath: /var/tempo
|
||||||
|
volumes:
|
||||||
|
- name: config
|
||||||
|
configMap:
|
||||||
|
name: tempo-config
|
||||||
|
- name: data
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: tempo-data
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
namespace: observability
|
||||||
|
|
||||||
|
resources:
|
||||||
|
- pvc.yaml
|
||||||
|
- deployment.yaml
|
||||||
|
- service.yaml
|
||||||
|
|
||||||
|
configMapGenerator:
|
||||||
|
- name: tempo-config
|
||||||
|
files:
|
||||||
|
- tempo.yaml
|
||||||
|
|
||||||
|
generatorOptions:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: tempo
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolumeClaim
|
||||||
|
metadata:
|
||||||
|
name: tempo-data
|
||||||
|
namespace: observability
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: tempo
|
||||||
|
app.kubernetes.io/component: storage
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 10Gi
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: tempo
|
||||||
|
namespace: observability
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: tempo
|
||||||
|
app.kubernetes.io/component: traces
|
||||||
|
app.kubernetes.io/part-of: observability
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
selector:
|
||||||
|
app: tempo
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 3200
|
||||||
|
targetPort: http
|
||||||
|
- name: grpc
|
||||||
|
port: 9095
|
||||||
|
targetPort: grpc
|
||||||
|
- name: otlp-grpc
|
||||||
|
port: 4317
|
||||||
|
targetPort: otlp-grpc
|
||||||
|
- name: otlp-http
|
||||||
|
port: 4318
|
||||||
|
targetPort: otlp-http
|
||||||
|
- name: zipkin
|
||||||
|
port: 9411
|
||||||
|
targetPort: zipkin
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
# Tempo 3.x in monolithic mode (-target=all), local storage on a PVC.
|
||||||
|
stream_over_http_enabled: true
|
||||||
|
multitenancy_enabled: false
|
||||||
|
|
||||||
|
server:
|
||||||
|
http_listen_address: 0.0.0.0
|
||||||
|
http_listen_port: 3200
|
||||||
|
grpc_listen_address: 0.0.0.0
|
||||||
|
grpc_listen_port: 9095
|
||||||
|
log_level: info
|
||||||
|
|
||||||
|
distributor:
|
||||||
|
receivers:
|
||||||
|
otlp:
|
||||||
|
protocols:
|
||||||
|
grpc:
|
||||||
|
endpoint: 0.0.0.0:4317
|
||||||
|
http:
|
||||||
|
endpoint: 0.0.0.0:4318
|
||||||
|
zipkin:
|
||||||
|
endpoint: 0.0.0.0:9411
|
||||||
|
|
||||||
|
# Generates span metrics and service graphs, pushed to Prometheus through its
|
||||||
|
# remote-write receiver (--web.enable-remote-write-receiver).
|
||||||
|
metrics_generator:
|
||||||
|
registry:
|
||||||
|
external_labels:
|
||||||
|
source: tempo
|
||||||
|
cluster: orbstack
|
||||||
|
storage:
|
||||||
|
path: /var/tempo/generator/wal
|
||||||
|
remote_write:
|
||||||
|
- url: http://prometheus:9090/api/v1/write
|
||||||
|
send_exemplars: true
|
||||||
|
|
||||||
|
query_frontend:
|
||||||
|
search:
|
||||||
|
duration_slo: 5s
|
||||||
|
throughput_bytes_slo: 1073741824
|
||||||
|
trace_by_id:
|
||||||
|
duration_slo: 5s
|
||||||
|
|
||||||
|
storage:
|
||||||
|
trace:
|
||||||
|
backend: local
|
||||||
|
wal:
|
||||||
|
path: /var/tempo/wal
|
||||||
|
local:
|
||||||
|
path: /var/tempo/blocks
|
||||||
|
|
||||||
|
overrides:
|
||||||
|
defaults:
|
||||||
|
metrics_generator:
|
||||||
|
processors:
|
||||||
|
- service-graphs
|
||||||
|
- span-metrics
|
||||||
|
generate_native_histograms: both
|
||||||
|
# Tempo 3.x: retention is no longer a top-level `compactor:` block; it is
|
||||||
|
# driven from here (backend_scheduler/backend_worker otherwise).
|
||||||
|
compaction:
|
||||||
|
block_retention: 168h
|
||||||
|
|
||||||
|
usage_report:
|
||||||
|
reporting_enabled: false
|
||||||
Reference in New Issue
Block a user