Declarative Grafana / Prometheus / Loki / Tempo observability stack

Deploys the whole stack into the observability namespace of the local
cluster with Kustomize alone — no helm install, no dashboard created by
hand in the UI. The Makefile carries every command: deploy, dashboards,
diagnostics, PVC flush, teardown.

Dashboards are files, never UI objects. Each carries a stable uid, which
is what keeps URLs, cross-links and alerts alive across redeployments, and
references datasources by stable uid rather than by name. The dashboard
ConfigMap is generated without a name-suffix hash so applying it updates
it in place and Grafana reloads without a restart, while the provisioning
ConfigMaps keep their hash precisely so that a change there does force one.

Prometheus scrapes the kubelet /metrics/resource endpoint rather than
/metrics/cadvisor: on this cluster cadvisor exposes machine_* series only
and no container_* at all, which would leave every CPU and memory panel
empty.

Alloy ships every pod's logs to Loki through the Kubernetes API, so no
/var/log mount is needed. Tempo's metrics_generator feeds span metrics and
the service graph back into Prometheus, and the datasources are wired so a
log line's trace_id opens the matching trace.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Stéphane Tailland
2026-08-25 07:27:38 +02:00
commit 3937b4f366
42 changed files with 7597 additions and 0 deletions
+31
View File
@@ -0,0 +1,31 @@
# Grafana dashboards -- declarative provisioning.
#
# Rules:
# * one .json file per dashboard, named after its UID;
# * every JSON carries a STABLE "uid" field (never regenerated): that is what
# keeps URLs, cross-links and alerts alive across redeployments;
# * `disableNameSuffixHash: true` -> the ConfigMap always keeps the same name,
# so `kubectl apply` updates it in place and Grafana reloads WITHOUT
# restarting the pod, in about 70s (see dashboard-providers.yaml).
#
# Adding a dashboard = drop the .json here and add it to `files:` below.
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: observability
configMapGenerator:
- name: grafana-dashboards
files:
# >>> dashboards (generated by `make dashboards-sync`) >>>
- obs-classic-models-api.json
- obs-keycloak.json
- obs-librechat.json
- obs-stack-health.json
# <<< dashboards <<<
generatorOptions:
disableNameSuffixHash: true
labels:
app.kubernetes.io/name: grafana
app.kubernetes.io/component: dashboards
app.kubernetes.io/part-of: observability
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,437 @@
{
"uid": "obs-stack-health",
"title": "Observability — Stack Health",
"description": "Health of the Prometheus / Loki / Tempo / Alloy stack. Reference dashboard: stable UID, declaratively provisioned.",
"tags": [
"observability",
"stack",
"provisioned"
],
"editable": false,
"schemaVersion": 39,
"version": 1,
"refresh": "30s",
"time": {
"from": "now-1h",
"to": "now"
},
"timezone": "browser",
"templating": {
"list": [
{
"name": "namespace",
"label": "Namespace",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"query": "label_values(kube_pod_info, namespace)",
"refresh": 2,
"includeAll": true,
"multi": true,
"current": {
"text": "observability",
"value": "observability"
}
}
]
},
"panels": [
{
"id": 1,
"type": "row",
"title": "Collection",
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"collapsed": false,
"panels": []
},
{
"id": 2,
"type": "stat",
"title": "Prometheus targets UP",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 5,
"w": 6,
"x": 0,
"y": 1
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum(up)",
"legendFormat": "up"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"textMode": "auto"
}
},
{
"id": 3,
"type": "stat",
"title": "Prometheus targets DOWN",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 5,
"w": 6,
"x": 6,
"y": 1
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum(up == bool 0)",
"legendFormat": "down"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "none",
"textMode": "auto"
}
},
{
"id": 4,
"type": "stat",
"title": "Log lines ingested (/s)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 5,
"w": 6,
"x": 12,
"y": 1
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum(rate(loki_distributor_lines_received_total[5m]))",
"legendFormat": "lines/s"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 1,
"color": {
"mode": "fixed",
"fixedColor": "blue"
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"textMode": "auto"
}
},
{
"id": 5,
"type": "stat",
"title": "Spans ingested (/s)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 5,
"w": 6,
"x": 18,
"y": 1
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum(rate(tempo_distributor_spans_received_total[5m]))",
"legendFormat": "spans/s"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 1,
"color": {
"mode": "fixed",
"fixedColor": "purple"
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"textMode": "auto"
}
},
{
"id": 6,
"type": "row",
"title": "Stack resources",
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 6
},
"collapsed": false,
"panels": []
},
{
"id": 7,
"type": "timeseries",
"title": "CPU per component (observability namespace)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 7
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"observability\", container!=\"\"}[$__rate_interval]))",
"legendFormat": "{{pod}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"mean",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 8,
"type": "timeseries",
"title": "Memory per component (observability namespace)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 7
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"observability\", container!=\"\"})",
"legendFormat": "{{pod}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"mean",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 9,
"type": "row",
"title": "Logs",
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 15
},
"collapsed": false,
"panels": []
},
{
"id": 10,
"type": "logs",
"title": "Recent stack errors",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 16
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "{namespace=\"observability\"} |~ \"(?i)(error|level=error|fatal)\"",
"queryType": "range"
}
],
"options": {
"showTime": true,
"showLabels": false,
"wrapLogMessage": true,
"sortOrder": "Descending",
"dedupStrategy": "none",
"enableLogDetails": true
}
}
]
}
+112
View File
@@ -0,0 +1,112 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: grafana
namespace: observability
labels:
app.kubernetes.io/name: grafana
app.kubernetes.io/component: ui
app.kubernetes.io/part-of: observability
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app: grafana
template:
metadata:
labels:
app: grafana
app.kubernetes.io/name: grafana
app.kubernetes.io/component: ui
app.kubernetes.io/part-of: observability
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "3000"
prometheus.io/path: "/metrics"
spec:
securityContext:
runAsUser: 472
runAsGroup: 472
fsGroup: 472
containers:
- name: grafana
image: grafana/grafana:13.2.0
ports:
- name: http
containerPort: 3000
env:
- name: GF_SECURITY_ADMIN_USER
valueFrom:
secretKeyRef:
name: observability-secrets
key: GF_SECURITY_ADMIN_USER
- name: GF_SECURITY_ADMIN_PASSWORD
valueFrom:
secretKeyRef:
name: observability-secrets
key: GF_SECURITY_ADMIN_PASSWORD
- name: GF_SERVER_ROOT_URL
value: https://grafana.sttlab.local
- name: GF_PATHS_PROVISIONING
value: /etc/grafana/provisioning
# Everything is driven by the provisioning files, so creating
# dashboards or datasources from the UI is disabled.
- name: GF_USERS_ALLOW_SIGN_UP
value: "false"
- name: GF_ANALYTICS_REPORTING_ENABLED
value: "false"
- name: GF_ANALYTICS_CHECK_FOR_UPDATES
value: "false"
- name: GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES
value: "false"
- name: GF_NEWS_NEWS_FEED_ENABLED
value: "false"
- name: GF_LOG_LEVEL
value: info
- name: GF_FEATURE_TOGGLES_ENABLE
value: traceqlEditor,traceQLStreaming,correlations,lokiLogsDataplane
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: "1"
memory: 1Gi
readinessProbe:
httpGet:
path: /api/health
port: http
initialDelaySeconds: 15
periodSeconds: 10
livenessProbe:
httpGet:
path: /api/health
port: http
initialDelaySeconds: 60
periodSeconds: 30
volumeMounts:
- name: data
mountPath: /var/lib/grafana
- name: provisioning-datasources
mountPath: /etc/grafana/provisioning/datasources
- name: provisioning-dashboards
mountPath: /etc/grafana/provisioning/dashboards
- name: dashboards
mountPath: /var/lib/grafana/dashboards
volumes:
- name: data
persistentVolumeClaim:
claimName: grafana-data
- name: provisioning-datasources
configMap:
name: grafana-datasources
- name: provisioning-dashboards
configMap:
name: grafana-dashboard-providers
- name: dashboards
# Stable-named ConfigMap: updated in place by `make dashboards` and
# re-read by Grafana without a restart (~70s: kubelet sync + 10s poll).
configMap:
name: grafana-dashboards
+27
View File
@@ -0,0 +1,27 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: observability
resources:
- pvc.yaml
- deployment.yaml
- service.yaml
# Dashboards live in ./dashboards and are referenced by the root kustomization:
# `k8s/grafana` is the runtime alone, `k8s/grafana/dashboards` the content alone.
# Both are deployable independently.
configMapGenerator:
# Name-suffix hash KEPT here: datasource and provider provisioning is only
# read at startup, so the hash deliberately forces a pod restart whenever
# these files change.
- name: grafana-datasources
files:
- datasources.yaml=provisioning/datasources.yaml
- name: grafana-dashboard-providers
files:
- dashboard-providers.yaml=provisioning/dashboard-providers.yaml
generatorOptions:
labels:
app.kubernetes.io/name: grafana
app.kubernetes.io/part-of: observability
@@ -0,0 +1,19 @@
# File-based dashboard provider.
# The JSON files are mounted from the `grafana-dashboards` ConfigMap (generated
# without a name-suffix hash): `make dashboards` updates the ConfigMap in place
# and Grafana reloads WITHOUT restarting the pod, in about 70s
# (kubelet volume sync ~60s + updateIntervalSeconds below).
apiVersion: 1
providers:
- name: infra-observability
orgId: 1
folder: Infra Observability
folderUid: infra-observability
type: file
disableDeletion: false
allowUiUpdates: false
updateIntervalSeconds: 10
options:
path: /var/lib/grafana/dashboards
foldersFromFilesStructure: false
+82
View File
@@ -0,0 +1,82 @@
# Declarative datasource provisioning.
# The `uid` values are STABLE and are what dashboard JSON files reference:
# "datasource": { "type": "prometheus", "uid": "prometheus" }
apiVersion: 1
deleteDatasources:
- name: Prometheus
orgId: 1
- name: Loki
orgId: 1
- name: Tempo
orgId: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
orgId: 1
url: http://prometheus:9090
isDefault: true
editable: false
jsonData:
httpMethod: POST
timeInterval: 30s
manageAlerts: false
exemplarTraceIdDestinations:
- name: trace_id
datasourceUid: tempo
- name: Loki
uid: loki
type: loki
access: proxy
orgId: 1
url: http://loki:3100
editable: false
jsonData:
timeout: 60
maxLines: 1000
# Logs -> traces: a trace_id in the log line opens Tempo.
derivedFields:
- name: TraceID
matcherType: regex
matcherRegex: '"?trace[_-]?id"?[:=]"?([a-f0-9]{16,32})'
url: '$${__value.raw}'
datasourceUid: tempo
urlDisplayLabel: View trace
- name: Tempo
uid: tempo
type: tempo
access: proxy
orgId: 1
url: http://tempo:3200
editable: false
jsonData:
httpMethod: GET
streamingEnabled:
search: true
metrics: true
# Service graph fed by the metrics_generator -> Prometheus.
serviceMap:
datasourceUid: prometheus
nodeGraph:
enabled: true
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: -5m
spanEndTimeShift: 5m
filterByTraceID: true
filterBySpanID: false
tags:
- key: service.name
value: app
tracesToMetrics:
datasourceUid: prometheus
spanStartTimeShift: -5m
spanEndTimeShift: 5m
tags:
- key: service.name
value: service
+15
View File
@@ -0,0 +1,15 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: grafana-data
namespace: observability
labels:
app.kubernetes.io/name: grafana
app.kubernetes.io/component: storage
app.kubernetes.io/part-of: observability
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 5Gi
+17
View File
@@ -0,0 +1,17 @@
apiVersion: v1
kind: Service
metadata:
name: grafana
namespace: observability
labels:
app.kubernetes.io/name: grafana
app.kubernetes.io/component: ui
app.kubernetes.io/part-of: observability
spec:
type: ClusterIP
selector:
app: grafana
ports:
- name: http
port: 3000
targetPort: http