stt/infra-observability
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Declarative Grafana / Prometheus / Loki / Tempo observability stack

Browse Source 
Deploys the whole stack into the observability namespace of the local
cluster with Kustomize alone — no helm install, no dashboard created by
hand in the UI. The Makefile carries every command: deploy, dashboards,
diagnostics, PVC flush, teardown.

Dashboards are files, never UI objects. Each carries a stable uid, which
is what keeps URLs, cross-links and alerts alive across redeployments, and
references datasources by stable uid rather than by name. The dashboard
ConfigMap is generated without a name-suffix hash so applying it updates
it in place and Grafana reloads without a restart, while the provisioning
ConfigMaps keep their hash precisely so that a change there does force one.

Prometheus scrapes the kubelet /metrics/resource endpoint rather than
/metrics/cadvisor: on this cluster cadvisor exposes machine_* series only
and no container_* at all, which would leave every CPU and memory panel
empty.

Alloy ships every pod's logs to Loki through the Kubernetes API, so no
/var/log mount is needed. Tempo's metrics_generator feeds span metrics and
the service graph back into Prometheus, and the datasources are wired so a
log line's trace_id opens the matching trace.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Stéphane Tailland
2026-08-25 07:27:38 +02:00
commit 3937b4f366
42 changed files with 7597 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
k8s/prometheus/deployment.yaml
+86
View File
@@ -0,0 +1,86 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: prometheus
namespace: observability
labels:
app.kubernetes.io/name: prometheus
app.kubernetes.io/component: metrics
app.kubernetes.io/part-of: observability
spec:
replicas: 1
strategy:
# ReadWriteOnce PVC: the old pod must release the volume first.
type: Recreate
selector:
matchLabels:
app: prometheus
template:
metadata:
labels:
app: prometheus
app.kubernetes.io/name: prometheus
app.kubernetes.io/component: metrics
app.kubernetes.io/part-of: observability
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "9090"
spec:
serviceAccountName: prometheus
securityContext:
runAsUser: 65534
runAsGroup: 65534
fsGroup: 65534
containers:
- name: prometheus
image: prom/prometheus:v3.14.0
args:
- --config.file=/etc/prometheus/prometheus.yaml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=15d
- --storage.tsdb.retention.size=8GB
- --web.enable-lifecycle
# Receives remote_write from Tempo's metrics_generator (span metrics + service graphs).
- --web.enable-remote-write-receiver
- --web.external-url=https://prometheus.sttlab.local
- --web.route-prefix=/
ports:
- name: http
containerPort: 9090
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
cpu: "2"
memory: 2Gi
readinessProbe:
httpGet:
path: /-/ready
port: http
initialDelaySeconds: 10
periodSeconds: 10
livenessProbe:
httpGet:
path: /-/healthy
port: http
initialDelaySeconds: 30
periodSeconds: 30
volumeMounts:
- name: config
mountPath: /etc/prometheus
- name: data
mountPath: /prometheus
- name: librechat-metrics
mountPath: /etc/prometheus/secrets/librechat
readOnly: true
volumes:
- name: config
configMap:
name: prometheus-config
- name: librechat-metrics
secret:
secretName: librechat-metrics
- name: data
persistentVolumeClaim:
claimName: prometheus-data
k8s/prometheus/kustomization.yaml
+20
View File
@@ -0,0 +1,20 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: observability
resources:
- secret.yaml
- rbac.yaml
- pvc.yaml
- deployment.yaml
- service.yaml
configMapGenerator:
- name: prometheus-config
files:
- prometheus.yaml
generatorOptions:
labels:
app.kubernetes.io/name: prometheus
app.kubernetes.io/part-of: observability
k8s/prometheus/prometheus.yaml
+177
View File
@@ -0,0 +1,177 @@
global:
scrape_interval: 30s
evaluation_interval: 30s
external_labels:
cluster: orbstack
source: prometheus
scrape_configs:
# --- Control plane -------------------------------------------------------
- job_name: kubernetes-apiservers
kubernetes_sd_configs:
- role: endpoints
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: true
authorization:
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- source_labels:
[__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
action: keep
regex: default;kubernetes;https
# --- Kubelet -------------------------------------------------------------
- job_name: kubernetes-nodes
kubernetes_sd_configs:
- role: node
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: true
authorization:
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: kubernetes.default.svc:443
- source_labels: [__meta_kubernetes_node_name]
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/$1/proxy/metrics
# --- Container resource metrics -----------------------------------------
# /metrics/resource and NOT /metrics/cadvisor: on this cluster (OrbStack) the
# kubelet's cadvisor endpoint only exposes `machine_*` series and no
# `container_*` at all. /metrics/resource -- the source metrics-server reads --
# provides container_cpu_usage_seconds_total and
# container_memory_working_set_bytes with namespace/pod/container labels.
- job_name: kubernetes-kubelet-resource
kubernetes_sd_configs:
- role: node
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: true
authorization:
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: kubernetes.default.svc:443
- source_labels: [__meta_kubernetes_node_name]
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/$1/proxy/metrics/resource
- source_labels: [__meta_kubernetes_node_name]
target_label: node
# --- Services annotated with prometheus.io/scrape -----------------------
- job_name: kubernetes-service-endpoints
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
action: keep
regex: "true"
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
action: replace
target_label: __scheme__
regex: (https?)
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
action: replace
target_label: __address__
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
target_label: namespace
- source_labels: [__meta_kubernetes_service_name]
target_label: service
- source_labels: [__meta_kubernetes_pod_name]
target_label: pod
- source_labels: [__meta_kubernetes_pod_node_name]
target_label: node
# --- Pods annotated with prometheus.io/scrape ---------------------------
- job_name: kubernetes-pods
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
action: keep
regex: "true"
# A pod yields one target per container: without this filter the init
# containers duplicate the real target.
- source_labels: [__meta_kubernetes_pod_container_init]
action: drop
regex: "true"
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
action: replace
target_label: __scheme__
regex: (https?)
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
action: replace
target_label: __address__
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
target_label: namespace
- source_labels: [__meta_kubernetes_pod_name]
target_label: pod
- source_labels: [__meta_kubernetes_pod_node_name]
target_label: node
- source_labels: [__meta_kubernetes_pod_container_name]
target_label: container
- source_labels: [__meta_kubernetes_pod_phase]
action: drop
regex: (Succeeded|Failed|Pending)
# --- LibreChat: /metrics behind a Bearer token --------------------------
# Annotation-based discovery cannot carry credentials, so LibreChat gets its
# own job. The token comes from the `librechat-metrics` secret mounted into
# the Prometheus pod and must match METRICS_SECRET on the LibreChat side.
- job_name: librechat
kubernetes_sd_configs:
- role: pod
namespaces:
names: [librechat]
authorization:
type: Bearer
credentials_file: /etc/prometheus/secrets/librechat/token
relabel_configs:
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape_with_auth]
action: keep
regex: "true"
# A pod yields one target per container: without this filter the init
# containers duplicate the real target.
- source_labels: [__meta_kubernetes_pod_container_init]
action: drop
regex: "true"
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
action: replace
target_label: __address__
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
- source_labels: [__meta_kubernetes_namespace]
target_label: namespace
- source_labels: [__meta_kubernetes_pod_name]
target_label: pod
- source_labels: [__meta_kubernetes_pod_container_name]
target_label: container
- target_label: service
replacement: librechat
k8s/prometheus/pvc.yaml
+15
View File
@@ -0,0 +1,15 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: prometheus-data
namespace: observability
labels:
app.kubernetes.io/name: prometheus
app.kubernetes.io/component: storage
app.kubernetes.io/part-of: observability
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
k8s/prometheus/rbac.yaml
+51
View File
@@ -0,0 +1,51 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: prometheus
namespace: observability
labels:
app.kubernetes.io/name: prometheus
app.kubernetes.io/part-of: observability
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: observability-prometheus
labels:
app.kubernetes.io/name: prometheus
app.kubernetes.io/part-of: observability
rules:
- apiGroups: [""]
resources:
- nodes
- nodes/metrics
- nodes/proxy
- services
- endpoints
- pods
- namespaces
verbs: ["get", "list", "watch"]
- apiGroups: ["discovery.k8s.io"]
resources: ["endpointslices"]
verbs: ["get", "list", "watch"]
- apiGroups: ["networking.k8s.io"]
resources: ["ingresses"]
verbs: ["get", "list", "watch"]
- nonResourceURLs: ["/metrics", "/metrics/cadvisor"]
verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: observability-prometheus
labels:
app.kubernetes.io/name: prometheus
app.kubernetes.io/part-of: observability
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: observability-prometheus
subjects:
- kind: ServiceAccount
name: prometheus
namespace: observability
k8s/prometheus/secret.example.yaml
+11
View File
@@ -0,0 +1,11 @@
# Template. The token must be IDENTICAL to the METRICS_SECRET key of the
# `librechat-secrets` secret in the librechat namespace (infra-librechat repo):
# it is the Bearer token Prometheus uses to scrape LibreChat's /metrics.
apiVersion: v1
kind: Secret
metadata:
name: librechat-metrics
namespace: observability
type: Opaque
stringData:
token: <CHANGE_ME>
k8s/prometheus/service.yaml
+17
View File
@@ -0,0 +1,17 @@
apiVersion: v1
kind: Service
metadata:
name: prometheus
namespace: observability
labels:
app.kubernetes.io/name: prometheus
app.kubernetes.io/component: metrics
app.kubernetes.io/part-of: observability
spec:
type: ClusterIP
selector:
app: prometheus
ports:
- name: http
port: 9090
targetPort: http