Stéphane Tailland
3937b4f366
Deploys the whole stack into the observability namespace of the local cluster with Kustomize alone — no helm install, no dashboard created by hand in the UI. The Makefile carries every command: deploy, dashboards, diagnostics, PVC flush, teardown. Dashboards are files, never UI objects. Each carries a stable uid, which is what keeps URLs, cross-links and alerts alive across redeployments, and references datasources by stable uid rather than by name. The dashboard ConfigMap is generated without a name-suffix hash so applying it updates it in place and Grafana reloads without a restart, while the provisioning ConfigMaps keep their hash precisely so that a change there does force one. Prometheus scrapes the kubelet /metrics/resource endpoint rather than /metrics/cadvisor: on this cluster cadvisor exposes machine_* series only and no container_* at all, which would leave every CPU and memory panel empty. Alloy ships every pod's logs to Loki through the Kubernetes API, so no /var/log mount is needed. Tempo's metrics_generator feeds span metrics and the service graph back into Prometheus, and the datasources are wired so a log line's trace_id opens the matching trace. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
178 lines
6.8 KiB
YAML
178 lines
6.8 KiB
YAML
global:
|
|
scrape_interval: 30s
|
|
evaluation_interval: 30s
|
|
external_labels:
|
|
cluster: orbstack
|
|
source: prometheus
|
|
|
|
scrape_configs:
|
|
# --- Control plane -------------------------------------------------------
|
|
- job_name: kubernetes-apiservers
|
|
kubernetes_sd_configs:
|
|
- role: endpoints
|
|
scheme: https
|
|
tls_config:
|
|
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
|
insecure_skip_verify: true
|
|
authorization:
|
|
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
|
relabel_configs:
|
|
- source_labels:
|
|
[__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
|
|
action: keep
|
|
regex: default;kubernetes;https
|
|
|
|
# --- Kubelet -------------------------------------------------------------
|
|
- job_name: kubernetes-nodes
|
|
kubernetes_sd_configs:
|
|
- role: node
|
|
scheme: https
|
|
tls_config:
|
|
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
|
insecure_skip_verify: true
|
|
authorization:
|
|
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
|
relabel_configs:
|
|
- action: labelmap
|
|
regex: __meta_kubernetes_node_label_(.+)
|
|
- target_label: __address__
|
|
replacement: kubernetes.default.svc:443
|
|
- source_labels: [__meta_kubernetes_node_name]
|
|
regex: (.+)
|
|
target_label: __metrics_path__
|
|
replacement: /api/v1/nodes/$1/proxy/metrics
|
|
|
|
# --- Container resource metrics -----------------------------------------
|
|
# /metrics/resource and NOT /metrics/cadvisor: on this cluster (OrbStack) the
|
|
# kubelet's cadvisor endpoint only exposes `machine_*` series and no
|
|
# `container_*` at all. /metrics/resource -- the source metrics-server reads --
|
|
# provides container_cpu_usage_seconds_total and
|
|
# container_memory_working_set_bytes with namespace/pod/container labels.
|
|
- job_name: kubernetes-kubelet-resource
|
|
kubernetes_sd_configs:
|
|
- role: node
|
|
scheme: https
|
|
tls_config:
|
|
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
|
insecure_skip_verify: true
|
|
authorization:
|
|
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
|
relabel_configs:
|
|
- action: labelmap
|
|
regex: __meta_kubernetes_node_label_(.+)
|
|
- target_label: __address__
|
|
replacement: kubernetes.default.svc:443
|
|
- source_labels: [__meta_kubernetes_node_name]
|
|
regex: (.+)
|
|
target_label: __metrics_path__
|
|
replacement: /api/v1/nodes/$1/proxy/metrics/resource
|
|
- source_labels: [__meta_kubernetes_node_name]
|
|
target_label: node
|
|
|
|
# --- Services annotated with prometheus.io/scrape -----------------------
|
|
- job_name: kubernetes-service-endpoints
|
|
kubernetes_sd_configs:
|
|
- role: endpoints
|
|
relabel_configs:
|
|
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
|
|
action: keep
|
|
regex: "true"
|
|
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
|
|
action: replace
|
|
target_label: __scheme__
|
|
regex: (https?)
|
|
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
|
|
action: replace
|
|
target_label: __metrics_path__
|
|
regex: (.+)
|
|
- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
|
|
action: replace
|
|
target_label: __address__
|
|
regex: ([^:]+)(?::\d+)?;(\d+)
|
|
replacement: $1:$2
|
|
- action: labelmap
|
|
regex: __meta_kubernetes_service_label_(.+)
|
|
- source_labels: [__meta_kubernetes_namespace]
|
|
target_label: namespace
|
|
- source_labels: [__meta_kubernetes_service_name]
|
|
target_label: service
|
|
- source_labels: [__meta_kubernetes_pod_name]
|
|
target_label: pod
|
|
- source_labels: [__meta_kubernetes_pod_node_name]
|
|
target_label: node
|
|
|
|
# --- Pods annotated with prometheus.io/scrape ---------------------------
|
|
- job_name: kubernetes-pods
|
|
kubernetes_sd_configs:
|
|
- role: pod
|
|
relabel_configs:
|
|
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
|
|
action: keep
|
|
regex: "true"
|
|
# A pod yields one target per container: without this filter the init
|
|
# containers duplicate the real target.
|
|
- source_labels: [__meta_kubernetes_pod_container_init]
|
|
action: drop
|
|
regex: "true"
|
|
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
|
|
action: replace
|
|
target_label: __scheme__
|
|
regex: (https?)
|
|
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
|
|
action: replace
|
|
target_label: __metrics_path__
|
|
regex: (.+)
|
|
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
|
|
action: replace
|
|
target_label: __address__
|
|
regex: ([^:]+)(?::\d+)?;(\d+)
|
|
replacement: $1:$2
|
|
- action: labelmap
|
|
regex: __meta_kubernetes_pod_label_(.+)
|
|
- source_labels: [__meta_kubernetes_namespace]
|
|
target_label: namespace
|
|
- source_labels: [__meta_kubernetes_pod_name]
|
|
target_label: pod
|
|
- source_labels: [__meta_kubernetes_pod_node_name]
|
|
target_label: node
|
|
- source_labels: [__meta_kubernetes_pod_container_name]
|
|
target_label: container
|
|
- source_labels: [__meta_kubernetes_pod_phase]
|
|
action: drop
|
|
regex: (Succeeded|Failed|Pending)
|
|
|
|
# --- LibreChat: /metrics behind a Bearer token --------------------------
|
|
# Annotation-based discovery cannot carry credentials, so LibreChat gets its
|
|
# own job. The token comes from the `librechat-metrics` secret mounted into
|
|
# the Prometheus pod and must match METRICS_SECRET on the LibreChat side.
|
|
- job_name: librechat
|
|
kubernetes_sd_configs:
|
|
- role: pod
|
|
namespaces:
|
|
names: [librechat]
|
|
authorization:
|
|
type: Bearer
|
|
credentials_file: /etc/prometheus/secrets/librechat/token
|
|
relabel_configs:
|
|
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape_with_auth]
|
|
action: keep
|
|
regex: "true"
|
|
# A pod yields one target per container: without this filter the init
|
|
# containers duplicate the real target.
|
|
- source_labels: [__meta_kubernetes_pod_container_init]
|
|
action: drop
|
|
regex: "true"
|
|
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
|
|
action: replace
|
|
target_label: __address__
|
|
regex: ([^:]+)(?::\d+)?;(\d+)
|
|
replacement: $1:$2
|
|
- source_labels: [__meta_kubernetes_namespace]
|
|
target_label: namespace
|
|
- source_labels: [__meta_kubernetes_pod_name]
|
|
target_label: pod
|
|
- source_labels: [__meta_kubernetes_pod_container_name]
|
|
target_label: container
|
|
- target_label: service
|
|
replacement: librechat
|