global: scrape_interval: 30s evaluation_interval: 30s external_labels: cluster: orbstack source: prometheus scrape_configs: # --- Control plane ------------------------------------------------------- - job_name: kubernetes-apiservers kubernetes_sd_configs: - role: endpoints scheme: https tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt insecure_skip_verify: true authorization: credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token relabel_configs: - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] action: keep regex: default;kubernetes;https # --- Kubelet ------------------------------------------------------------- - job_name: kubernetes-nodes kubernetes_sd_configs: - role: node scheme: https tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt insecure_skip_verify: true authorization: credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token relabel_configs: - action: labelmap regex: __meta_kubernetes_node_label_(.+) - target_label: __address__ replacement: kubernetes.default.svc:443 - source_labels: [__meta_kubernetes_node_name] regex: (.+) target_label: __metrics_path__ replacement: /api/v1/nodes/$1/proxy/metrics # --- Container resource metrics ----------------------------------------- # /metrics/resource and NOT /metrics/cadvisor: on this cluster (OrbStack) the # kubelet's cadvisor endpoint only exposes `machine_*` series and no # `container_*` at all. /metrics/resource -- the source metrics-server reads -- # provides container_cpu_usage_seconds_total and # container_memory_working_set_bytes with namespace/pod/container labels. - job_name: kubernetes-kubelet-resource kubernetes_sd_configs: - role: node scheme: https tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt insecure_skip_verify: true authorization: credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token relabel_configs: - action: labelmap regex: __meta_kubernetes_node_label_(.+) - target_label: __address__ replacement: kubernetes.default.svc:443 - source_labels: [__meta_kubernetes_node_name] regex: (.+) target_label: __metrics_path__ replacement: /api/v1/nodes/$1/proxy/metrics/resource - source_labels: [__meta_kubernetes_node_name] target_label: node # --- Services annotated with prometheus.io/scrape ----------------------- - job_name: kubernetes-service-endpoints kubernetes_sd_configs: - role: endpoints relabel_configs: - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] action: keep regex: "true" - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] action: replace target_label: __scheme__ regex: (https?) - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] action: replace target_label: __metrics_path__ regex: (.+) - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] action: replace target_label: __address__ regex: ([^:]+)(?::\d+)?;(\d+) replacement: $1:$2 - action: labelmap regex: __meta_kubernetes_service_label_(.+) - source_labels: [__meta_kubernetes_namespace] target_label: namespace - source_labels: [__meta_kubernetes_service_name] target_label: service - source_labels: [__meta_kubernetes_pod_name] target_label: pod - source_labels: [__meta_kubernetes_pod_node_name] target_label: node # --- Pods annotated with prometheus.io/scrape --------------------------- - job_name: kubernetes-pods kubernetes_sd_configs: - role: pod relabel_configs: - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] action: keep regex: "true" # A pod yields one target per container: without this filter the init # containers duplicate the real target. - source_labels: [__meta_kubernetes_pod_container_init] action: drop regex: "true" - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme] action: replace target_label: __scheme__ regex: (https?) - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] action: replace target_label: __metrics_path__ regex: (.+) - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] action: replace target_label: __address__ regex: ([^:]+)(?::\d+)?;(\d+) replacement: $1:$2 - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - source_labels: [__meta_kubernetes_namespace] target_label: namespace - source_labels: [__meta_kubernetes_pod_name] target_label: pod - source_labels: [__meta_kubernetes_pod_node_name] target_label: node - source_labels: [__meta_kubernetes_pod_container_name] target_label: container - source_labels: [__meta_kubernetes_pod_phase] action: drop regex: (Succeeded|Failed|Pending) # --- LibreChat: /metrics behind a Bearer token -------------------------- # Annotation-based discovery cannot carry credentials, so LibreChat gets its # own job. The token comes from the `librechat-metrics` secret mounted into # the Prometheus pod and must match METRICS_SECRET on the LibreChat side. - job_name: librechat kubernetes_sd_configs: - role: pod namespaces: names: [librechat] authorization: type: Bearer credentials_file: /etc/prometheus/secrets/librechat/token relabel_configs: - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape_with_auth] action: keep regex: "true" # A pod yields one target per container: without this filter the init # containers duplicate the real target. - source_labels: [__meta_kubernetes_pod_container_init] action: drop regex: "true" - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] action: replace target_label: __address__ regex: ([^:]+)(?::\d+)?;(\d+) replacement: $1:$2 - source_labels: [__meta_kubernetes_namespace] target_label: namespace - source_labels: [__meta_kubernetes_pod_name] target_label: pod - source_labels: [__meta_kubernetes_pod_container_name] target_label: container - target_label: service replacement: librechat