infra-observability/k8s/prometheus/prometheus.yaml

global:
  scrape_interval: 30s
  evaluation_interval: 30s
  external_labels:
    cluster: orbstack
    source: prometheus

scrape_configs:
  # --- Control plane -------------------------------------------------------
  - job_name: kubernetes-apiservers
    kubernetes_sd_configs:
      - role: endpoints
    scheme: https
    tls_config:
      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      insecure_skip_verify: true
    authorization:
      credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
    relabel_configs:
      - source_labels:
          [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
        action: keep
        regex: default;kubernetes;https

  # --- Kubelet -------------------------------------------------------------
  - job_name: kubernetes-nodes
    kubernetes_sd_configs:
      - role: node
    scheme: https
    tls_config:
      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      insecure_skip_verify: true
    authorization:
      credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
    relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - target_label: __address__
        replacement: kubernetes.default.svc:443
      - source_labels: [__meta_kubernetes_node_name]
        regex: (.+)
        target_label: __metrics_path__
        replacement: /api/v1/nodes/$1/proxy/metrics

  # --- Container resource metrics -----------------------------------------
  # /metrics/resource and NOT /metrics/cadvisor: on this cluster (OrbStack) the
  # kubelet's cadvisor endpoint only exposes `machine_*` series and no
  # `container_*` at all. /metrics/resource -- the source metrics-server reads --
  # provides container_cpu_usage_seconds_total and
  # container_memory_working_set_bytes with namespace/pod/container labels.
  - job_name: kubernetes-kubelet-resource
    kubernetes_sd_configs:
      - role: node
    scheme: https
    tls_config:
      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      insecure_skip_verify: true
    authorization:
      credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
    relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - target_label: __address__
        replacement: kubernetes.default.svc:443
      - source_labels: [__meta_kubernetes_node_name]
        regex: (.+)
        target_label: __metrics_path__
        replacement: /api/v1/nodes/$1/proxy/metrics/resource
      - source_labels: [__meta_kubernetes_node_name]
        target_label: node

  # --- Services annotated with prometheus.io/scrape -----------------------
  - job_name: kubernetes-service-endpoints
    kubernetes_sd_configs:
      - role: endpoints
    relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
        action: keep
        regex: "true"
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
        action: replace
        target_label: __scheme__
        regex: (https?)
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        target_label: namespace
      - source_labels: [__meta_kubernetes_service_name]
        target_label: service
      - source_labels: [__meta_kubernetes_pod_name]
        target_label: pod
      - source_labels: [__meta_kubernetes_pod_node_name]
        target_label: node

  # --- Pods annotated with prometheus.io/scrape ---------------------------
  - job_name: kubernetes-pods
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: "true"
      # A pod yields one target per container: without this filter the init
      # containers duplicate the real target.
      - source_labels: [__meta_kubernetes_pod_container_init]
        action: drop
        regex: "true"
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
        action: replace
        target_label: __scheme__
        regex: (https?)
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        target_label: namespace
      - source_labels: [__meta_kubernetes_pod_name]
        target_label: pod
      - source_labels: [__meta_kubernetes_pod_node_name]
        target_label: node
      - source_labels: [__meta_kubernetes_pod_container_name]
        target_label: container
      - source_labels: [__meta_kubernetes_pod_phase]
        action: drop
        regex: (Succeeded|Failed|Pending)

  # --- LibreChat: /metrics behind a Bearer token --------------------------
  # Annotation-based discovery cannot carry credentials, so LibreChat gets its
  # own job. The token comes from the `librechat-metrics` secret mounted into
  # the Prometheus pod and must match METRICS_SECRET on the LibreChat side.
  - job_name: librechat
    kubernetes_sd_configs:
      - role: pod
        namespaces:
          names: [librechat]
    authorization:
      type: Bearer
      credentials_file: /etc/prometheus/secrets/librechat/token
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape_with_auth]
        action: keep
        regex: "true"
      # A pod yields one target per container: without this filter the init
      # containers duplicate the real target.
      - source_labels: [__meta_kubernetes_pod_container_init]
        action: drop
        regex: "true"
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
      - source_labels: [__meta_kubernetes_namespace]
        target_label: namespace
      - source_labels: [__meta_kubernetes_pod_name]
        target_label: pod
      - source_labels: [__meta_kubernetes_pod_container_name]
        target_label: container
      - target_label: service
        replacement: librechat

  # --- Kong gateways ------------------------------------------------------
  # Every dataplane pod the Kong operator creates declares a container port
  # named `metrics` (8100) and carries the gateway-name label, so no
  # annotation is needed on the pods themselves — which matters because the
  # operator owns that pod template, not us. Keeping on the port NAME rather
  # than a number also drops the init containers, which declare no port.
  #
  # The RED series (kong_http_requests_total, kong_*_latency_ms,
  # kong_bandwidth_bytes) only appear once the gateway has the `prometheus`
  # KongPlugin bound; without it the endpoint still answers, but with node
  # and shared-dict metrics only.
  - job_name: kong-gateways
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_label_gateway_networking_k8s_io_gateway_name]
        action: keep
        regex: .+
      - source_labels: [__meta_kubernetes_pod_container_port_name]
        action: keep
        regex: metrics
      - source_labels: [__meta_kubernetes_pod_label_gateway_networking_k8s_io_gateway_name]
        target_label: gateway
      - source_labels: [__meta_kubernetes_namespace]
        target_label: namespace
      - source_labels: [__meta_kubernetes_pod_name]
        target_label: pod
      - source_labels: [__meta_kubernetes_pod_container_name]
        target_label: container
      - source_labels: [__meta_kubernetes_pod_node_name]
        target_label: node