Declarative Grafana / Prometheus / Loki / Tempo observability stack
Deploys the whole stack into the observability namespace of the local cluster with Kustomize alone — no helm install, no dashboard created by hand in the UI. The Makefile carries every command: deploy, dashboards, diagnostics, PVC flush, teardown. Dashboards are files, never UI objects. Each carries a stable uid, which is what keeps URLs, cross-links and alerts alive across redeployments, and references datasources by stable uid rather than by name. The dashboard ConfigMap is generated without a name-suffix hash so applying it updates it in place and Grafana reloads without a restart, while the provisioning ConfigMaps keep their hash precisely so that a change there does force one. Prometheus scrapes the kubelet /metrics/resource endpoint rather than /metrics/cadvisor: on this cluster cadvisor exposes machine_* series only and no container_* at all, which would leave every CPU and memory panel empty. Alloy ships every pod's logs to Loki through the Kubernetes API, so no /var/log mount is needed. Tempo's metrics_generator feeds span metrics and the service graph back into Prometheus, and the datasources are wired so a log line's trace_id opens the matching trace. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Stéphane Tailland
@@ -0,0 +1,66 @@
|
||||
// Grafana Alloy -- ships the logs of every pod in the cluster to Loki.
|
||||
// Reads through the Kubernetes API (no /var/log mount required).
|
||||
|
||||
logging {
|
||||
level = "info"
|
||||
format = "logfmt"
|
||||
}
|
||||
|
||||
discovery.kubernetes "pods" {
|
||||
role = "pod"
|
||||
}
|
||||
|
||||
discovery.relabel "pod_logs" {
|
||||
targets = discovery.kubernetes.pods.targets
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_namespace"]
|
||||
target_label = "namespace"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_name"]
|
||||
target_label = "pod"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_container_name"]
|
||||
target_label = "container"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_node_name"]
|
||||
target_label = "node"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
|
||||
target_label = "app"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_component"]
|
||||
target_label = "component"
|
||||
}
|
||||
// job = <namespace>/<container>: the usual Loki convention.
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"]
|
||||
separator = "/"
|
||||
target_label = "job"
|
||||
}
|
||||
// Only collect containers on the current node (DaemonSet).
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_node_name"]
|
||||
regex = env("HOSTNAME_NODE")
|
||||
action = "keep"
|
||||
}
|
||||
}
|
||||
|
||||
loki.source.kubernetes "pods" {
|
||||
targets = discovery.relabel.pod_logs.output
|
||||
forward_to = [loki.write.default.receiver]
|
||||
}
|
||||
|
||||
loki.write "default" {
|
||||
endpoint {
|
||||
url = "http://loki:3100/loki/api/v1/push"
|
||||
}
|
||||
external_labels = {
|
||||
cluster = "orbstack",
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: alloy
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: alloy
|
||||
app.kubernetes.io/component: collector
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: alloy
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: alloy
|
||||
app.kubernetes.io/name: alloy
|
||||
app.kubernetes.io/component: collector
|
||||
app.kubernetes.io/part-of: observability
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "12345"
|
||||
spec:
|
||||
serviceAccountName: alloy
|
||||
tolerations:
|
||||
- operator: Exists
|
||||
containers:
|
||||
- name: alloy
|
||||
image: grafana/alloy:v1.19.0
|
||||
args:
|
||||
- run
|
||||
- /etc/alloy/config.alloy
|
||||
- --storage.path=/var/lib/alloy/data
|
||||
- --server.http.listen-addr=0.0.0.0:12345
|
||||
- --disable-reporting
|
||||
env:
|
||||
# Used by config.alloy to tail only the pods on the current node.
|
||||
- name: HOSTNAME_NODE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 12345
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 512Mi
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /-/ready
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /etc/alloy
|
||||
- name: storage
|
||||
mountPath: /var/lib/alloy
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: alloy-config
|
||||
- name: storage
|
||||
# Read positions persisted on the node: no full log re-ingestion
|
||||
# when the pod restarts.
|
||||
hostPath:
|
||||
path: /var/lib/alloy
|
||||
type: DirectoryOrCreate
|
||||
@@ -0,0 +1,18 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: observability
|
||||
|
||||
resources:
|
||||
- rbac.yaml
|
||||
- daemonset.yaml
|
||||
- service.yaml
|
||||
|
||||
configMapGenerator:
|
||||
- name: alloy-config
|
||||
files:
|
||||
- config.alloy
|
||||
|
||||
generatorOptions:
|
||||
labels:
|
||||
app.kubernetes.io/name: alloy
|
||||
app.kubernetes.io/part-of: observability
|
||||
@@ -0,0 +1,52 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: alloy
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: alloy
|
||||
app.kubernetes.io/part-of: observability
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: observability-alloy
|
||||
labels:
|
||||
app.kubernetes.io/name: alloy
|
||||
app.kubernetes.io/part-of: observability
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources:
|
||||
- nodes
|
||||
- nodes/proxy
|
||||
- nodes/metrics
|
||||
- services
|
||||
- endpoints
|
||||
- pods
|
||||
- namespaces
|
||||
- events
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/log"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["discovery.k8s.io"]
|
||||
resources: ["endpointslices"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- nonResourceURLs: ["/metrics"]
|
||||
verbs: ["get"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: observability-alloy
|
||||
labels:
|
||||
app.kubernetes.io/name: alloy
|
||||
app.kubernetes.io/part-of: observability
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: observability-alloy
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: alloy
|
||||
namespace: observability
|
||||
@@ -0,0 +1,17 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: alloy
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: alloy
|
||||
app.kubernetes.io/component: collector
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: alloy
|
||||
ports:
|
||||
- name: http
|
||||
port: 12345
|
||||
targetPort: http
|
||||
@@ -0,0 +1,31 @@
|
||||
# Grafana dashboards -- declarative provisioning.
|
||||
#
|
||||
# Rules:
|
||||
# * one .json file per dashboard, named after its UID;
|
||||
# * every JSON carries a STABLE "uid" field (never regenerated): that is what
|
||||
# keeps URLs, cross-links and alerts alive across redeployments;
|
||||
# * `disableNameSuffixHash: true` -> the ConfigMap always keeps the same name,
|
||||
# so `kubectl apply` updates it in place and Grafana reloads WITHOUT
|
||||
# restarting the pod, in about 70s (see dashboard-providers.yaml).
|
||||
#
|
||||
# Adding a dashboard = drop the .json here and add it to `files:` below.
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: observability
|
||||
|
||||
configMapGenerator:
|
||||
- name: grafana-dashboards
|
||||
files:
|
||||
# >>> dashboards (generated by `make dashboards-sync`) >>>
|
||||
- obs-classic-models-api.json
|
||||
- obs-keycloak.json
|
||||
- obs-librechat.json
|
||||
- obs-stack-health.json
|
||||
# <<< dashboards <<<
|
||||
|
||||
generatorOptions:
|
||||
disableNameSuffixHash: true
|
||||
labels:
|
||||
app.kubernetes.io/name: grafana
|
||||
app.kubernetes.io/component: dashboards
|
||||
app.kubernetes.io/part-of: observability
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,437 @@
|
||||
{
|
||||
"uid": "obs-stack-health",
|
||||
"title": "Observability — Stack Health",
|
||||
"description": "Health of the Prometheus / Loki / Tempo / Alloy stack. Reference dashboard: stable UID, declaratively provisioned.",
|
||||
"tags": [
|
||||
"observability",
|
||||
"stack",
|
||||
"provisioned"
|
||||
],
|
||||
"editable": false,
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "30s",
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timezone": "browser",
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "namespace",
|
||||
"label": "Namespace",
|
||||
"type": "query",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"query": "label_values(kube_pod_info, namespace)",
|
||||
"refresh": 2,
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"current": {
|
||||
"text": "observability",
|
||||
"value": "observability"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "row",
|
||||
"title": "Collection",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "Prometheus targets UP",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum(up)",
|
||||
"legendFormat": "up"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"textMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "stat",
|
||||
"title": "Prometheus targets DOWN",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"y": 1
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum(up == bool 0)",
|
||||
"legendFormat": "down"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"textMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "stat",
|
||||
"title": "Log lines ingested (/s)",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 6,
|
||||
"x": 12,
|
||||
"y": 1
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum(rate(loki_distributor_lines_received_total[5m]))",
|
||||
"legendFormat": "lines/s"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 1,
|
||||
"color": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "blue"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"textMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "stat",
|
||||
"title": "Spans ingested (/s)",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 6,
|
||||
"x": 18,
|
||||
"y": 1
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum(rate(tempo_distributor_spans_received_total[5m]))",
|
||||
"legendFormat": "spans/s"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 1,
|
||||
"color": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "purple"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"textMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "row",
|
||||
"title": "Stack resources",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 6
|
||||
},
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "timeseries",
|
||||
"title": "CPU per component (observability namespace)",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 7
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"observability\", container!=\"\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{pod}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"mean",
|
||||
"max"
|
||||
]
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "timeseries",
|
||||
"title": "Memory per component (observability namespace)",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 7
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"observability\", container!=\"\"})",
|
||||
"legendFormat": "{{pod}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes",
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": [
|
||||
"mean",
|
||||
"max"
|
||||
]
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "row",
|
||||
"title": "Logs",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 15
|
||||
},
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "logs",
|
||||
"title": "Recent stack errors",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 10,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"expr": "{namespace=\"observability\"} |~ \"(?i)(error|level=error|fatal)\"",
|
||||
"queryType": "range"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"showLabels": false,
|
||||
"wrapLogMessage": true,
|
||||
"sortOrder": "Descending",
|
||||
"dedupStrategy": "none",
|
||||
"enableLogDetails": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: grafana
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: grafana
|
||||
app.kubernetes.io/component: ui
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: grafana
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: grafana
|
||||
app.kubernetes.io/name: grafana
|
||||
app.kubernetes.io/component: ui
|
||||
app.kubernetes.io/part-of: observability
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "3000"
|
||||
prometheus.io/path: "/metrics"
|
||||
spec:
|
||||
securityContext:
|
||||
runAsUser: 472
|
||||
runAsGroup: 472
|
||||
fsGroup: 472
|
||||
containers:
|
||||
- name: grafana
|
||||
image: grafana/grafana:13.2.0
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 3000
|
||||
env:
|
||||
- name: GF_SECURITY_ADMIN_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: observability-secrets
|
||||
key: GF_SECURITY_ADMIN_USER
|
||||
- name: GF_SECURITY_ADMIN_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: observability-secrets
|
||||
key: GF_SECURITY_ADMIN_PASSWORD
|
||||
- name: GF_SERVER_ROOT_URL
|
||||
value: https://grafana.sttlab.local
|
||||
- name: GF_PATHS_PROVISIONING
|
||||
value: /etc/grafana/provisioning
|
||||
# Everything is driven by the provisioning files, so creating
|
||||
# dashboards or datasources from the UI is disabled.
|
||||
- name: GF_USERS_ALLOW_SIGN_UP
|
||||
value: "false"
|
||||
- name: GF_ANALYTICS_REPORTING_ENABLED
|
||||
value: "false"
|
||||
- name: GF_ANALYTICS_CHECK_FOR_UPDATES
|
||||
value: "false"
|
||||
- name: GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES
|
||||
value: "false"
|
||||
- name: GF_NEWS_NEWS_FEED_ENABLED
|
||||
value: "false"
|
||||
- name: GF_LOG_LEVEL
|
||||
value: info
|
||||
- name: GF_FEATURE_TOGGLES_ENABLE
|
||||
value: traceqlEditor,traceQLStreaming,correlations,lokiLogsDataplane
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /api/health
|
||||
port: http
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /api/health
|
||||
port: http
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 30
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /var/lib/grafana
|
||||
- name: provisioning-datasources
|
||||
mountPath: /etc/grafana/provisioning/datasources
|
||||
- name: provisioning-dashboards
|
||||
mountPath: /etc/grafana/provisioning/dashboards
|
||||
- name: dashboards
|
||||
mountPath: /var/lib/grafana/dashboards
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: grafana-data
|
||||
- name: provisioning-datasources
|
||||
configMap:
|
||||
name: grafana-datasources
|
||||
- name: provisioning-dashboards
|
||||
configMap:
|
||||
name: grafana-dashboard-providers
|
||||
- name: dashboards
|
||||
# Stable-named ConfigMap: updated in place by `make dashboards` and
|
||||
# re-read by Grafana without a restart (~70s: kubelet sync + 10s poll).
|
||||
configMap:
|
||||
name: grafana-dashboards
|
||||
@@ -0,0 +1,27 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: observability
|
||||
|
||||
resources:
|
||||
- pvc.yaml
|
||||
- deployment.yaml
|
||||
- service.yaml
|
||||
# Dashboards live in ./dashboards and are referenced by the root kustomization:
|
||||
# `k8s/grafana` is the runtime alone, `k8s/grafana/dashboards` the content alone.
|
||||
# Both are deployable independently.
|
||||
|
||||
configMapGenerator:
|
||||
# Name-suffix hash KEPT here: datasource and provider provisioning is only
|
||||
# read at startup, so the hash deliberately forces a pod restart whenever
|
||||
# these files change.
|
||||
- name: grafana-datasources
|
||||
files:
|
||||
- datasources.yaml=provisioning/datasources.yaml
|
||||
- name: grafana-dashboard-providers
|
||||
files:
|
||||
- dashboard-providers.yaml=provisioning/dashboard-providers.yaml
|
||||
|
||||
generatorOptions:
|
||||
labels:
|
||||
app.kubernetes.io/name: grafana
|
||||
app.kubernetes.io/part-of: observability
|
||||
@@ -0,0 +1,19 @@
|
||||
# File-based dashboard provider.
|
||||
# The JSON files are mounted from the `grafana-dashboards` ConfigMap (generated
|
||||
# without a name-suffix hash): `make dashboards` updates the ConfigMap in place
|
||||
# and Grafana reloads WITHOUT restarting the pod, in about 70s
|
||||
# (kubelet volume sync ~60s + updateIntervalSeconds below).
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: infra-observability
|
||||
orgId: 1
|
||||
folder: Infra Observability
|
||||
folderUid: infra-observability
|
||||
type: file
|
||||
disableDeletion: false
|
||||
allowUiUpdates: false
|
||||
updateIntervalSeconds: 10
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
foldersFromFilesStructure: false
|
||||
@@ -0,0 +1,82 @@
|
||||
# Declarative datasource provisioning.
|
||||
# The `uid` values are STABLE and are what dashboard JSON files reference:
|
||||
# "datasource": { "type": "prometheus", "uid": "prometheus" }
|
||||
apiVersion: 1
|
||||
|
||||
deleteDatasources:
|
||||
- name: Prometheus
|
||||
orgId: 1
|
||||
- name: Loki
|
||||
orgId: 1
|
||||
- name: Tempo
|
||||
orgId: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
orgId: 1
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
httpMethod: POST
|
||||
timeInterval: 30s
|
||||
manageAlerts: false
|
||||
exemplarTraceIdDestinations:
|
||||
- name: trace_id
|
||||
datasourceUid: tempo
|
||||
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
orgId: 1
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
jsonData:
|
||||
timeout: 60
|
||||
maxLines: 1000
|
||||
# Logs -> traces: a trace_id in the log line opens Tempo.
|
||||
derivedFields:
|
||||
- name: TraceID
|
||||
matcherType: regex
|
||||
matcherRegex: '"?trace[_-]?id"?[:=]"?([a-f0-9]{16,32})'
|
||||
url: '$${__value.raw}'
|
||||
datasourceUid: tempo
|
||||
urlDisplayLabel: View trace
|
||||
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
orgId: 1
|
||||
url: http://tempo:3200
|
||||
editable: false
|
||||
jsonData:
|
||||
httpMethod: GET
|
||||
streamingEnabled:
|
||||
search: true
|
||||
metrics: true
|
||||
# Service graph fed by the metrics_generator -> Prometheus.
|
||||
serviceMap:
|
||||
datasourceUid: prometheus
|
||||
nodeGraph:
|
||||
enabled: true
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
spanStartTimeShift: -5m
|
||||
spanEndTimeShift: 5m
|
||||
filterByTraceID: true
|
||||
filterBySpanID: false
|
||||
tags:
|
||||
- key: service.name
|
||||
value: app
|
||||
tracesToMetrics:
|
||||
datasourceUid: prometheus
|
||||
spanStartTimeShift: -5m
|
||||
spanEndTimeShift: 5m
|
||||
tags:
|
||||
- key: service.name
|
||||
value: service
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: grafana-data
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: grafana
|
||||
app.kubernetes.io/component: storage
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 5Gi
|
||||
@@ -0,0 +1,17 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: grafana
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: grafana
|
||||
app.kubernetes.io/component: ui
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: grafana
|
||||
ports:
|
||||
- name: http
|
||||
port: 3000
|
||||
targetPort: http
|
||||
@@ -0,0 +1,21 @@
|
||||
# Certificates issued by the `sttlab-local-ca` cert-manager ClusterIssuer
|
||||
# (already present on the cluster). No private key is ever versioned.
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: observability-tls
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
secretName: observability-tls
|
||||
issuerRef:
|
||||
name: sttlab-local-ca
|
||||
kind: ClusterIssuer
|
||||
commonName: grafana.sttlab.local
|
||||
dnsNames:
|
||||
- grafana.sttlab.local
|
||||
- prometheus.sttlab.local
|
||||
- alloy.sttlab.local
|
||||
- loki.sttlab.local
|
||||
- tempo.sttlab.local
|
||||
@@ -0,0 +1,88 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: grafana
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: grafana
|
||||
app.kubernetes.io/part-of: observability
|
||||
annotations:
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "20m"
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
|
||||
nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
|
||||
nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
|
||||
# Required for TraceQL streaming and Loki live tail.
|
||||
nginx.ingress.kubernetes.io/proxy-buffering: "off"
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
tls:
|
||||
- hosts:
|
||||
- grafana.sttlab.local
|
||||
secretName: observability-tls
|
||||
rules:
|
||||
- host: grafana.sttlab.local
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: grafana
|
||||
port:
|
||||
number: 3000
|
||||
---
|
||||
# Exposed for diagnostics (targets, TSDB status, expression browser).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: prometheus
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
app.kubernetes.io/part-of: observability
|
||||
annotations:
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
tls:
|
||||
- hosts:
|
||||
- prometheus.sttlab.local
|
||||
secretName: observability-tls
|
||||
rules:
|
||||
- host: prometheus.sttlab.local
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: prometheus
|
||||
port:
|
||||
number: 9090
|
||||
---
|
||||
# Alloy UI: state of the log collection pipeline components.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: alloy
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: alloy
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
tls:
|
||||
- hosts:
|
||||
- alloy.sttlab.local
|
||||
secretName: observability-tls
|
||||
rules:
|
||||
- host: alloy.sttlab.local
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: alloy
|
||||
port:
|
||||
number: 12345
|
||||
@@ -0,0 +1,7 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: observability
|
||||
|
||||
resources:
|
||||
- certificate.yaml
|
||||
- ingress.yaml
|
||||
@@ -0,0 +1,20 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: observability
|
||||
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- secret.yaml
|
||||
- prometheus
|
||||
- loki
|
||||
- tempo
|
||||
- alloy
|
||||
- grafana
|
||||
- grafana/dashboards
|
||||
- ingress
|
||||
|
||||
labels:
|
||||
- includeSelectors: false
|
||||
pairs:
|
||||
app.kubernetes.io/managed-by: kustomize
|
||||
app.kubernetes.io/part-of: observability
|
||||
@@ -0,0 +1,73 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: loki
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: loki
|
||||
app.kubernetes.io/component: logs
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: loki
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: loki
|
||||
app.kubernetes.io/name: loki
|
||||
app.kubernetes.io/component: logs
|
||||
app.kubernetes.io/part-of: observability
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "3100"
|
||||
spec:
|
||||
securityContext:
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
fsGroup: 10001
|
||||
containers:
|
||||
- name: loki
|
||||
image: grafana/loki:3.7.6
|
||||
args:
|
||||
- -config.file=/etc/loki/loki.yaml
|
||||
- -target=all
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 3100
|
||||
- name: grpc
|
||||
containerPort: 9096
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 1Gi
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /ready
|
||||
port: http
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /ready
|
||||
port: http
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 30
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /etc/loki
|
||||
- name: data
|
||||
mountPath: /loki
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: loki-config
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: loki-data
|
||||
@@ -0,0 +1,18 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: observability
|
||||
|
||||
resources:
|
||||
- pvc.yaml
|
||||
- deployment.yaml
|
||||
- service.yaml
|
||||
|
||||
configMapGenerator:
|
||||
- name: loki-config
|
||||
files:
|
||||
- loki.yaml
|
||||
|
||||
generatorOptions:
|
||||
labels:
|
||||
app.kubernetes.io/name: loki
|
||||
app.kubernetes.io/part-of: observability
|
||||
@@ -0,0 +1,85 @@
|
||||
# Loki 3.x in monolithic mode (-target=all), filesystem storage on a PVC.
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_address: 0.0.0.0
|
||||
http_listen_port: 3100
|
||||
grpc_listen_address: 0.0.0.0
|
||||
grpc_listen_port: 9096
|
||||
log_level: info
|
||||
grpc_server_max_concurrent_streams: 1000
|
||||
|
||||
common:
|
||||
instance_addr: 127.0.0.1
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: "2024-01-01"
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
storage_config:
|
||||
tsdb_shipper:
|
||||
active_index_directory: /loki/tsdb-index
|
||||
cache_location: /loki/tsdb-cache
|
||||
filesystem:
|
||||
directory: /loki/chunks
|
||||
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
compaction_interval: 10m
|
||||
retention_delete_delay: 2h
|
||||
|
||||
limits_config:
|
||||
retention_period: 168h
|
||||
max_query_series: 5000
|
||||
ingestion_rate_mb: 16
|
||||
ingestion_burst_size_mb: 32
|
||||
reject_old_samples: true
|
||||
reject_old_samples_max_age: 168h
|
||||
volume_enabled: true
|
||||
metric_aggregation_enabled: true
|
||||
allow_structured_metadata: true
|
||||
|
||||
pattern_ingester:
|
||||
enabled: true
|
||||
metric_aggregation:
|
||||
loki_address: localhost:3100
|
||||
|
||||
query_range:
|
||||
align_queries_with_step: true
|
||||
cache_results: true
|
||||
results_cache:
|
||||
cache:
|
||||
embedded_cache:
|
||||
enabled: true
|
||||
max_size_mb: 100
|
||||
|
||||
frontend:
|
||||
encoding: protobuf
|
||||
|
||||
ruler:
|
||||
storage:
|
||||
type: local
|
||||
local:
|
||||
directory: /loki/rules
|
||||
rule_path: /loki/rules-tmp
|
||||
enable_api: true
|
||||
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: loki-data
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: loki
|
||||
app.kubernetes.io/component: storage
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
@@ -0,0 +1,20 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: loki
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: loki
|
||||
app.kubernetes.io/component: logs
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: loki
|
||||
ports:
|
||||
- name: http
|
||||
port: 3100
|
||||
targetPort: http
|
||||
- name: grpc
|
||||
port: 9096
|
||||
targetPort: grpc
|
||||
@@ -0,0 +1,6 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: observability
|
||||
labels:
|
||||
app.kubernetes.io/part-of: observability
|
||||
@@ -0,0 +1,86 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: prometheus
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
app.kubernetes.io/component: metrics
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
# ReadWriteOnce PVC: the old pod must release the volume first.
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: prometheus
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: prometheus
|
||||
app.kubernetes.io/name: prometheus
|
||||
app.kubernetes.io/component: metrics
|
||||
app.kubernetes.io/part-of: observability
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9090"
|
||||
spec:
|
||||
serviceAccountName: prometheus
|
||||
securityContext:
|
||||
runAsUser: 65534
|
||||
runAsGroup: 65534
|
||||
fsGroup: 65534
|
||||
containers:
|
||||
- name: prometheus
|
||||
image: prom/prometheus:v3.14.0
|
||||
args:
|
||||
- --config.file=/etc/prometheus/prometheus.yaml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=15d
|
||||
- --storage.tsdb.retention.size=8GB
|
||||
- --web.enable-lifecycle
|
||||
# Receives remote_write from Tempo's metrics_generator (span metrics + service graphs).
|
||||
- --web.enable-remote-write-receiver
|
||||
- --web.external-url=https://prometheus.sttlab.local
|
||||
- --web.route-prefix=/
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 9090
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 2Gi
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /-/ready
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /-/healthy
|
||||
port: http
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /etc/prometheus
|
||||
- name: data
|
||||
mountPath: /prometheus
|
||||
- name: librechat-metrics
|
||||
mountPath: /etc/prometheus/secrets/librechat
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: prometheus-config
|
||||
- name: librechat-metrics
|
||||
secret:
|
||||
secretName: librechat-metrics
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: prometheus-data
|
||||
@@ -0,0 +1,20 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: observability
|
||||
|
||||
resources:
|
||||
- secret.yaml
|
||||
- rbac.yaml
|
||||
- pvc.yaml
|
||||
- deployment.yaml
|
||||
- service.yaml
|
||||
|
||||
configMapGenerator:
|
||||
- name: prometheus-config
|
||||
files:
|
||||
- prometheus.yaml
|
||||
|
||||
generatorOptions:
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
app.kubernetes.io/part-of: observability
|
||||
@@ -0,0 +1,177 @@
|
||||
global:
|
||||
scrape_interval: 30s
|
||||
evaluation_interval: 30s
|
||||
external_labels:
|
||||
cluster: orbstack
|
||||
source: prometheus
|
||||
|
||||
scrape_configs:
|
||||
# --- Control plane -------------------------------------------------------
|
||||
- job_name: kubernetes-apiservers
|
||||
kubernetes_sd_configs:
|
||||
- role: endpoints
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
insecure_skip_verify: true
|
||||
authorization:
|
||||
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
relabel_configs:
|
||||
- source_labels:
|
||||
[__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
|
||||
action: keep
|
||||
regex: default;kubernetes;https
|
||||
|
||||
# --- Kubelet -------------------------------------------------------------
|
||||
- job_name: kubernetes-nodes
|
||||
kubernetes_sd_configs:
|
||||
- role: node
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
insecure_skip_verify: true
|
||||
authorization:
|
||||
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
relabel_configs:
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_node_label_(.+)
|
||||
- target_label: __address__
|
||||
replacement: kubernetes.default.svc:443
|
||||
- source_labels: [__meta_kubernetes_node_name]
|
||||
regex: (.+)
|
||||
target_label: __metrics_path__
|
||||
replacement: /api/v1/nodes/$1/proxy/metrics
|
||||
|
||||
# --- Container resource metrics -----------------------------------------
|
||||
# /metrics/resource and NOT /metrics/cadvisor: on this cluster (OrbStack) the
|
||||
# kubelet's cadvisor endpoint only exposes `machine_*` series and no
|
||||
# `container_*` at all. /metrics/resource -- the source metrics-server reads --
|
||||
# provides container_cpu_usage_seconds_total and
|
||||
# container_memory_working_set_bytes with namespace/pod/container labels.
|
||||
- job_name: kubernetes-kubelet-resource
|
||||
kubernetes_sd_configs:
|
||||
- role: node
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
insecure_skip_verify: true
|
||||
authorization:
|
||||
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
relabel_configs:
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_node_label_(.+)
|
||||
- target_label: __address__
|
||||
replacement: kubernetes.default.svc:443
|
||||
- source_labels: [__meta_kubernetes_node_name]
|
||||
regex: (.+)
|
||||
target_label: __metrics_path__
|
||||
replacement: /api/v1/nodes/$1/proxy/metrics/resource
|
||||
- source_labels: [__meta_kubernetes_node_name]
|
||||
target_label: node
|
||||
|
||||
# --- Services annotated with prometheus.io/scrape -----------------------
|
||||
- job_name: kubernetes-service-endpoints
|
||||
kubernetes_sd_configs:
|
||||
- role: endpoints
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
|
||||
action: keep
|
||||
regex: "true"
|
||||
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
|
||||
action: replace
|
||||
target_label: __scheme__
|
||||
regex: (https?)
|
||||
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
|
||||
action: replace
|
||||
target_label: __metrics_path__
|
||||
regex: (.+)
|
||||
- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
|
||||
action: replace
|
||||
target_label: __address__
|
||||
regex: ([^:]+)(?::\d+)?;(\d+)
|
||||
replacement: $1:$2
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_service_label_(.+)
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
target_label: namespace
|
||||
- source_labels: [__meta_kubernetes_service_name]
|
||||
target_label: service
|
||||
- source_labels: [__meta_kubernetes_pod_name]
|
||||
target_label: pod
|
||||
- source_labels: [__meta_kubernetes_pod_node_name]
|
||||
target_label: node
|
||||
|
||||
# --- Pods annotated with prometheus.io/scrape ---------------------------
|
||||
- job_name: kubernetes-pods
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
|
||||
action: keep
|
||||
regex: "true"
|
||||
# A pod yields one target per container: without this filter the init
|
||||
# containers duplicate the real target.
|
||||
- source_labels: [__meta_kubernetes_pod_container_init]
|
||||
action: drop
|
||||
regex: "true"
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
|
||||
action: replace
|
||||
target_label: __scheme__
|
||||
regex: (https?)
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
|
||||
action: replace
|
||||
target_label: __metrics_path__
|
||||
regex: (.+)
|
||||
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
|
||||
action: replace
|
||||
target_label: __address__
|
||||
regex: ([^:]+)(?::\d+)?;(\d+)
|
||||
replacement: $1:$2
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_pod_label_(.+)
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
target_label: namespace
|
||||
- source_labels: [__meta_kubernetes_pod_name]
|
||||
target_label: pod
|
||||
- source_labels: [__meta_kubernetes_pod_node_name]
|
||||
target_label: node
|
||||
- source_labels: [__meta_kubernetes_pod_container_name]
|
||||
target_label: container
|
||||
- source_labels: [__meta_kubernetes_pod_phase]
|
||||
action: drop
|
||||
regex: (Succeeded|Failed|Pending)
|
||||
|
||||
# --- LibreChat: /metrics behind a Bearer token --------------------------
|
||||
# Annotation-based discovery cannot carry credentials, so LibreChat gets its
|
||||
# own job. The token comes from the `librechat-metrics` secret mounted into
|
||||
# the Prometheus pod and must match METRICS_SECRET on the LibreChat side.
|
||||
- job_name: librechat
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
namespaces:
|
||||
names: [librechat]
|
||||
authorization:
|
||||
type: Bearer
|
||||
credentials_file: /etc/prometheus/secrets/librechat/token
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape_with_auth]
|
||||
action: keep
|
||||
regex: "true"
|
||||
# A pod yields one target per container: without this filter the init
|
||||
# containers duplicate the real target.
|
||||
- source_labels: [__meta_kubernetes_pod_container_init]
|
||||
action: drop
|
||||
regex: "true"
|
||||
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
|
||||
action: replace
|
||||
target_label: __address__
|
||||
regex: ([^:]+)(?::\d+)?;(\d+)
|
||||
replacement: $1:$2
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
target_label: namespace
|
||||
- source_labels: [__meta_kubernetes_pod_name]
|
||||
target_label: pod
|
||||
- source_labels: [__meta_kubernetes_pod_container_name]
|
||||
target_label: container
|
||||
- target_label: service
|
||||
replacement: librechat
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: prometheus-data
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
app.kubernetes.io/component: storage
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
@@ -0,0 +1,51 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: prometheus
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
app.kubernetes.io/part-of: observability
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: observability-prometheus
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
app.kubernetes.io/part-of: observability
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources:
|
||||
- nodes
|
||||
- nodes/metrics
|
||||
- nodes/proxy
|
||||
- services
|
||||
- endpoints
|
||||
- pods
|
||||
- namespaces
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["discovery.k8s.io"]
|
||||
resources: ["endpointslices"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["networking.k8s.io"]
|
||||
resources: ["ingresses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- nonResourceURLs: ["/metrics", "/metrics/cadvisor"]
|
||||
verbs: ["get"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: observability-prometheus
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
app.kubernetes.io/part-of: observability
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: observability-prometheus
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: prometheus
|
||||
namespace: observability
|
||||
@@ -0,0 +1,11 @@
|
||||
# Template. The token must be IDENTICAL to the METRICS_SECRET key of the
|
||||
# `librechat-secrets` secret in the librechat namespace (infra-librechat repo):
|
||||
# it is the Bearer token Prometheus uses to scrape LibreChat's /metrics.
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: librechat-metrics
|
||||
namespace: observability
|
||||
type: Opaque
|
||||
stringData:
|
||||
token: <CHANGE_ME>
|
||||
@@ -0,0 +1,17 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: prometheus
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: prometheus
|
||||
app.kubernetes.io/component: metrics
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: prometheus
|
||||
ports:
|
||||
- name: http
|
||||
port: 9090
|
||||
targetPort: http
|
||||
@@ -0,0 +1,11 @@
|
||||
# Template: copy to secret.yaml (gitignored) and fill in the values.
|
||||
# make secret
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: observability-secrets
|
||||
namespace: observability
|
||||
type: Opaque
|
||||
stringData:
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD: <CHANGE_ME>
|
||||
@@ -0,0 +1,79 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: tempo
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: tempo
|
||||
app.kubernetes.io/component: traces
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: tempo
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: tempo
|
||||
app.kubernetes.io/name: tempo
|
||||
app.kubernetes.io/component: traces
|
||||
app.kubernetes.io/part-of: observability
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "3200"
|
||||
spec:
|
||||
securityContext:
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
fsGroup: 10001
|
||||
containers:
|
||||
- name: tempo
|
||||
image: grafana/tempo:3.0.3
|
||||
args:
|
||||
- -config.file=/etc/tempo/tempo.yaml
|
||||
- -target=all
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 3200
|
||||
- name: grpc
|
||||
containerPort: 9095
|
||||
- name: otlp-grpc
|
||||
containerPort: 4317
|
||||
- name: otlp-http
|
||||
containerPort: 4318
|
||||
- name: zipkin
|
||||
containerPort: 9411
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 2Gi
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /ready
|
||||
port: http
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /ready
|
||||
port: http
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 30
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /etc/tempo
|
||||
- name: data
|
||||
mountPath: /var/tempo
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: tempo-config
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: tempo-data
|
||||
@@ -0,0 +1,18 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: observability
|
||||
|
||||
resources:
|
||||
- pvc.yaml
|
||||
- deployment.yaml
|
||||
- service.yaml
|
||||
|
||||
configMapGenerator:
|
||||
- name: tempo-config
|
||||
files:
|
||||
- tempo.yaml
|
||||
|
||||
generatorOptions:
|
||||
labels:
|
||||
app.kubernetes.io/name: tempo
|
||||
app.kubernetes.io/part-of: observability
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: tempo-data
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: tempo
|
||||
app.kubernetes.io/component: storage
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
@@ -0,0 +1,29 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: tempo
|
||||
namespace: observability
|
||||
labels:
|
||||
app.kubernetes.io/name: tempo
|
||||
app.kubernetes.io/component: traces
|
||||
app.kubernetes.io/part-of: observability
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app: tempo
|
||||
ports:
|
||||
- name: http
|
||||
port: 3200
|
||||
targetPort: http
|
||||
- name: grpc
|
||||
port: 9095
|
||||
targetPort: grpc
|
||||
- name: otlp-grpc
|
||||
port: 4317
|
||||
targetPort: otlp-grpc
|
||||
- name: otlp-http
|
||||
port: 4318
|
||||
targetPort: otlp-http
|
||||
- name: zipkin
|
||||
port: 9411
|
||||
targetPort: zipkin
|
||||
@@ -0,0 +1,64 @@
|
||||
# Tempo 3.x in monolithic mode (-target=all), local storage on a PVC.
|
||||
stream_over_http_enabled: true
|
||||
multitenancy_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_address: 0.0.0.0
|
||||
http_listen_port: 3200
|
||||
grpc_listen_address: 0.0.0.0
|
||||
grpc_listen_port: 9095
|
||||
log_level: info
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
zipkin:
|
||||
endpoint: 0.0.0.0:9411
|
||||
|
||||
# Generates span metrics and service graphs, pushed to Prometheus through its
|
||||
# remote-write receiver (--web.enable-remote-write-receiver).
|
||||
metrics_generator:
|
||||
registry:
|
||||
external_labels:
|
||||
source: tempo
|
||||
cluster: orbstack
|
||||
storage:
|
||||
path: /var/tempo/generator/wal
|
||||
remote_write:
|
||||
- url: http://prometheus:9090/api/v1/write
|
||||
send_exemplars: true
|
||||
|
||||
query_frontend:
|
||||
search:
|
||||
duration_slo: 5s
|
||||
throughput_bytes_slo: 1073741824
|
||||
trace_by_id:
|
||||
duration_slo: 5s
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
|
||||
overrides:
|
||||
defaults:
|
||||
metrics_generator:
|
||||
processors:
|
||||
- service-graphs
|
||||
- span-metrics
|
||||
generate_native_histograms: both
|
||||
# Tempo 3.x: retention is no longer a top-level `compactor:` block; it is
|
||||
# driven from here (backend_scheduler/backend_worker otherwise).
|
||||
compaction:
|
||||
block_retention: 168h
|
||||
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
Reference in New Issue
Block a user