Declarative Grafana / Prometheus / Loki / Tempo observability stack

Deploys the whole stack into the observability namespace of the local
cluster with Kustomize alone — no helm install, no dashboard created by
hand in the UI. The Makefile carries every command: deploy, dashboards,
diagnostics, PVC flush, teardown.

Dashboards are files, never UI objects. Each carries a stable uid, which
is what keeps URLs, cross-links and alerts alive across redeployments, and
references datasources by stable uid rather than by name. The dashboard
ConfigMap is generated without a name-suffix hash so applying it updates
it in place and Grafana reloads without a restart, while the provisioning
ConfigMaps keep their hash precisely so that a change there does force one.

Prometheus scrapes the kubelet /metrics/resource endpoint rather than
/metrics/cadvisor: on this cluster cadvisor exposes machine_* series only
and no container_* at all, which would leave every CPU and memory panel
empty.

Alloy ships every pod's logs to Loki through the Kubernetes API, so no
/var/log mount is needed. Tempo's metrics_generator feeds span metrics and
the service graph back into Prometheus, and the datasources are wired so a
log line's trace_id opens the matching trace.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

k8s/alloy/config.alloy

@@ -0,0 +1,66 @@
+// Grafana Alloy -- ships the logs of every pod in the cluster to Loki.
+// Reads through the Kubernetes API (no /var/log mount required).
+
+logging {
+  level  = "info"
+  format = "logfmt"
+}
+
+discovery.kubernetes "pods" {
+  role = "pod"
+}
+
+discovery.relabel "pod_logs" {
+  targets = discovery.kubernetes.pods.targets
+
+  rule {
+    source_labels = ["__meta_kubernetes_namespace"]
+    target_label  = "namespace"
+  }
+  rule {
+    source_labels = ["__meta_kubernetes_pod_name"]
+    target_label  = "pod"
+  }
+  rule {
+    source_labels = ["__meta_kubernetes_pod_container_name"]
+    target_label  = "container"
+  }
+  rule {
+    source_labels = ["__meta_kubernetes_pod_node_name"]
+    target_label  = "node"
+  }
+  rule {
+    source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
+    target_label  = "app"
+  }
+  rule {
+    source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_component"]
+    target_label  = "component"
+  }
+  // job = <namespace>/<container>: the usual Loki convention.
+  rule {
+    source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"]
+    separator     = "/"
+    target_label  = "job"
+  }
+  // Only collect containers on the current node (DaemonSet).
+  rule {
+    source_labels = ["__meta_kubernetes_pod_node_name"]
+    regex         = env("HOSTNAME_NODE")
+    action        = "keep"
+  }
+}
+
+loki.source.kubernetes "pods" {
+  targets    = discovery.relabel.pod_logs.output
+  forward_to = [loki.write.default.receiver]
+}
+
+loki.write "default" {
+  endpoint {
+    url = "http://loki:3100/loki/api/v1/push"
+  }
+  external_labels = {
+    cluster = "orbstack",
+  }
+}

k8s/alloy/daemonset.yaml

@@ -0,0 +1,73 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: alloy
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: alloy
+    app.kubernetes.io/component: collector
+    app.kubernetes.io/part-of: observability
+spec:
+  selector:
+    matchLabels:
+      app: alloy
+  template:
+    metadata:
+      labels:
+        app: alloy
+        app.kubernetes.io/name: alloy
+        app.kubernetes.io/component: collector
+        app.kubernetes.io/part-of: observability
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "12345"
+    spec:
+      serviceAccountName: alloy
+      tolerations:
+        - operator: Exists
+      containers:
+        - name: alloy
+          image: grafana/alloy:v1.19.0
+          args:
+            - run
+            - /etc/alloy/config.alloy
+            - --storage.path=/var/lib/alloy/data
+            - --server.http.listen-addr=0.0.0.0:12345
+            - --disable-reporting
+          env:
+            # Used by config.alloy to tail only the pods on the current node.
+            - name: HOSTNAME_NODE
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          ports:
+            - name: http
+              containerPort: 12345
+          resources:
+            requests:
+              cpu: 50m
+              memory: 128Mi
+            limits:
+              cpu: "1"
+              memory: 512Mi
+          readinessProbe:
+            httpGet:
+              path: /-/ready
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 10
+          volumeMounts:
+            - name: config
+              mountPath: /etc/alloy
+            - name: storage
+              mountPath: /var/lib/alloy
+      volumes:
+        - name: config
+          configMap:
+            name: alloy-config
+        - name: storage
+          # Read positions persisted on the node: no full log re-ingestion
+          # when the pod restarts.
+          hostPath:
+            path: /var/lib/alloy
+            type: DirectoryOrCreate

k8s/alloy/kustomization.yaml

@@ -0,0 +1,18 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: observability
+
+resources:
+  - rbac.yaml
+  - daemonset.yaml
+  - service.yaml
+
+configMapGenerator:
+  - name: alloy-config
+    files:
+      - config.alloy
+
+generatorOptions:
+  labels:
+    app.kubernetes.io/name: alloy
+    app.kubernetes.io/part-of: observability

k8s/alloy/rbac.yaml

@@ -0,0 +1,52 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: alloy
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: alloy
+    app.kubernetes.io/part-of: observability
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: observability-alloy
+  labels:
+    app.kubernetes.io/name: alloy
+    app.kubernetes.io/part-of: observability
+rules:
+  - apiGroups: [""]
+    resources:
+      - nodes
+      - nodes/proxy
+      - nodes/metrics
+      - services
+      - endpoints
+      - pods
+      - namespaces
+      - events
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["pods/log"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["discovery.k8s.io"]
+    resources: ["endpointslices"]
+    verbs: ["get", "list", "watch"]
+  - nonResourceURLs: ["/metrics"]
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: observability-alloy
+  labels:
+    app.kubernetes.io/name: alloy
+    app.kubernetes.io/part-of: observability
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: observability-alloy
+subjects:
+  - kind: ServiceAccount
+    name: alloy
+    namespace: observability

k8s/alloy/service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: alloy
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: alloy
+    app.kubernetes.io/component: collector
+    app.kubernetes.io/part-of: observability
+spec:
+  type: ClusterIP
+  selector:
+    app: alloy
+  ports:
+    - name: http
+      port: 12345
+      targetPort: http