From 9bf9ecc358cd9915d83b481ea27b10ac2dd58cc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Tailland?= Date: Thu, 27 Aug 2026 17:21:32 +0200 Subject: [PATCH] Add Grafana's MCP server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exposes Prometheus, Loki and Tempo as tools an LLM can call, so a question like "what failed in the last transaction" is answered by querying rather than by building another panel — which is where the trace dashboard kept running out of road. Works against OSS Grafana; the Cloud-only tools (Incidents, OnCall, Sift, Agent Observability, Assistant) are simply absent. Verified end to end: the MCP handshake succeeds and list_datasources returns Loki, Prometheus and Tempo. Authenticates with a Grafana service account token, Viewer role, created once through /api/serviceaccounts and stored in the gitignored k8s/secret.yaml. secret.example.yaml documents how to mint it, since Grafana returns the value only at creation. Two things the deployment needs that are not obvious: * runAsUser must be numeric. The image declares USER mcp-grafana by name, Kubernetes cannot resolve that to check it is not root, and refuses to start the container under runAsNonRoot with "image has non-numeric user". 1000/999 is what the image actually uses. * --allowed-hosts must name the Service. The server validates the Host header against a loopback-only default as DNS-rebinding protection, so the first in-cluster call returned "forbidden: host not allowed". Named rather than disabled with "*": the check is worth keeping. Deliberately in-cluster only, with no OIDC and no audit trail — anything that can reach this Service reads all the telemetry. The intended end state is the same treatment as the five domain MCP servers, behind a Kong gateway with openid-connect, consumer groups and http-log. Co-Authored-By: Claude Opus 5 --- k8s/kustomization.yaml | 1 + k8s/mcp-grafana/deployment.yaml | 78 ++++++++++++++++++++++++++++++ k8s/mcp-grafana/kustomization.yaml | 5 ++ k8s/mcp-grafana/service.yaml | 14 ++++++ k8s/secret.example.yaml | 6 +++ 5 files changed, 104 insertions(+) create mode 100644 k8s/mcp-grafana/deployment.yaml create mode 100644 k8s/mcp-grafana/kustomization.yaml create mode 100644 k8s/mcp-grafana/service.yaml diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index 72a2903..f2b86ea 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -11,6 +11,7 @@ resources: - alloy - grafana - grafana/dashboards + - mcp-grafana - ingress labels: diff --git a/k8s/mcp-grafana/deployment.yaml b/k8s/mcp-grafana/deployment.yaml new file mode 100644 index 0000000..11b49d8 --- /dev/null +++ b/k8s/mcp-grafana/deployment.yaml @@ -0,0 +1,78 @@ +# Grafana's own MCP server: it exposes Prometheus, Loki and Tempo as tools an +# LLM can call, so a question like "what failed in the last transaction" is +# answered by querying, rather than by building another panel. +# +# Works against OSS Grafana 9.0+. The Cloud-only tools — Incidents, OnCall, +# Sift, Agent Observability, Assistant — are simply absent here; the +# datasource, dashboard and alerting tools are the ones that matter. +# +# Reachable in-cluster only for now, with no OIDC and no audit trail. The +# intended end state is the same treatment as the five domain MCP servers: +# behind a Kong gateway with openid-connect, consumer groups and http-log. +# Until then, anything that can reach this Service can read all the telemetry. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mcp-grafana + labels: + app.kubernetes.io/name: mcp-grafana + app.kubernetes.io/component: mcp +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: mcp-grafana + template: + metadata: + labels: + app.kubernetes.io/name: mcp-grafana + app.kubernetes.io/component: mcp + spec: + containers: + - name: mcp-grafana + image: mcp/grafana:latest + args: + # Streamable HTTP rather than stdio: this runs as a service other + # pods connect to, not as a subprocess of one client. + - "-t" + - "streamable-http" + - "--log-level" + - "info" + # DNS-rebinding protection: the server rejects any Host header not + # on this list, defaulting to loopback only — which is why the first + # in-cluster probe got "forbidden: host not allowed". Name the + # Service instead of disabling validation with "*": the point of the + # check is that only callers using the expected name get through. + - "--allowed-hosts" + - "mcp-grafana:8000,mcp-grafana.observability.svc.cluster.local:8000" + env: + - name: GRAFANA_URL + # In-cluster address, not the ingress: no TLS hop, no dependency + # on sttlab.local resolving inside the cluster. + value: http://grafana:3000 + - name: GRAFANA_SERVICE_ACCOUNT_TOKEN + valueFrom: + secretKeyRef: + name: observability-secrets + key: GRAFANA_SERVICE_ACCOUNT_TOKEN + ports: + - name: http + containerPort: 8000 + resources: + requests: + cpu: 20m + memory: 64Mi + limits: + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # Numeric, because the image declares its user by name + # (USER mcp-grafana). Kubernetes cannot resolve a name to check it + # is not root, and refuses to start the container with + # runAsNonRoot: "image has non-numeric user". 1000/999 are what the + # image actually uses. + runAsUser: 1000 + runAsGroup: 999 + capabilities: + drop: ["ALL"] diff --git a/k8s/mcp-grafana/kustomization.yaml b/k8s/mcp-grafana/kustomization.yaml new file mode 100644 index 0000000..a33121c --- /dev/null +++ b/k8s/mcp-grafana/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - deployment.yaml + - service.yaml diff --git a/k8s/mcp-grafana/service.yaml b/k8s/mcp-grafana/service.yaml new file mode 100644 index 0000000..fee8025 --- /dev/null +++ b/k8s/mcp-grafana/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: mcp-grafana + labels: + app.kubernetes.io/name: mcp-grafana +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: mcp-grafana + ports: + - name: http + port: 8000 + targetPort: http diff --git a/k8s/secret.example.yaml b/k8s/secret.example.yaml index 5cb375e..ed36533 100644 --- a/k8s/secret.example.yaml +++ b/k8s/secret.example.yaml @@ -9,3 +9,9 @@ type: Opaque stringData: GF_SECURITY_ADMIN_USER: admin GF_SECURITY_ADMIN_PASSWORD: + # Token of a Grafana service account, created once with the admin credentials: + # POST /api/serviceaccounts {"name":"mcp-grafana","role":"Viewer"} + # POST /api/serviceaccounts//tokens {"name":"mcp-grafana-token"} + # Grafana returns the value once, at creation. Viewer is enough to query the + # datasources and list dashboards; raise it only if a tool needs writes. + GRAFANA_SERVICE_ACCOUNT_TOKEN: glsa_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx