From 5770f78c991fe215d26d4bcc6a94ca54edf75901 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Tailland?= Date: Mon, 31 Aug 2026 07:34:55 +0200 Subject: [PATCH] Declare the classic-models topics and a credentials store Topics are declared rather than auto-created: the application account has no Create ACL, so a producer would have been denied, and an auto-created topic takes broker defaults nobody chose. The credentials store lets an application namespace read its own KafkaUser password and the cluster CA from the kafka namespace, so the password stays generated and rotatable by Strimzi instead of being copied by hand. Co-Authored-By: Claude Opus 5 --- Makefile | 15 ++++++--- credentials-access.yaml | 68 +++++++++++++++++++++++++++++++++++++++++ kafka-topics.yaml | 56 +++++++++++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 5 deletions(-) create mode 100644 credentials-access.yaml create mode 100644 kafka-topics.yaml diff --git a/Makefile b/Makefile index ca4f039..ebe60d5 100644 --- a/Makefile +++ b/Makefile @@ -59,28 +59,31 @@ help: # ── Cluster lifecycle ───────────────────────────────────────────────────────── deploy: - @echo "==> [1/5] Creating namespace '$(NAMESPACE)'" + @echo "==> [1/6] Creating namespace '$(NAMESPACE)'" @kubectl create namespace $(NAMESPACE) --dry-run=client -o yaml | kubectl apply -f - - @echo "==> [2/5] Installing Strimzi operator" + @echo "==> [2/6] Installing Strimzi operator" @helm repo add strimzi https://strimzi.io/charts/ 2>/dev/null || true @helm repo update strimzi 2>/dev/null @helm upgrade --install strimzi-kafka-operator strimzi/strimzi-kafka-operator \ --namespace $(NAMESPACE) \ --set watchAnyNamespace=false \ --wait --timeout 5m - @echo "==> [3/5] Waiting for Strimzi CRDs" + @echo "==> [3/6] Waiting for Strimzi CRDs" @for crd in kafkas.kafka.strimzi.io kafkanodepools.kafka.strimzi.io kafkausers.kafka.strimzi.io; do \ until kubectl get crd "$$crd" -o jsonpath='{.status.conditions[?(@.type=="Established")].status}' 2>/dev/null | grep -q True; do sleep 2; done; \ echo " $$crd ready"; \ done - @echo "==> [4/5] Deploying Kafka cluster (KRaft, SASL_SSL, SCRAM-SHA-512)" + @echo "==> [4/6] Deploying Kafka cluster (KRaft, SASL_SSL, SCRAM-SHA-512)" @kubectl apply -f $(SCRIPT_DIR)kafka.yaml @kubectl wait kafka/$(KAFKA_NAME) --for=condition=Ready --timeout=10m -n $(NAMESPACE) - @echo "==> [5/5] Deploying KafkaUsers" + @echo "==> [5/6] Deploying KafkaUsers" @kubectl apply -f $(SCRIPT_DIR)kafka-users.yaml @for user in kafka-admin kafka-client; do \ kubectl wait kafkauser/$$user --for=condition=Ready --timeout=120s -n $(NAMESPACE); \ done + @echo "==> [6/6] Deploying topics and the credentials store" + @kubectl apply -f $(SCRIPT_DIR)kafka-topics.yaml + @kubectl apply -f $(SCRIPT_DIR)credentials-access.yaml @echo "" @echo "Cluster ready. Bootstrap: $(BOOTSTRAP)" @@ -112,6 +115,8 @@ upgrade: teardown: @echo "==> Deleting KafkaUsers and Kafka cluster" + @kubectl delete -f $(SCRIPT_DIR)credentials-access.yaml --ignore-not-found + @kubectl delete -f $(SCRIPT_DIR)kafka-topics.yaml --ignore-not-found @kubectl delete -f $(SCRIPT_DIR)kafka-users.yaml --ignore-not-found @kubectl delete -f $(SCRIPT_DIR)kafka.yaml --ignore-not-found @echo "==> Waiting for pods to terminate" diff --git a/credentials-access.yaml b/credentials-access.yaml new file mode 100644 index 0000000..2091a3b --- /dev/null +++ b/credentials-access.yaml @@ -0,0 +1,68 @@ +--- +# Letting an application namespace read its own Kafka credentials. +# +# Strimzi writes a KafkaUser's SCRAM password into a Secret in *this* namespace, +# and a Secret cannot be mounted across namespaces. The alternatives are to copy +# the password by hand (it then drifts the first time the user is regenerated) +# or to put it in Vault (a second copy of a credential the cluster already +# owns, kept in sync by nobody). +# +# Instead External Secrets reads it here and materialises it there. The password +# stays generated by, and rotatable by, Strimzi; the copy follows. +# +# The ServiceAccount below can read Secrets in this namespace ONLY, which +# includes every KafkaUser's password. That is the trade: any namespace allowed +# to create an ExternalSecret against this store can pull any Kafka account's +# credentials. Acceptable in a lab with one operator; in a shared cluster, +# narrow the Role with resourceNames and run one store per application. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kafka-credentials-reader + namespace: kafka +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kafka-credentials-reader + namespace: kafka +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kafka-credentials-reader + namespace: kafka +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kafka-credentials-reader +subjects: + - kind: ServiceAccount + name: kafka-credentials-reader + namespace: kafka +--- +# External Secrets authenticates as that ServiceAccount rather than as itself, +# so what the store can reach is bounded by the Role above and visible in RBAC +# rather than in the operator's own permissions. +apiVersion: external-secrets.io/v1 +kind: ClusterSecretStore +metadata: + name: kafka-credentials +spec: + provider: + kubernetes: + remoteNamespace: kafka + server: + caProvider: + type: ConfigMap + name: kube-root-ca.crt + key: ca.crt + namespace: kafka + auth: + serviceAccount: + name: kafka-credentials-reader + namespace: kafka diff --git a/kafka-topics.yaml b/kafka-topics.yaml new file mode 100644 index 0000000..dc18cbe --- /dev/null +++ b/kafka-topics.yaml @@ -0,0 +1,56 @@ +--- +# Topics are declared, never auto-created. +# +# Two reasons. An application account has no Create ACL — a producer that can +# invent topics can also invent a typo and publish into it for weeks with nobody +# noticing. And auto-created topics take the broker defaults, which is how a +# topic that should be compacted and retained for a year ends up deleted after +# seven days. +# +# One topic per aggregate. Partition count is the ceiling on consumer +# parallelism and cannot be lowered later, so it is set for the traffic +# expected, not for the traffic today. +apiVersion: kafka.strimzi.io/v1 +kind: KafkaTopic +metadata: + name: classic-models.catalog + namespace: kafka + labels: + strimzi.io/cluster: kafka +spec: + partitions: 3 + replicas: 1 + config: + retention.ms: "604800000" # 7 days + cleanup.policy: delete + min.insync.replicas: "1" +--- +apiVersion: kafka.strimzi.io/v1 +kind: KafkaTopic +metadata: + name: classic-models.sales + namespace: kafka + labels: + strimzi.io/cluster: kafka +spec: + partitions: 3 + replicas: 1 + config: + retention.ms: "604800000" + cleanup.policy: delete + min.insync.replicas: "1" +--- +apiVersion: kafka.strimzi.io/v1 +kind: KafkaTopic +metadata: + name: classic-models.payments + namespace: kafka + labels: + strimzi.io/cluster: kafka +spec: + partitions: 3 + replicas: 1 + config: + retention.ms: "604800000" + cleanup.policy: delete + min.insync.replicas: "1"