feat: deploy n8n on the cluster
The runtime for the estate's autonomous agents. Postgres as a StatefulSet with its own claim, n8n behind a cert-manager certificate and the nginx ingress, and both secrets in Vault behind an AppRole scoped to n8n/* alone. The certificate is issued into the cluster rather than copied into the repo, so the private key never reaches git. The CA certificate is committed and mounted as NODE_EXTRA_CA_CERTS, so a workflow can reach keycloak and the domain gateways without anyone disabling verification to make it work. One replica and strategy: Recreate, both deliberate — n8n needs Redis and queue mode before a second scheduler is safe, and a ReadWriteOnce volume deadlocks a rolling update. N8N_ENCRYPTION_KEY is generated once and kept in Vault. Regenerating it does not re-encrypt the stored credentials, it orphans them. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Stéphane Tailland
@@ -0,0 +1,36 @@
|
||||
# The lab CA, so n8n can verify keycloak.sttlab.local and the domain gateways
|
||||
# rather than disabling verification. A CA certificate is public by
|
||||
# construction — this is the certificate, never the key, and it is checked in so
|
||||
# the deployment has no hidden prerequisite.
|
||||
#
|
||||
# Issued by the sttlab-local-ca ClusterIssuer, valid until 2026-11-24.
|
||||
# Regenerate on rotation:
|
||||
# kubectl -n cert-manager get secret sttlab-local-ca \
|
||||
# -o jsonpath='{.data.tls\.crt}' | base64 -d
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: sttlab-ca
|
||||
namespace: n8n
|
||||
data:
|
||||
ca.crt: |
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDPDCCAiSgAwIBAgIUd/utDf+f0yQR3v3qTjZqx+wvTZMwDQYJKoZIhvcNAQEL
|
||||
BQAwGTEXMBUGA1UEAwwOKi5zdHRsYWIubG9jYWwwHhcNMjUxMTI0MTUzODQ1WhcN
|
||||
MjYxMTI0MTUzODQ1WjAZMRcwFQYDVQQDDA4qLnN0dGxhYi5sb2NhbDCCASIwDQYJ
|
||||
KoZIhvcNAQEBBQADggEPADCCAQoCggEBALj0sryq8qXtkShT/B+/EjVJNN/TLHIn
|
||||
QJYc6xuUH/f7q3dYdLm4iH0XnFgrSQLUo4iqd/Mczef5iWCYwT5FFa7P17/TcLVf
|
||||
gso0j41qNLBjGxDQGQ6Z1VCxi6oan8ow98rumjVj0E0Qgs5CX/rN+gcEmZ/TZFWU
|
||||
qudVoztpeiVqJ0dxCIq+KMutzd1BU7BXx4VRO9Y+t1idUc7oY7BPmS2qPAN7TtXP
|
||||
j1KhH2BgLsmF5GRhYHNS578eCltcmS2sbQ3c7GwOV96lpgF9TixEvFXeN4xk43E6
|
||||
1COUD7CRiWAZ0XI8FMlpZGeNZeV4bbV/MqVig1ybWNOz9MDKYj3eoXUCAwEAAaN8
|
||||
MHowHQYDVR0OBBYEFDbzYWOPr4vxa+VONs7xHbcNx70YMB8GA1UdIwQYMBaAFDbz
|
||||
YWOPr4vxa+VONs7xHbcNx70YMA8GA1UdEwEB/wQFMAMBAf8wJwYDVR0RBCAwHoIO
|
||||
Ki5zdHRsYWIubG9jYWyCDHN0dGxhYi5sb2NhbDANBgkqhkiG9w0BAQsFAAOCAQEA
|
||||
tU4p5/ONXm7x4YDzSeL8NdThVLguiFur0jcTxvvPUS4NFNsnQJk4OXxBgjy1YwpL
|
||||
3i56/4mCEd9OCQF/Hl0eFwGZBzNKlMoTA08AC/lBDvPrnVjuUu0qe/b/2KOkhESa
|
||||
x8rQNp0o8pcj0f29nu0JfRqYGkzy0Lh2qgXhPu2iC+t2PfdleRAky1tt68wogdFL
|
||||
TvHDsJbVo/ihsPpONlSyHfxG6bb/0Mkz64PH46TTt9GvJflEKMqDwivNFUA1P5S/
|
||||
U4Ic8Qapnxc7h2+WbRjmYrnuygC2wMBE6rL9OF3gwCr1fkgGGRr5lVRUq2X2HZV5
|
||||
5jqViv6eurq8O61N3vWGCQ==
|
||||
-----END CERTIFICATE-----
|
||||
@@ -0,0 +1,17 @@
|
||||
# cert-manager issues this into the `n8n-tls` Secret, from the same
|
||||
# ClusterIssuer as the rest of the estate.
|
||||
#
|
||||
# Deliberately not a copy of a certificate committed to git: the private key
|
||||
# stays in the cluster, is rotated by cert-manager, and never reaches a repo.
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: n8n-tls
|
||||
namespace: n8n
|
||||
spec:
|
||||
secretName: n8n-tls
|
||||
dnsNames:
|
||||
- n8n.sttlab.local
|
||||
issuerRef:
|
||||
name: sttlab-local-ca
|
||||
kind: ClusterIssuer
|
||||
@@ -0,0 +1,31 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: n8n
|
||||
namespace: n8n
|
||||
annotations:
|
||||
# The editor pushes execution progress over a long-lived connection, and
|
||||
# imports can be large. Buffering it defeats the first; the default body
|
||||
# size defeats the second.
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "50m"
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
|
||||
nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
|
||||
nginx.ingress.kubernetes.io/proxy-buffering: "off"
|
||||
nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
tls:
|
||||
- hosts:
|
||||
- n8n.sttlab.local
|
||||
secretName: n8n-tls
|
||||
rules:
|
||||
- host: n8n.sttlab.local
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: n8n
|
||||
port:
|
||||
number: 80
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- ca-configmap.yaml
|
||||
- secrets/clustersecretstore.yaml
|
||||
- secrets/externalsecret.yaml
|
||||
- postgres/statefulset.yaml
|
||||
- postgres/service.yaml
|
||||
- n8n/pvc.yaml
|
||||
- n8n/configmap.yaml
|
||||
- n8n/deployment.yaml
|
||||
- n8n/service.yaml
|
||||
- ingress/certificate.yaml
|
||||
- ingress/ingress.yaml
|
||||
@@ -0,0 +1,35 @@
|
||||
# Nothing here is sensitive. The two values that are — the database password and
|
||||
# the credential encryption key — arrive through n8n-secrets.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: n8n-config
|
||||
namespace: n8n
|
||||
data:
|
||||
GENERIC_TIMEZONE: Europe/Paris
|
||||
TZ: Europe/Paris
|
||||
NODE_ENV: production
|
||||
|
||||
# What n8n believes it is reachable at. It goes into the webhook URLs it
|
||||
# generates and into the links it puts in its own UI, so it must be the
|
||||
# ingress host and not the Service — a webhook advertised as `n8n` inside the
|
||||
# cluster is a webhook nothing outside can call.
|
||||
N8N_HOST: n8n.sttlab.local
|
||||
N8N_PORT: "5678"
|
||||
N8N_PROTOCOL: https
|
||||
WEBHOOK_URL: https://n8n.sttlab.local/
|
||||
N8N_EDITOR_BASE_URL: https://n8n.sttlab.local/
|
||||
|
||||
# TLS terminates at the ingress, so n8n itself serves plain HTTP and must be
|
||||
# told it is behind a proxy — otherwise it reads the hop's address instead of
|
||||
# the caller's.
|
||||
N8N_PROXY_HOPS: "1"
|
||||
|
||||
# The default is to keep every execution for ever, which turns the database
|
||||
# into a log. Two weeks is long enough to debug a failed run.
|
||||
EXECUTIONS_DATA_PRUNE: "true"
|
||||
EXECUTIONS_DATA_MAX_AGE: "336"
|
||||
|
||||
N8N_DIAGNOSTICS_ENABLED: "false"
|
||||
N8N_VERSION_NOTIFICATIONS_ENABLED: "false"
|
||||
N8N_RUNNERS_ENABLED: "true"
|
||||
@@ -0,0 +1,110 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: n8n
|
||||
namespace: n8n
|
||||
spec:
|
||||
# One replica. n8n's queue mode splits the editor from workers and needs Redis
|
||||
# to do it; without that, a second replica is two schedulers racing over the
|
||||
# same cron triggers. See the README before scaling this.
|
||||
replicas: 1
|
||||
strategy:
|
||||
# The volume is ReadWriteOnce, so a rolling update deadlocks: the new pod
|
||||
# waits for a volume the old pod has not released.
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: n8n
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: n8n
|
||||
spec:
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
fsGroup: 1000
|
||||
initContainers:
|
||||
- name: wait-for-postgres
|
||||
image: postgres:16.10-alpine
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
until pg_isready -h n8n-postgres -U n8n -d n8n; do
|
||||
echo "waiting for postgres..."; sleep 3
|
||||
done
|
||||
containers:
|
||||
- name: n8n
|
||||
image: docker.n8n.io/n8nio/n8n:1.128.2
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 5678
|
||||
env:
|
||||
- name: DB_TYPE
|
||||
value: postgresdb
|
||||
- name: DB_POSTGRESDB_HOST
|
||||
value: n8n-postgres
|
||||
- name: DB_POSTGRESDB_PORT
|
||||
value: "5432"
|
||||
- name: DB_POSTGRESDB_DATABASE
|
||||
value: n8n
|
||||
- name: DB_POSTGRESDB_USER
|
||||
value: n8n
|
||||
- name: DB_POSTGRESDB_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: n8n-secrets
|
||||
key: DB_PASSWORD
|
||||
# Encrypts stored credentials. Held in Vault, never regenerated —
|
||||
# a new key does not re-encrypt, it orphans everything already
|
||||
# stored.
|
||||
- name: N8N_ENCRYPTION_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: n8n-secrets
|
||||
key: N8N_ENCRYPTION_KEY
|
||||
# The lab CA, so a workflow can call keycloak.sttlab.local and the
|
||||
# domain gateways without disabling verification. n8n is Node, hence
|
||||
# this variable rather than a trust store.
|
||||
- name: NODE_EXTRA_CA_CERTS
|
||||
value: /etc/sttlab-ca/ca.crt
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: n8n-config
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /home/node/.n8n
|
||||
- name: sttlab-ca
|
||||
mountPath: /etc/sttlab-ca
|
||||
readOnly: true
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 20
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 2Gi
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: n8n-data
|
||||
- name: sttlab-ca
|
||||
configMap:
|
||||
name: sttlab-ca
|
||||
@@ -0,0 +1,13 @@
|
||||
# n8n keeps its own state here — including the SQLite of an unconfigured
|
||||
# install, which is not used since the database is Postgres, but also binary
|
||||
# data and anything a node writes. Small, and it must survive a restart.
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: n8n-data
|
||||
namespace: n8n
|
||||
spec:
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
resources:
|
||||
requests:
|
||||
storage: 2Gi
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: n8n
|
||||
namespace: n8n
|
||||
spec:
|
||||
selector:
|
||||
app: n8n
|
||||
ports:
|
||||
- name: http
|
||||
port: 80
|
||||
targetPort: http
|
||||
@@ -0,0 +1,4 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: n8n
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: n8n-postgres
|
||||
namespace: n8n
|
||||
spec:
|
||||
selector:
|
||||
app: n8n-postgres
|
||||
ports:
|
||||
- name: postgres
|
||||
port: 5432
|
||||
targetPort: postgres
|
||||
@@ -0,0 +1,71 @@
|
||||
# A StatefulSet rather than a Deployment: one identity, one volume, and no
|
||||
# window where two pods hold the same data directory. The volumeClaimTemplate
|
||||
# also means the claim is not deleted with the workload — losing the workflows
|
||||
# to a `kubectl delete deploy` would be an avoidable way to lose them.
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: n8n-postgres
|
||||
namespace: n8n
|
||||
spec:
|
||||
serviceName: n8n-postgres
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: n8n-postgres
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: n8n-postgres
|
||||
spec:
|
||||
securityContext:
|
||||
fsGroup: 999
|
||||
containers:
|
||||
- name: postgres
|
||||
image: postgres:16.10-alpine
|
||||
ports:
|
||||
- name: postgres
|
||||
containerPort: 5432
|
||||
env:
|
||||
- name: POSTGRES_USER
|
||||
value: n8n
|
||||
- name: POSTGRES_DB
|
||||
value: n8n
|
||||
- name: POSTGRES_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: n8n-secrets
|
||||
key: DB_PASSWORD
|
||||
# The image initialises into this directory. Naming a subdirectory
|
||||
# of the mount rather than the mount itself keeps initdb away from
|
||||
# the lost+found some volume types leave at the root.
|
||||
- name: PGDATA
|
||||
value: /var/lib/postgresql/data/pgdata
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /var/lib/postgresql/data
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: ["pg_isready", "-U", "n8n", "-d", "n8n"]
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["pg_isready", "-U", "n8n", "-d", "n8n"]
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 20
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: data
|
||||
spec:
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
resources:
|
||||
requests:
|
||||
storage: 5Gi
|
||||
@@ -0,0 +1,37 @@
|
||||
# One AppRole per estate, each with a policy that can read only its own path.
|
||||
# The n8n store cannot read `keycloak/*` and the keycloak store cannot read
|
||||
# `n8n/*` — so a compromised workload in one namespace does not become a way
|
||||
# into the other's secrets.
|
||||
#
|
||||
# Its Vault side is created by hand, once:
|
||||
#
|
||||
# vault policy write n8n-readonly - <<'HCL'
|
||||
# path "secrets/data/n8n/*" { capabilities = ["read"] }
|
||||
# path "secrets/metadata/n8n/*" { capabilities = ["list", "read"] }
|
||||
# HCL
|
||||
# vault write auth/approle/role/n8n-eso token_policies=n8n-readonly \
|
||||
# token_ttl=1h token_max_ttl=4h
|
||||
# kubectl -n external-secrets create secret generic vault-approle-n8n \
|
||||
# --from-literal=roleId=$(vault read -field=role_id auth/approle/role/n8n-eso/role-id) \
|
||||
# --from-literal=secretId=$(vault write -f -field=secret_id auth/approle/role/n8n-eso/secret-id)
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ClusterSecretStore
|
||||
metadata:
|
||||
name: vault-n8n
|
||||
spec:
|
||||
provider:
|
||||
vault:
|
||||
server: https://vault.sttlab.eu
|
||||
path: secrets
|
||||
version: v2
|
||||
auth:
|
||||
appRole:
|
||||
path: approle
|
||||
roleRef:
|
||||
name: vault-approle-n8n
|
||||
namespace: external-secrets
|
||||
key: roleId
|
||||
secretRef:
|
||||
name: vault-approle-n8n
|
||||
namespace: external-secrets
|
||||
key: secretId
|
||||
@@ -0,0 +1,22 @@
|
||||
# Two values, both of which must outlive any pod.
|
||||
#
|
||||
# N8N_ENCRYPTION_KEY encrypts the credentials n8n stores in its database. Lose
|
||||
# it and every stored credential becomes unreadable — the workflows survive, the
|
||||
# credentials do not. It is generated once and kept in Vault for that reason,
|
||||
# never regenerated on a redeploy.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: n8n-secrets
|
||||
namespace: n8n
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: vault-n8n
|
||||
target:
|
||||
name: n8n-secrets
|
||||
creationPolicy: Owner
|
||||
dataFrom:
|
||||
- extract:
|
||||
key: n8n/instance
|
||||
Reference in New Issue
Block a user