From 5a4815082066430d07ddf6e7c311a63a14397938 Mon Sep 17 00:00:00 2001 From: sttlab Date: Sun, 3 May 2026 12:38:46 +0000 Subject: [PATCH] Simplification of Mongo and ES config --- README.md | 233 ++++++++++++++++++++++++++++++++++++++++++++++++ apim-values.yml | 89 +++++++++++------- 2 files changed, 288 insertions(+), 34 deletions(-) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..76f95d2 --- /dev/null +++ b/README.md @@ -0,0 +1,233 @@ +# Gravitee APIM — k3s single-node deployment + +Production-like deployment of Gravitee APIM OSS on k3s with end-to-end TLS, credentials injected from Kubernetes secrets, and an internal CA managed by cert-manager. + +--- + +## Architecture + +``` +Internet + └── nginx ingress (TLS termination + HTTPS re-encryption to backends) + ├── console.gravitee.sttlab.pc → graviteeio-apim-ui (nginx, port 8080) + ├── portal.gravitee.sttlab.pc → graviteeio-apim-portal (nginx, port 8080) + ├── api.gravitee.sttlab.pc → graviteeio-apim-api (Jetty/HTTPS, port 8083) + └── gateway.gravitee.sttlab.pc → graviteeio-apim-gateway (Vert.x/HTTPS, port 8082) + +Internal backends: + graviteeio-apim-api ──TLS──► mongodb:27017 + graviteeio-apim-api ──TLS──► gravitee-es-master:9200 + graviteeio-apim-gateway ──TLS──► mongodb:27017 + graviteeio-apim-gateway ──TLS──► gravitee-es-master:9200 +``` + +**Stack:** +- **Gravitee APIM 4.x** — Management API (Jetty), Gateway (Vert.x/Netty), Console UI and Developer Portal (Angular/nginx) +- **MongoDB 8.x** (Bitnami) — management and rate limiting persistence +- **Elasticsearch 8.x** (Elastic) — analytics and reporting +- **cert-manager** — internal self-signed PKI with automatic renewal +- **nginx ingress** — external TLS termination + TLS proxy to backends + +--- + +## PKI and certificates + +### Trust chain + +``` +gravitee-selfsigned-bootstrap (Issuer, self-signed) + └── gravitee-ca (Certificate isCA:true) → secret: gravitee-ca-tls + └── gravitee-ca-issuer (CA Issuer) + ├── console-tls (ingress console) + ├── portal-tls (ingress portal) + ├── api-tls (ingress api) + ├── gateway-tls (ingress gateway) + ├── mongodb-tls (MongoDB server) + ├── elasticsearch-tls (ES server + truststore.jks) + ├── api-internal-tls (Jetty API server + keystore.p12) + └── gateway-internal-tls (Vert.x gateway server + keystore.p12) +``` + +### cert-manager keystores + +Two secrets include PKCS12 keystores automatically generated by cert-manager: +- `api-internal-tls` → `keystore.p12` (used by Jetty) +- `gateway-internal-tls` → `keystore.p12` (used by Vert.x) +- `elasticsearch-tls` → `truststore.jks` (used by the JVM to trust the ES certificate) + +Keystore password is stored in the `gravitee-jks-password` secret. + +### Notable SANs + +- `api-internal-tls` includes `api.gravitee.sttlab.pc` — required for nginx → pod TLS verification +- `gateway-internal-tls` includes `gateway.gravitee.sttlab.pc` — same reason +- `elasticsearch-tls` includes `gravitee-es-master.gravitee-apim.svc.cluster.local` + +### Browser trust + +Export the CA for browser import: +```bash +kubectl get secret gravitee-ca-tls -n gravitee-apim \ + -o jsonpath='{.data.ca\.crt}' | base64 -d > gravitee-ca.crt +``` +Import `gravitee-ca.crt` into the OS keychain or browser trust store. + +--- + +## Kubernetes secrets + +All secrets are created by `secrets-create.sh` before the first deployment. + +| Secret | Contents | Consumed by | +|--------|----------|-------------| +| `mongodb-credentials` | root/gravitee MongoDB passwords | Bitnami MongoDB chart | +| `gravitee-mongodb-uri` | `GRAVITEE_MANAGEMENT_MONGODB_URI`, `GRAVITEE_RATELIMIT_MONGODB_URI` | api, gateway (env) | +| `gravitee-jwt` | `GRAVITEE_JWT_SECRET` | api (env) | +| `gravitee-admin` | `admin-password-plain`, `admin-password-bcrypt` | api (env) | +| `gravitee-es-master-credentials` | ES `username`, `password` | api, gateway (env) | +| `gravitee-jks-password` | keystore password | api, gateway (env) + cert-manager | +| `gravitee-ca-trust` | `ca.crt` only — no tls.crt/key | nginx ingress proxy-ssl-secret | + +> `gravitee-ca-trust` is a dedicated secret containing only `ca.crt`. The `proxy-ssl-secret` nginx annotation presents `tls.crt`/`tls.key` as a client certificate if they exist, which would unintentionally trigger mTLS toward backends. + +### Credential injection + +All credentials are injected via `env[].valueFrom.secretKeyRef` (no `envFrom`). The rule is: +- **Non-sensitive config** (endpoints, flags) → `value:` directly in `env` +- **Secrets** (passwords, tokens, URIs with credentials) → `valueFrom.secretKeyRef` + +Example from `apim-values.yml`: +```yaml +env: + - name: GRAVITEE_MANAGEMENT_MONGODB_URI # credentials + TLS embedded in URI + valueFrom: + secretKeyRef: + name: gravitee-mongodb-uri + key: GRAVITEE_MANAGEMENT_MONGODB_URI + - name: GRAVITEE_ANALYTICS_ELASTICSEARCH_ENDPOINTS_0 # non-sensitive + value: "https://gravitee-es-master.gravitee-apim.svc.cluster.local:9200" +``` + +--- + +## Internal TLS + +### MongoDB + +TLS enabled via `tls=true` in the connection URI. Server certificate validation is handled by the JVM truststore: +``` +JAVA_OPTS: -Djavax.net.ssl.trustStore=/run/secrets/truststore/truststore.jks + -Djavax.net.ssl.trustStorePassword=${JKS_PASSWORD} +``` +The `truststore.jks` is sourced from the `elasticsearch-tls` secret (contains the Gravitee CA). + +MongoDB runs with `--tlsAllowConnectionsWithoutCertificates` (no client mTLS) but requires server TLS (`--tlsMode=requireTLS`). + +### Elasticsearch + +HTTPS with basic auth. Server certificate signed by the Gravitee CA, validated by the same JVM truststore. Username/password injected via Gravitee property-path env vars: +- `GRAVITEE_ANALYTICS_ELASTICSEARCH_SECURITY_USERNAME/PASSWORD` → Management API +- `GRAVITEE_REPORTERS_ELASTICSEARCH_SECURITY_USERNAME/PASSWORD` → Gateway + +### nginx → backends (proxy TLS) + +nginx verifies backend certificates: +```yaml +nginx.ingress.kubernetes.io/proxy-ssl-verify: "on" +nginx.ingress.kubernetes.io/proxy-ssl-secret: "gravitee-apim/gravitee-ca-trust" +nginx.ingress.kubernetes.io/configuration-snippet: | + proxy_ssl_name api.gravitee.sttlab.pc; +``` + +- `proxy-ssl-secret` provides the CA for chain verification +- `proxy_ssl_name` (via snippet) tells nginx which hostname to check against the backend cert SANs +- `proxy_ssl_server_name on` is intentionally **absent**: it modifies the TLS ClientHello in a way that triggers an `illegal parameter` alert in Jetty 12 + +Snippets require `annotations-risk-level: Critical` in the `ingress-nginx-controller` ConfigMap. + +--- + +## UI ingress (SPA) + +The console and portal are Angular SPAs. The Gravitee chart injects `rewrite-target: /$1` by default. Without a capture group in the path, all asset requests are rewritten to `/` (index.html served instead of CSS/JS). + +Fix applied: +```yaml +path: /(.*) +pathType: ImplementationSpecific +annotations: + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/rewrite-target: /$1 +``` + +--- + +## Deployment + +### Prerequisites + +- k3s with nginx ingress and cert-manager installed +- Helm repos added: `bitnami`, `elastic`, `graviteeio` +- `htpasswd` available (for admin BCrypt hash generation) + +### First install + +```bash +# 1. Create secrets +./secrets-create.sh + +# 2. Deploy PKI +kubectl apply -f certificates.yml + +# 3. Deploy backends +helm upgrade --install mongodb bitnami/mongodb -n gravitee-apim -f mongo-values.yml +helm upgrade --install elasticsearch elastic/elasticsearch -n gravitee-apim -f elastic-values.yml + +# 4. Deploy Gravitee +helm upgrade --install graviteeio-apim graviteeio/apim -n gravitee-apim -f apim-values.yml +``` + +### Rebuild workloads only (secrets and certs preserved) + +```bash +helm uninstall graviteeio-apim mongodb elasticsearch -n gravitee-apim +kubectl delete pvc -n gravitee-apim --all + +helm upgrade --install mongodb bitnami/mongodb -n gravitee-apim -f mongo-values.yml +helm upgrade --install elasticsearch elastic/elasticsearch -n gravitee-apim -f elastic-values.yml +helm upgrade --install graviteeio-apim graviteeio/apim -n gravitee-apim -f apim-values.yml +``` + +### /etc/hosts + +``` +192.168.1.18 console.gravitee.sttlab.pc portal.gravitee.sttlab.pc api.gravitee.sttlab.pc gateway.gravitee.sttlab.pc +``` + +--- + +## Access + +| URL | Description | +|-----|-------------| +| https://console.gravitee.sttlab.pc | Management console | +| https://portal.gravitee.sttlab.pc | Developer portal | +| https://api.gravitee.sttlab.pc/management | Management API | +| https://gateway.gravitee.sttlab.pc | Gateway (data plane) | + +**Admin credentials:** +```bash +# Username: admin +# Password (plaintext): +kubectl get secret gravitee-admin -n gravitee-apim \ + -o jsonpath='{.data.admin-password-plain}' | base64 -d && echo +``` + +--- + +## Security notes + +- The Gravitee CA is self-signed. Replace with a Vault PKI Issuer in production (see comment in `certificates.yml`). +- `proxy-ssl-verify: "on"` is enabled on all backend ingresses — nginx validates pod certificates. +- Client mTLS is not enabled for MongoDB (complexity vs. benefit for intra-cluster traffic). +- `annotations-risk-level: Critical` on the ingress controller is required for `proxy_ssl_name` snippets — document this in the ops runbook. diff --git a/apim-values.yml b/apim-values.yml index f42a555..a7818be 100644 --- a/apim-values.yml +++ b/apim-values.yml @@ -7,24 +7,6 @@ adminAccountEnable: true adminPasswordBcrypt: "${GRAVITEE_ADMIN_PASSWORD_BCRYPT}" -# External MongoDB — URI injected at runtime via GRAVITEE_MANAGEMENT/RATELIMIT_MONGODB_URI -# from the gravitee-mongodb-uri secret (see deployment.envFrom below) -mongo: - dbhost: mongodb.gravitee-apim.svc.cluster.local - dbname: gravitee - dbport: 27017 - rsEnabled: false - -# External Elasticsearch (HTTPS + basic auth) -# Password injected at runtime via env var from gravitee-es-master-credentials secret -es: - endpoints: - - https://gravitee-es-master.gravitee-apim.svc.cluster.local:9200 - security: - enabled: true - username: elastic - password: "" - # ============================================================ # API Gateway (data plane) - 2 replicas # ============================================================ @@ -60,12 +42,26 @@ gateway: mountPath: /run/secrets/truststore readOnly: true - deployment: - envFrom: - - secretRef: - name: gravitee-mongodb-uri - env: + - name: GRAVITEE_MANAGEMENT_MONGODB_URI + valueFrom: + secretKeyRef: + name: gravitee-mongodb-uri + key: GRAVITEE_MANAGEMENT_MONGODB_URI + - name: GRAVITEE_RATELIMIT_MONGODB_URI + valueFrom: + secretKeyRef: + name: gravitee-mongodb-uri + key: GRAVITEE_RATELIMIT_MONGODB_URI + - name: GRAVITEE_REPORTERS_ELASTICSEARCH_ENDPOINTS_0 + value: "https://gravitee-es-master.gravitee-apim.svc.cluster.local:9200" + - name: GRAVITEE_REPORTERS_ELASTICSEARCH_SECURITY_ENABLED + value: "true" + - name: GRAVITEE_REPORTERS_ELASTICSEARCH_SECURITY_USERNAME + valueFrom: + secretKeyRef: + name: gravitee-es-master-credentials + key: username - name: GRAVITEE_REPORTERS_ELASTICSEARCH_SECURITY_PASSWORD valueFrom: secretKeyRef: @@ -104,9 +100,11 @@ gateway: enabled: true ingressClassName: nginx annotations: - # Gateway already terminates TLS internally; nginx forwards as HTTPS nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" - nginx.ingress.kubernetes.io/proxy-ssl-verify: "off" + nginx.ingress.kubernetes.io/proxy-ssl-verify: "on" + nginx.ingress.kubernetes.io/proxy-ssl-secret: "gravitee-apim/gravitee-ca-trust" + nginx.ingress.kubernetes.io/configuration-snippet: | + proxy_ssl_name gateway.gravitee.sttlab.pc; hosts: - gateway.gravitee.sttlab.pc path: / @@ -153,19 +151,36 @@ api: mountPath: /run/secrets/truststore readOnly: true - deployment: - envFrom: - - secretRef: - name: gravitee-mongodb-uri - - secretRef: - name: gravitee-jwt - env: + - name: GRAVITEE_MANAGEMENT_MONGODB_URI + valueFrom: + secretKeyRef: + name: gravitee-mongodb-uri + key: GRAVITEE_MANAGEMENT_MONGODB_URI + - name: GRAVITEE_RATELIMIT_MONGODB_URI + valueFrom: + secretKeyRef: + name: gravitee-mongodb-uri + key: GRAVITEE_RATELIMIT_MONGODB_URI + - name: GRAVITEE_ANALYTICS_ELASTICSEARCH_ENDPOINTS_0 + value: "https://gravitee-es-master.gravitee-apim.svc.cluster.local:9200" + - name: GRAVITEE_ANALYTICS_ELASTICSEARCH_SECURITY_ENABLED + value: "true" + - name: GRAVITEE_JWT_SECRET + valueFrom: + secretKeyRef: + name: gravitee-jwt + key: GRAVITEE_JWT_SECRET - name: GRAVITEE_ADMIN_PASSWORD_BCRYPT valueFrom: secretKeyRef: name: gravitee-admin key: admin-password-bcrypt + - name: GRAVITEE_ANALYTICS_ELASTICSEARCH_SECURITY_USERNAME + valueFrom: + secretKeyRef: + name: gravitee-es-master-credentials + key: username - name: GRAVITEE_ANALYTICS_ELASTICSEARCH_SECURITY_PASSWORD valueFrom: secretKeyRef: @@ -209,7 +224,10 @@ api: ingressClassName: nginx annotations: nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" - nginx.ingress.kubernetes.io/proxy-ssl-verify: "off" + nginx.ingress.kubernetes.io/proxy-ssl-verify: "on" + nginx.ingress.kubernetes.io/proxy-ssl-secret: "gravitee-apim/gravitee-ca-trust" + nginx.ingress.kubernetes.io/configuration-snippet: | + proxy_ssl_name api.gravitee.sttlab.pc; path: /management pathType: Prefix hosts: @@ -223,7 +241,10 @@ api: ingressClassName: nginx annotations: nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" - nginx.ingress.kubernetes.io/proxy-ssl-verify: "off" + nginx.ingress.kubernetes.io/proxy-ssl-verify: "on" + nginx.ingress.kubernetes.io/proxy-ssl-secret: "gravitee-apim/gravitee-ca-trust" + nginx.ingress.kubernetes.io/configuration-snippet: | + proxy_ssl_name api.gravitee.sttlab.pc; path: /portal pathType: Prefix hosts: