commit e1577fe06113c83595ce86009b304c82d980fc27 Author: sttlab Date: Wed Jul 22 14:41:10 2026 +0000 first commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f8bc430 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*secret.yaml +.claude \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..8c402b8 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,218 @@ +# AGENTS.md — Tech Watch Automation Stack + +Instructions for implementing an automated technology-watch (veille) pipeline on Kubernetes. + +## Goal + +Deploy a self-hosted stack that: +1. Aggregates tech news sources (RSS and APIs) via **Miniflux** +2. Orchestrates workflows via **n8n** (fetch unread articles, deduplicate, summarize, deliver digest) +3. Uses an **LLM through an OpenAI-compatible API** (OpenRouter today, possibly a local LLM proxy later — never hardcode OpenRouter specifics) +4. Produces a **weekly (and optionally daily) digest**, prioritized by topic, delivered by email and/or saved as Markdown + +## Architecture + +``` +[RSS/API sources] --> [Miniflux] --(REST API)--> [n8n workflows] --> [LLM (OpenAI-compatible)] --> [Digest: email / file / webhook] + | | + [PostgreSQL] <------------------+ (n8n also uses Postgres) +``` + +## Stack & Constraints + +- **Kubernetes** deployment (target: any conformant cluster). Prefer **Helm charts** where official/community charts exist; otherwise write plain manifests (Kustomize layout welcome). +- **Miniflux**: official container image `miniflux/miniflux`. Needs PostgreSQL. Enable API access. Run DB migrations via env `RUN_MIGRATIONS=1`. +- **PostgreSQL**: one instance (or one per app if simpler), e.g. Bitnami chart or CloudNativePG. Persistent volumes required. +- **n8n**: official image `n8nio/n8n`. Configure with Postgres backend (not SQLite). Persist `/home/node/.n8n`. +- **LLM access**: strictly through the **OpenAI-compatible chat completions interface** (`/v1/chat/completions`). Configuration via env vars only: + - `LLM_BASE_URL` (e.g. `https://openrouter.ai/api/v1`, later a local proxy URL) + - `LLM_API_KEY` + - `LLM_MODEL` (model identifier string, treat as opaque) + - Do not use vendor-specific SDK features; plain HTTP calls or the generic OpenAI node in n8n with a custom base URL. +- **Secrets**: Kubernetes Secrets (no secrets in git). Provide `*.example` files. +- **Ingress**: expose Miniflux and n8n UIs behind an Ingress (TLS assumed handled by cluster / cert-manager). Make hostnames configurable. +- **Code and comments in English.** + +## Implementation Tasks + +1. **Manifests/Charts** + - Namespace `veille` (configurable) + - PostgreSQL with PVC, credentials in Secret + - Miniflux Deployment + Service + Ingress; admin credentials in Secret; `DATABASE_URL` from Secret + - n8n Deployment + Service + Ingress; `DB_TYPE=postgresdb` config; encryption key in Secret + - NetworkPolicies (optional, nice to have): only n8n → Miniflux, apps → Postgres +2. **Miniflux bootstrap** + - Script or Job that creates the feed list (see Sources below) via the Miniflux REST API (`POST /v1/feeds`), organized in categories matching the topics + - Create an API token for n8n +3. **n8n workflows** (export as JSON in repo, `workflows/`) + - **Digest workflow** (cron, weekly Friday 07:00 Europe/Paris; optional daily variant): + 1. `GET /v1/entries?status=unread&limit=...` from Miniflux + 2. Group entries by category/topic + 3. For each topic: call LLM to summarize + rank top items (relevance to the topics listed below); one LLM call per topic to control context size + 4. Compose final digest (Markdown): per topic — 3–5 top items with 1–2 sentence summaries + links, then a short "weak signals" section + 5. Deliver: send email (SMTP node) and/or commit/save Markdown; then mark entries as read (`PUT /v1/entries` status=read) + - **Error handling**: retries on HTTP calls, dead-letter notification on failure +4. **Docs** + - `README.md`: deployment steps, secret setup, how to add a source, how to change LLM endpoint +5. **Validation** + - Lint manifests (`kubectl apply --dry-run=client` or `kubeconform`) + - Provide a smoke-test checklist + +## Digest Priorities (for LLM prompt) + +Rank content by relevance to, in order: +1. Application & data integration (iPaaS, ESB, API management) +2. Event-driven architecture, messaging, streaming (Kafka, AMQP, MQTT, AsyncAPI, CloudEvents) +3. Agentic AI, LLM integration patterns, MCP (Model Context Protocol) +4. Cloud native / Kubernetes / platform engineering +5. MFT (Managed File Transfer) and B2B/EDI integration +6. General cloud computing announcements (only if significant) + +## Sources to Configure in Miniflux + +All sources below are RSS/Atom unless stated otherwise. If a feed URL is broken at implementation time, locate the current one (feeds move); as a fallback use an HTML-to-RSS bridge (e.g. RSSHub) or n8n HTTP scraping. + +### Cloud Native / Kubernetes +| Source | Access | URL | +|---|---|---| +| CNCF blog | RSS | https://www.cncf.io/feed/ | +| Kubernetes blog | RSS | https://kubernetes.io/feed.xml | +| The New Stack | RSS | https://thenewstack.io/feed/ | +| InfoQ – Cloud | RSS | https://feed.infoq.com/cloud-computing/ | +| InfoQ – DevOps | RSS | https://feed.infoq.com/devops/ | + +### Cloud Providers +| Source | Access | URL | +|---|---|---| +| AWS News Blog | RSS | https://aws.amazon.com/blogs/aws/feed/ | +| AWS Compute/Integration blogs | RSS | discover per-category feeds under aws.amazon.com/blogs/ | +| Azure updates | RSS | https://azurecomm.azure.com/updates/feed/ (verify; fallback: Azure blog https://azure.microsoft.com/en-us/blog/feed/) | +| Google Cloud blog | RSS | https://cloudblog.withgoogle.com/rss/ | + +### Integration / iPaaS / API +| Source | Access | URL | +|---|---|---| +| MuleSoft blog | RSS | https://blogs.mulesoft.com/feed/ | +| Boomi blog | RSS | https://boomi.com/blog/feed/ (verify) | +| Kong blog | RSS | https://konghq.com/blog/rss.xml (verify) | +| Apache Camel blog | RSS | https://camel.apache.org/feed.xml (verify) | +| WSO2 blog | RSS | https://wso2.com/feed/ (verify) | + +### EDA / Messaging / Streaming +| Source | Access | URL | +|---|---|---| +| Confluent blog | RSS | https://www.confluent.io/blog/feed/ (verify) | +| Kai Waehner | RSS | https://www.kai-waehner.de/blog/feed/ | +| Solace blog | RSS | https://solace.com/feed/ | +| AsyncAPI blog | RSS | https://www.asyncapi.com/rss.xml (verify) | +| RabbitMQ blog | RSS | https://www.rabbitmq.com/blog/rss.xml (verify) | +| Apache Kafka releases | Atom (GitHub) | https://github.com/apache/kafka/releases.atom | + +### Agentic AI / LLM +| Source | Access | URL | +|---|---|---| +| Anthropic news | RSS | https://www.anthropic.com/rss.xml (verify; fallback RSSHub) | +| OpenAI blog | RSS | https://openai.com/blog/rss.xml (verify) | +| LangChain blog | RSS | https://blog.langchain.dev/rss/ | +| Simon Willison | RSS | https://simonwillison.net/atom/everything/ | +| Latent Space | RSS | https://www.latent.space/feed | +| MCP spec repo releases | Atom (GitHub) | https://github.com/modelcontextprotocol/modelcontextprotocol/releases.atom (verify repo path) | + +### MFT / B2B / EDI +| Source | Access | URL | +|---|---|---| +| Axway blog | RSS | https://blog.axway.com/feed (verify) | +| Cleo blog | RSS | https://www.cleo.com/blog/rss.xml (verify) | +| SEEBURGER blog | RSS | https://blog.seeburger.com/feed/ (verify) | +| IBM Sterling / integration topics | RSS or scrape | discover on ibm.com/blog; fallback RSSHub | + +### Cross-cutting / Signal +| Source | Access | URL | +|---|---|---| +| Hacker News (front page) | RSS | https://news.ycombinator.com/rss | +| HN best (less noise) | RSS | https://hnrss.org/best | +| ThoughtWorks Technology Radar | RSS | https://www.thoughtworks.com/rss/insights.xml (verify) — biannual, high value | +| Reddit r/devops | RSS | https://www.reddit.com/r/devops/.rss | +| Reddit r/kubernetes | RSS | https://www.reddit.com/r/kubernetes/.rss | +| Reddit r/dataengineering | RSS | https://www.reddit.com/r/dataengineering/.rss | + +### Security / Vulnerabilities +| Source | Access | URL | +|---|---|---| +| CISA Known Exploited Vulnerabilities | RSS/JSON | https://www.cisa.gov/cybersecurity-advisories/all.xml (advisories); KEV catalog JSON: https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json (poll via n8n) | +| Kubernetes security announces | Google Group RSS | https://groups.google.com/g/kubernetes-security-announce (verify RSS availability; fallback: kubernetes.io blog security tag) | +| GitHub Security Advisories | Atom per ecosystem/repo | https://github.com/advisories (filterable; per-repo: `https://github.com/{org}/{repo}/security/advisories` — poll via n8n/API) | +| The Hacker News | RSS | https://feeds.feedburner.com/TheHackersNews | +| Bleeping Computer | RSS | https://www.bleepingcomputer.com/feed/ | +| NVD CVE feed | JSON API | https://services.nvd.nist.gov/rest/json/cves/2.0 — poll via n8n with keyword filters (kafka, kubernetes, n8n, miniflux, postgresql…); too noisy for raw RSS | + +Digest handling: security items form a dedicated "Security" section in the digest, filtered by relevance to the deployed/covered stack (Kubernetes, Kafka, PostgreSQL, n8n, integration middleware). Critical/KEV items may trigger an immediate notification instead of waiting for the weekly digest. + +### Non-RSS sources (handled in n8n, not Miniflux) +- **GitHub releases** for key projects: use `https://github.com/{org}/{repo}/releases.atom` (works in Miniflux) — add repos as needed (kubernetes, istio, keda, camel, kafka, langchain, n8n…) +- **Google Alerts**: create alerts ("MFT managed file transfer", "EDI modernization", "iPaaS"), deliver as RSS, add to Miniflux +- **LinkedIn / Gartner summaries**: no reliable API; keep manual + +### Market Trends / Business (M&A, funding, analyst views) +| Source | Access | URL | +|---|---|---| +| TechCrunch – Enterprise | RSS | https://techcrunch.com/category/enterprise/feed/ | +| The Register | RSS | https://www.theregister.com/headlines.atom | +| SiliconANGLE | RSS | https://siliconangle.com/feed/ (enterprise/cloud coverage, theCUBE analysts) | +| Futurum Group | RSS | https://futurumgroup.com/feed/ (verify) — accessible analyst research | +| Constellation Research blog | RSS | https://www.constellationr.com/rss.xml (verify) | +| CNCF Annual Survey & reports | Scrape/manual | https://www.cncf.io/reports/ — annual, high value | +| DB-Engines ranking | Scrape/manual | https://db-engines.com/en/ranking — monthly trend check via n8n | +| Gartner / Forrester | Manual | paywalled; capture via vendors' "named leader" announcements (already covered by vendor blog feeds) and analyst LinkedIn posts | + +Annual reports worth flagging in the digest when released: CNCF Survey, Stack Overflow Developer Survey, DORA State of DevOps, ThoughtWorks Tech Radar (biannual), State of the API (Postman). + +Digest handling: market items go in a dedicated "Market & Trends" section — acquisitions, funding rounds, major partnerships, analyst rankings — restricted to the integration/EDA/cloud/AI space. Example of relevant signal: the IBM acquisition of Confluent ($11B, 2026). + +### People to Follow +For each person: prefer their **personal blog RSS** if one exists (check first); otherwise create a **Google Alert** on their quoted name, delivered as RSS, and add it to Miniflux in a "People" category. + +**Integration / iPaaS / API** +- Kai Waehner — blog RSS exists (already listed above); independent since leaving Confluent (2026) +- Sanjeev Mohan (data & analytics, ex-Gartner) +- Massimo Pezzini (ex-Gartner, iPaaS/HIP reference) +- Saurabh Sharma (integration analyst) +- Holger Mueller (Constellation Research) + +**EDA / Messaging / Streaming** +- Gwen Shapira (Kafka) +- Adam Bellemare (event-driven microservices) +- Hugo Guerrero (API/messaging, Red Hat) +- Clemens Vasters (Microsoft, messaging/CloudEvents) + +**Agentic AI / LLM** +- Simon Willison — blog RSS exists (already listed above) +- Andrew Ng +- Harrison Chase (LangChain) +- swyx / Shawn Wang (Latent Space — RSS already listed) +- Chip Huyen (ML systems) + +**Cloud native / Platform engineering** +- Kelsey Hightower +- Viktor Farcic (DevOps Toolkit) +- Sam Newman (microservices) +- Charity Majors (observability) + +**Architecture** +- Gregor Hohpe (Enterprise Integration Patterns) — check for blog RSS (architectelevator.com) +- Neal Ford (ThoughtWorks) +- Mark Richards (developertoasarchitect.com) + +**MFT / B2B / EDI**: no strong public figures; the iPaaS analysts above (Pezzini, Sharma) also cover B2B. + + +## Notes for the Agent + +- **Truncated feeds**: many feeds (media, vendor blogs) only carry title + excerpt, not full content. Strategy: + 1. Enable Miniflux **"Fetch original content"** (scraper) per feed for truncated sources — set `crawler: true` when creating feeds via the API (test each feed; disable if the site blocks scraping). + 2. In the n8n digest workflow, treat `content` defensively: if the entry content is below ~500 characters, either summarize from title + excerpt only, or fetch the article URL directly (HTTP node) as fallback. Never assume full text is available. +- Verify every feed URL with an HTTP GET before adding it; log and replace dead feeds. +- Keep the LLM prompt for summarization in a dedicated file (`prompts/digest.md`) so it can be tuned without touching workflows. +- Digest language: **French** output for the digest content; everything else (code, comments, docs) in English. +- Keep resource requests modest (this is a personal stack): e.g. 256Mi/0.1 CPU requests per app, adjust limits sensibly. +- Timezone: Europe/Paris for all cron schedules. diff --git a/README.md b/README.md new file mode 100644 index 0000000..f9c9a76 --- /dev/null +++ b/README.md @@ -0,0 +1,286 @@ +# Tech Watch Automation Stack — Miniflux + n8n + digest workflow + +This covers the full stack described in `AGENTS.md` (namespace `watch`, +shared PostgreSQL in namespace `postgres`, Miniflux with the initial feed +bootstrap, n8n for workflow orchestration, and the daily LLM digest +workflow) plus a small read-only file server for the generated digests. +Everything below is deployed and active on this cluster. + +## Layout + +``` +k8s/ + namespace.yaml # namespace: watch (apps) + postgres/ # shared PostgreSQL instance (namespace: postgres) + namespace.yaml + secret.example.yaml # superuser credentials + statefulset.yaml + service.yaml + miniflux/ # namespace: watch (except create-db-job.yaml) + secret.example.yaml # DATABASE_URL, admin credentials + deployment.yaml + service.yaml + gateway.yaml # Gateway API (Envoy), HTTPS listener on 443 + httproute.yaml # routes miniflux.sttlab.pc to the Service + create-db-job.yaml # runs in ns postgres: creates the "miniflux" database + bootstrap-feeds-job.yaml # runs miniflux-initializer to create categories/feeds + n8n/ # namespace: watch (except create-db-job.yaml) + secret.example.yaml # DB credentials, N8N_ENCRYPTION_KEY + miniflux-token-secret.example.yaml # Miniflux API token, backup/reference for the n8n credential + pvc.yaml # n8n-data (n8n's own /home/node/.n8n) + n8n-digest-workspace (/data, for Read/Write File nodes) + deployment.yaml + service.yaml + gateway.yaml # Gateway API (Envoy), HTTPS listener on 443 + httproute.yaml # routes n8n.sttlab.pc to the Service + create-db-job.yaml # runs in ns postgres: creates the "n8n" database + digest-files/ # namespace: watch — read-only nginx file server + configmap.yaml # nginx.conf: autoindex on + deployment.yaml # mounts n8n-digest-workspace PVC (digests/ subdir), read-only + service.yaml + gateway.yaml # routes digests.sttlab.pc + httproute.yaml +miniflux-initializer/ # container image used by bootstrap-feeds-job.yaml + Dockerfile + bootstrap_feeds.py + feeds.yaml + requirements.txt +prompts/ + digest.md # LLM system prompt for the digest workflow; mounted into n8n via + # the n8n-digest-prompt ConfigMap (regenerate after editing, see below) +workflows/ + digest.json # "Tech Watch Digest" n8n workflow (daily 06:00 Europe/Paris cron) +``` + +## Deploy + +1. **Namespaces** + ``` + kubectl apply -f k8s/namespace.yaml + kubectl apply -f k8s/postgres/namespace.yaml + ``` + +2. **PostgreSQL** — `k8s/postgres/{namespace,secret.example,statefulset,service}.yaml` + deploy a standalone instance and are kept for reference/portability, but + **on this cluster a shared PostgreSQL already runs** in the `postgres` + namespace (Bitnami Helm chart, release `postgres`, service + `postgres-postgresql`). Reuse it instead of applying the StatefulSet: + ``` + cp k8s/postgres/secret.example.yaml k8s/postgres/secret.yaml + # edit k8s/postgres/secret.yaml: POSTGRES_PASSWORD must match the real + # instance's password: + # kubectl get secret postgres-postgresql -n postgres -o jsonpath='{.data.postgres-password}' | base64 -d + kubectl apply -f k8s/postgres/secret.yaml + kubectl apply -f k8s/miniflux/create-db-job.yaml + kubectl wait --for=condition=complete job/create-db-miniflux -n postgres --timeout=60s + ``` + (If deploying to a cluster with no existing shared Postgres, apply + `statefulset.yaml`/`service.yaml` first and update + `create-db-job.yaml`'s host from `postgres-postgresql` to `postgres`.) + +3. **TLS secret** for the Gateway listeners (using the existing wildcard cert for `*.sttlab.pc`) + ``` + kubectl create secret tls sttlab-pc-tls \ + --cert=~/tls/sttlab.pc.crt --key=~/tls/sttlab.pc.key \ + -n watch + ``` + +4. **Miniflux** + ``` + cp k8s/miniflux/secret.example.yaml k8s/miniflux/secret.yaml + # edit k8s/miniflux/secret.yaml: + # - DATABASE_URL password must match k8s/postgres/secret.yaml + # - set ADMIN_USERNAME / ADMIN_PASSWORD for the Miniflux admin account + kubectl apply -f k8s/miniflux/secret.yaml + kubectl apply -f k8s/miniflux/deployment.yaml + kubectl apply -f k8s/miniflux/service.yaml + kubectl apply -f k8s/miniflux/gateway.yaml + kubectl apply -f k8s/miniflux/httproute.yaml + ``` + + Exposed via Gateway API (Envoy Gateway, `gatewayClassName: envoy`) on + port 443 — this cluster's `ingress-nginx` only exposes ports 80/9443 + externally, not 443, so a plain `Ingress` would not actually be + reachable over HTTPS. Point DNS for `miniflux.sttlab.pc` at the Gateway + address if not already done (`kubectl get gateway miniflux -n watch`). + + Feed `crawler` (fetch original article content instead of the raw + RSS/Atom excerpt) is enabled on all feeds except OpenAI's — its blog is + behind a Cloudflare bot challenge that always rejects Miniflux's + fetcher, so it stays on the RSS excerpt. See `crawler:` per feed in + `miniflux-initializer/feeds.yaml`. + +5. **Bootstrap feeds** — runs as a one-off Job using a locally built image + (no external registry needed on a single-node k3s cluster): + ``` + cd miniflux-initializer + docker build --network=host -t miniflux-initializer:latest . + docker save miniflux-initializer:latest | sudo k3s ctr images import - + cd .. + kubectl apply -f k8s/miniflux/bootstrap-feeds-job.yaml + kubectl wait --for=condition=complete job/bootstrap-feeds -n watch --timeout=60s + kubectl logs -n watch job/bootstrap-feeds + ``` + Safe to re-run: delete the Job (`kubectl delete job bootstrap-feeds -n + watch`) and re-apply after editing `miniflux-initializer/feeds.yaml` + (rebuild + re-import the image first if you changed it). Only adds new + feeds — it does not update `crawler`/settings on feeds that already + exist (use the Miniflux API or UI for that). + + > Note: `--network=host` is required for the `pip install` step during + > the image build — the local router does not answer DNS queries from + > Docker's bridge subnet, only the host's own resolver works. + +6. **API token for n8n**: create it manually in the Miniflux UI under + *Settings → API Keys* — there is no public REST endpoint to create one. + Kept as the source of truth in `k8s/n8n/miniflux-token-secret.yaml` (see + `.example` for the format); the n8n credential itself is created in step + 8 below, from the same token. + +7. **n8n** + ``` + cp k8s/n8n/secret.example.yaml k8s/n8n/secret.yaml + # edit k8s/n8n/secret.yaml: + # - DB_POSTGRESDB_PASSWORD must match k8s/postgres/secret.yaml + # - N8N_ENCRYPTION_KEY: generate once with `openssl rand -hex 24` and + # never rotate it afterwards (it decrypts every saved credential) + kubectl apply -f k8s/n8n/secret.yaml + kubectl apply -f k8s/n8n/create-db-job.yaml + kubectl wait --for=condition=complete job/create-db-n8n -n postgres --timeout=60s + kubectl apply -f k8s/n8n/pvc.yaml + # ConfigMap holding the digest LLM prompt (prompts/digest.md is the source + # of truth; regenerate and re-apply this any time the file changes): + kubectl create configmap n8n-digest-prompt --from-file=digest.md=prompts/digest.md \ + -n watch --dry-run=client -o yaml | kubectl apply -f - + kubectl apply -f k8s/n8n/deployment.yaml + kubectl apply -f k8s/n8n/service.yaml + kubectl apply -f k8s/n8n/gateway.yaml + kubectl apply -f k8s/n8n/httproute.yaml + ``` + Same Gateway API setup as Miniflux (TLS terminated by Envoy Gateway on + 443, reusing the `sttlab-pc-tls` secret). Point DNS for `n8n.sttlab.pc` + at the Gateway address once (`kubectl get gateway n8n -n watch`). + + On first login, n8n prompts to create the owner account (email/password) + — there's no `CREATE_ADMIN`-style env var like Miniflux, so this step is + manual in the UI. + + Two PVCs are mounted: `n8n-data` is n8n's own `/home/node/.n8n` (config, + binary data cache); `n8n-digest-workspace` is mounted separately at + `/data` for the digest workflow's Read/Write File nodes (prompt + + generated digests). They're kept apart because n8n unconditionally + blocks file-node access to its own `/home/node/.n8n` directory + (`N8N_BLOCK_FILE_ACCESS_TO_N8N_FILES`, not overridable via + `N8N_RESTRICT_FILE_ACCESS_TO`) — a dedicated volume avoids that + restriction honestly instead of working around it. Note also that + `N8N_RESTRICT_FILE_ACCESS_TO` only accepts a single path, not a list — + hence both `prompts/` and `digests/` live under the same `/data` root. + +8. **Digest workflow** — import `workflows/digest.json` (the "Tech Watch + Digest" workflow: daily 06:00 Europe/Paris cron) and wire its + credentials via the n8n Public API (Settings → n8n API → create a key + first): + ``` + N8N_KEY= + + # Miniflux credential (Header Auth, X-Auth-Token) — use the same token + # as k8s/n8n/miniflux-token-secret.yaml + curl -s https://n8n.sttlab.pc/api/v1/credentials -X POST \ + -H "X-N8N-API-KEY: $N8N_KEY" -H "Content-Type: application/json" \ + -d '{"name":"Miniflux API","type":"httpHeaderAuth","data":{"name":"X-Auth-Token","value":"","allowedHttpRequestDomains":"all"}}' + + # LLM credential (Header Auth, Authorization: Bearer ) — OpenRouter or + # any OpenAI-compatible provider + curl -s https://n8n.sttlab.pc/api/v1/credentials -X POST \ + -H "X-N8N-API-KEY: $N8N_KEY" -H "Content-Type: application/json" \ + -d '{"name":"LLM API","type":"httpHeaderAuth","data":{"name":"Authorization","value":"Bearer ","allowedHttpRequestDomains":"all"}}' + + # Import the workflow, then edit the two credential IDs in its HTTP + # Request nodes ("Get Unread Entries" / "Mark Entries As Read" -> Miniflux + # API; "Summarize Category" -> LLM API) to match the IDs returned above, + # and set the real llmBaseUrl / llmModel in the "Init" node's code. + curl -s https://n8n.sttlab.pc/api/v1/workflows -X POST \ + -H "X-N8N-API-KEY: $N8N_KEY" -H "Content-Type: application/json" \ + --data-binary @<(python3 -c "import json; wf=json.load(open('workflows/digest.json')); print(json.dumps({k: wf[k] for k in ('name','nodes','connections','settings')}))") + + # Activate once credentials are wired and a manual run succeeds: + curl -s https://n8n.sttlab.pc/api/v1/workflows//activate -X POST \ + -H "X-N8N-API-KEY: $N8N_KEY" + ``` + The workflow uses Miniflux's **internal** cluster Service DNS + (`http://miniflux.watch.svc.cluster.local/v1/entries`), not the external + HTTPS hostname — `miniflux.sttlab.pc` isn't resolvable from inside the + cluster (it only exists as a Gateway route, no DNS record). + + To test-run before activating (the running Deployment pod can't run + `n8n execute` itself — its Task Broker port is already bound — so use a + throwaway Job with the same image/env/volumes instead), or just use the + "Test workflow" button in the n8n UI from the Schedule Trigger node. + + Pipeline: fetch unread Miniflux entries from the last 24h → group by + category → for each category, call the LLM (system prompt from + `prompts/digest.md`) to pick the 3-5 most relevant items + weak signals, + strict JSON output → compose one Markdown file → write it to + `/data/digests/digest-.md` → mark the processed entries as read in + Miniflux. + +9. **Digest file server** — serves `/data/digests/` (read-only) so digests + can be browsed/downloaded from a browser instead of email: + ``` + kubectl apply -f k8s/digest-files/ + ``` + Same Gateway/TLS pattern, routes `digests.sttlab.pc`. Point DNS at the + Gateway address (`kubectl get gateway digest-files -n watch`). + +## How to add a source + +Add an entry under the relevant category (or a new category) in +`miniflux-initializer/feeds.yaml`, verify the URL responds with `curl -I +`, then rebuild the image and re-run the bootstrap Job (see step 5). + +## Validate manifests + +``` +kubectl apply --dry-run=client -f k8s/namespace.yaml +kubectl apply --dry-run=client -f k8s/postgres/ +kubectl apply --dry-run=client -f k8s/miniflux/ +kubectl apply --dry-run=client -f k8s/n8n/ +kubectl apply --dry-run=client -f k8s/digest-files/ +``` + +(Run against a cluster with the `watch`/`postgres` namespaces and secrets +already created, since some manifests reference them.) + +## Smoke-test checklist + +- [ ] `kubectl get pods -n postgres` — postgres pod `Running`, `1/1` ready +- [ ] `kubectl get pods -n watch` — miniflux pod `Running`, `1/1` ready +- [ ] `kubectl logs -n watch deploy/miniflux` shows migrations applied, no errors +- [ ] `https://miniflux.sttlab.pc/healthcheck` returns `OK` +- [ ] Log in to the Miniflux UI with the admin credentials from the secret +- [ ] `kubectl logs -n watch job/bootstrap-feeds` shows categories/feeds created +- [ ] Categories and feeds appear in the Miniflux UI +- [ ] At least one feed shows fetched entries (Miniflux polls periodically; use "Refresh" in the UI to force it) +- [ ] `GET /v1/entries?status=unread` (with an API token) returns entries +- [ ] `kubectl get pods -n watch` — n8n pod `Running`, `1/1` ready +- [ ] `kubectl logs -n watch deploy/n8n` shows no DB connection errors +- [ ] `https://n8n.sttlab.pc/healthz` returns `{"status":"ok"}` +- [ ] Log in to the n8n UI and create the owner account +- [ ] n8n can reach Miniflux: an HTTP Request node to + `http://miniflux.watch.svc.cluster.local/v1/me` with the API token + returns the admin user +- [ ] "Tech Watch Digest" workflow imported, both credentials (Miniflux API, + LLM API) attached to their HTTP Request nodes +- [ ] A manual run ("Test workflow" in the UI, or a throwaway + `n8n execute --id=` Job) produces a file under `/data/digests/` + in the n8n pod, and the corresponding Miniflux entries are marked read +- [ ] Workflow activated (`"active": true`) +- [ ] `kubectl get pods -n watch` — digest-files pod `Running`, `1/1` ready +- [ ] `https://digests.sttlab.pc/` lists the generated digest files + +## Current status + +Deployed and active: Miniflux (9 feeds), n8n, "Tech Watch Digest" workflow +(daily 06:00 Europe/Paris, OpenRouter `deepseek/deepseek-v4-flash`), and +the digest file server at `https://digests.sttlab.pc/`. No email delivery +(SMTP not configured) — digests are Markdown files served over HTTP +instead. diff --git a/k8s/digest-files/configmap.yaml b/k8s/digest-files/configmap.yaml new file mode 100644 index 0000000..582d762 --- /dev/null +++ b/k8s/digest-files/configmap.yaml @@ -0,0 +1,20 @@ +# nginx config for the digest file server: directory listing (autoindex) +# over the generated Markdown digests, no app logic involved. +apiVersion: v1 +kind: ConfigMap +metadata: + name: digest-files-nginx-conf + namespace: watch +data: + default.conf: | + server { + listen 8080; + server_name _; + root /usr/share/nginx/html; + + location / { + autoindex on; + autoindex_exact_size off; + autoindex_localtime on; + } + } diff --git a/k8s/digest-files/deployment.yaml b/k8s/digest-files/deployment.yaml new file mode 100644 index 0000000..47f30fd --- /dev/null +++ b/k8s/digest-files/deployment.yaml @@ -0,0 +1,61 @@ +# Read-only HTTP file server exposing the digest workflow's output +# (n8n-digest-workspace PVC, digests/ subdirectory) for browsing/download, +# instead of email delivery (no SMTP configured). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: digest-files + namespace: watch + labels: + app.kubernetes.io/name: digest-files +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: digest-files + template: + metadata: + labels: + app.kubernetes.io/name: digest-files + spec: + containers: + - name: nginx + image: nginxinc/nginx-unprivileged:alpine + ports: + - name: http + containerPort: 8080 + volumeMounts: + - name: digests + mountPath: /usr/share/nginx/html + subPath: digests + readOnly: true + - name: nginx-conf + mountPath: /etc/nginx/conf.d/default.conf + subPath: default.conf + readOnly: true + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 200m + memory: 128Mi + readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 10 + periodSeconds: 20 + volumes: + - name: digests + persistentVolumeClaim: + claimName: n8n-digest-workspace + - name: nginx-conf + configMap: + name: digest-files-nginx-conf diff --git a/k8s/digest-files/gateway.yaml b/k8s/digest-files/gateway.yaml new file mode 100644 index 0000000..c6db3f4 --- /dev/null +++ b/k8s/digest-files/gateway.yaml @@ -0,0 +1,20 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: digest-files + namespace: watch +spec: + gatewayClassName: envoy + listeners: + - name: https + protocol: HTTPS + port: 443 + hostname: digests.sttlab.pc + allowedRoutes: + namespaces: + from: Same + tls: + mode: Terminate + certificateRefs: + - kind: Secret + name: sttlab-pc-tls diff --git a/k8s/digest-files/httproute.yaml b/k8s/digest-files/httproute.yaml new file mode 100644 index 0000000..429e1a4 --- /dev/null +++ b/k8s/digest-files/httproute.yaml @@ -0,0 +1,21 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: digest-files + namespace: watch +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: digest-files + hostnames: + - digests.sttlab.pc + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - kind: Service + name: digest-files + port: 80 diff --git a/k8s/digest-files/service.yaml b/k8s/digest-files/service.yaml new file mode 100644 index 0000000..08fc2c3 --- /dev/null +++ b/k8s/digest-files/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: digest-files + namespace: watch + labels: + app.kubernetes.io/name: digest-files +spec: + selector: + app.kubernetes.io/name: digest-files + ports: + - name: http + port: 80 + targetPort: 8080 diff --git a/k8s/miniflux/bootstrap-feeds-job.yaml b/k8s/miniflux/bootstrap-feeds-job.yaml new file mode 100644 index 0000000..b29d465 --- /dev/null +++ b/k8s/miniflux/bootstrap-feeds-job.yaml @@ -0,0 +1,35 @@ +# Runs miniflux-initializer (see /miniflux-initializer) to create the +# categories/feeds from feeds.yaml via the Miniflux REST API. Idempotent, +# safe to re-run: delete the Job then re-apply after editing feeds.yaml. +# kubectl delete job bootstrap-feeds -n watch --ignore-not-found +# kubectl apply -f k8s/miniflux/bootstrap-feeds-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: bootstrap-feeds + namespace: watch +spec: + backoffLimit: 3 + template: + spec: + restartPolicy: Never + containers: + - name: bootstrap-feeds + image: docker.io/library/miniflux-initializer:latest + imagePullPolicy: Never + env: + - name: MINIFLUX_URL + # Trailing dot makes this an absolute FQDN: musl's resolver + # (Alpine base image) mishandles ndots-relative lookups here + # and fails with EAI_AGAIN otherwise. + value: http://miniflux.watch.svc.cluster.local. + - name: MINIFLUX_USERNAME + valueFrom: + secretKeyRef: + name: miniflux-credentials + key: ADMIN_USERNAME + - name: MINIFLUX_PASSWORD + valueFrom: + secretKeyRef: + name: miniflux-credentials + key: ADMIN_PASSWORD diff --git a/k8s/miniflux/create-db-job.yaml b/k8s/miniflux/create-db-job.yaml new file mode 100644 index 0000000..4b18062 --- /dev/null +++ b/k8s/miniflux/create-db-job.yaml @@ -0,0 +1,28 @@ +# One-off Job to create the "miniflux" database on the shared Postgres +# instance. Re-run (delete + kubectl apply) whenever a new app needs a +# database, with a copy of this Job using a different DB name. +apiVersion: batch/v1 +kind: Job +metadata: + name: create-db-miniflux + namespace: postgres +spec: + backoffLimit: 3 + template: + spec: + restartPolicy: Never + containers: + - name: create-db + image: postgres:18-alpine + envFrom: + - secretRef: + name: postgres-admin + command: + - sh + - -c + - | + set -e + psql "postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres-postgresql:5432/postgres" \ + -tc "SELECT 1 FROM pg_database WHERE datname = 'miniflux'" | grep -q 1 \ + || psql "postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres-postgresql:5432/postgres" \ + -c "CREATE DATABASE miniflux" diff --git a/k8s/miniflux/deployment.yaml b/k8s/miniflux/deployment.yaml new file mode 100644 index 0000000..a91fa67 --- /dev/null +++ b/k8s/miniflux/deployment.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: miniflux + namespace: watch + labels: + app.kubernetes.io/name: miniflux +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: miniflux + template: + metadata: + labels: + app.kubernetes.io/name: miniflux + spec: + containers: + - name: miniflux + image: miniflux/miniflux:latest + ports: + - name: http + containerPort: 8080 + envFrom: + - secretRef: + name: miniflux-credentials + env: + - name: RUN_MIGRATIONS + value: "1" + - name: CREATE_ADMIN + value: "1" + - name: BASE_URL + value: https://miniflux.sttlab.pc + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 1000m + memory: 768Mi + readinessProbe: + httpGet: + path: /healthcheck + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /healthcheck + port: http + initialDelaySeconds: 15 + periodSeconds: 20 diff --git a/k8s/miniflux/gateway.yaml b/k8s/miniflux/gateway.yaml new file mode 100644 index 0000000..036e877 --- /dev/null +++ b/k8s/miniflux/gateway.yaml @@ -0,0 +1,20 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: miniflux + namespace: watch +spec: + gatewayClassName: envoy + listeners: + - name: https + protocol: HTTPS + port: 443 + hostname: miniflux.sttlab.pc + allowedRoutes: + namespaces: + from: Same + tls: + mode: Terminate + certificateRefs: + - kind: Secret + name: sttlab-pc-tls diff --git a/k8s/miniflux/httproute.yaml b/k8s/miniflux/httproute.yaml new file mode 100644 index 0000000..41bcd2c --- /dev/null +++ b/k8s/miniflux/httproute.yaml @@ -0,0 +1,21 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: miniflux + namespace: watch +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: miniflux + hostnames: + - miniflux.sttlab.pc + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - kind: Service + name: miniflux + port: 80 diff --git a/k8s/miniflux/secret.example.yaml b/k8s/miniflux/secret.example.yaml new file mode 100644 index 0000000..9daa255 --- /dev/null +++ b/k8s/miniflux/secret.example.yaml @@ -0,0 +1,15 @@ +# Copy to secret.yaml, fill in real values, and apply with: +# kubectl apply -f secret.yaml +# Never commit the filled-in secret.yaml to git. +# +# POSTGRES_USER/POSTGRES_PASSWORD must match k8s/postgres/secret.yaml. +apiVersion: v1 +kind: Secret +metadata: + name: miniflux-credentials + namespace: watch +type: Opaque +stringData: + DATABASE_URL: postgres://postgres:changeme@postgres-postgresql.postgres.svc.cluster.local:5432/miniflux?sslmode=disable + ADMIN_USERNAME: admin + ADMIN_PASSWORD: changeme diff --git a/k8s/miniflux/service.yaml b/k8s/miniflux/service.yaml new file mode 100644 index 0000000..c23fff5 --- /dev/null +++ b/k8s/miniflux/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: miniflux + namespace: watch + labels: + app.kubernetes.io/name: miniflux +spec: + selector: + app.kubernetes.io/name: miniflux + ports: + - name: http + port: 80 + targetPort: 8080 diff --git a/k8s/n8n/create-db-job.yaml b/k8s/n8n/create-db-job.yaml new file mode 100644 index 0000000..ac239c8 --- /dev/null +++ b/k8s/n8n/create-db-job.yaml @@ -0,0 +1,27 @@ +# One-off Job to create the "n8n" database on the shared Postgres instance. +# Mirrors k8s/miniflux/create-db-job.yaml. +apiVersion: batch/v1 +kind: Job +metadata: + name: create-db-n8n + namespace: postgres +spec: + backoffLimit: 3 + template: + spec: + restartPolicy: Never + containers: + - name: create-db + image: postgres:18-alpine + envFrom: + - secretRef: + name: postgres-admin + command: + - sh + - -c + - | + set -e + psql "postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres-postgresql:5432/postgres" \ + -tc "SELECT 1 FROM pg_database WHERE datname = 'n8n'" | grep -q 1 \ + || psql "postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres-postgresql:5432/postgres" \ + -c "CREATE DATABASE n8n" diff --git a/k8s/n8n/deployment.yaml b/k8s/n8n/deployment.yaml new file mode 100644 index 0000000..2fd1c0e --- /dev/null +++ b/k8s/n8n/deployment.yaml @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: n8n + namespace: watch + labels: + app.kubernetes.io/name: n8n +spec: + replicas: 1 + strategy: + type: Recreate # single ReadWriteOnce PVC, avoid two pods mounting it at once + selector: + matchLabels: + app.kubernetes.io/name: n8n + template: + metadata: + labels: + app.kubernetes.io/name: n8n + spec: + securityContext: + fsGroup: 1000 # matches the "node" user baked into the n8n image + containers: + - name: n8n + image: n8nio/n8n:latest + ports: + - name: http + containerPort: 5678 + envFrom: + - secretRef: + name: n8n-credentials + env: + - name: DB_TYPE + value: postgresdb + - name: DB_POSTGRESDB_HOST + value: postgres-postgresql.postgres.svc.cluster.local + - name: DB_POSTGRESDB_PORT + value: "5432" + - name: DB_POSTGRESDB_DATABASE + value: n8n + - name: N8N_HOST + value: n8n.sttlab.pc + - name: N8N_PORT + value: "5678" + - name: N8N_PROTOCOL + value: https + - name: N8N_WEBHOOK_URL + value: https://n8n.sttlab.pc/ + - name: GENERIC_TIMEZONE + value: Europe/Paris + - name: TZ + value: Europe/Paris + - name: N8N_RESTRICT_FILE_ACCESS_TO + value: "/data" # single path only (no list support) + volumeMounts: + - name: data + mountPath: /home/node/.n8n + - name: digest-workspace + mountPath: /data # dedicated volume for Read/Write File nodes, separate from n8n's own + # internal directory (which n8n always blocks file-node access to, regardless of + # N8N_RESTRICT_FILE_ACCESS_TO, via N8N_BLOCK_FILE_ACCESS_TO_N8N_FILES) + - name: digest-prompt + mountPath: /data/prompts/digest.md + subPath: digest.md # avoids the ConfigMap symlink indirection, which trips n8n's file-access allowlist check + readOnly: true + - name: digest-prompt + mountPath: /data/prompts/editorial.md + subPath: editorial.md + readOnly: true + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1000m + memory: 2Gi + readinessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 30 + periodSeconds: 20 + volumes: + - name: data + persistentVolumeClaim: + claimName: n8n-data + - name: digest-workspace + persistentVolumeClaim: + claimName: n8n-digest-workspace + - name: digest-prompt + configMap: + name: n8n-digest-prompt diff --git a/k8s/n8n/gateway.yaml b/k8s/n8n/gateway.yaml new file mode 100644 index 0000000..54719ac --- /dev/null +++ b/k8s/n8n/gateway.yaml @@ -0,0 +1,20 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: n8n + namespace: watch +spec: + gatewayClassName: envoy + listeners: + - name: https + protocol: HTTPS + port: 443 + hostname: n8n.sttlab.pc + allowedRoutes: + namespaces: + from: Same + tls: + mode: Terminate + certificateRefs: + - kind: Secret + name: sttlab-pc-tls diff --git a/k8s/n8n/httproute.yaml b/k8s/n8n/httproute.yaml new file mode 100644 index 0000000..7c8ffff --- /dev/null +++ b/k8s/n8n/httproute.yaml @@ -0,0 +1,21 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: n8n + namespace: watch +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: n8n + hostnames: + - n8n.sttlab.pc + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - kind: Service + name: n8n + port: 80 diff --git a/k8s/n8n/miniflux-token-secret.example.yaml b/k8s/n8n/miniflux-token-secret.example.yaml new file mode 100644 index 0000000..48671f6 --- /dev/null +++ b/k8s/n8n/miniflux-token-secret.example.yaml @@ -0,0 +1,20 @@ +# Copy to miniflux-token-secret.yaml, fill in the real token, and apply with: +# kubectl apply -f miniflux-token-secret.yaml +# Never commit the filled-in file to git. +# +# Generate the token in the Miniflux UI under Settings -> API Keys (no +# public REST endpoint creates one). Kept here as the source of truth / +# backup; the digest workflow itself should use an n8n "Header Auth" +# credential (header name X-Auth-Token) created in the n8n UI from these +# same values, not a raw env var — n8n blocks $env access from workflow +# expressions by default (N8N_BLOCK_ENV_ACCESS_IN_NODE), so this Secret is +# not wired into the n8n Deployment's envFrom. +apiVersion: v1 +kind: Secret +metadata: + name: miniflux-api-token + namespace: watch +type: Opaque +stringData: + MINIFLUX_URL: https://miniflux.sttlab.pc/v1/ + MINIFLUX_API_TOKEN: changeme diff --git a/k8s/n8n/pvc.yaml b/k8s/n8n/pvc.yaml new file mode 100644 index 0000000..8e7ef2a --- /dev/null +++ b/k8s/n8n/pvc.yaml @@ -0,0 +1,26 @@ +# Persists /home/node/.n8n (encryption key backup, local binary data cache, +# community nodes). Workflow/execution data itself lives in Postgres. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: n8n-data + namespace: watch +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 2Gi +--- +# Separate volume for Read/Write File nodes (digest prompt + output), kept +# apart from n8n-data (n8n's own internal directory, which n8n always +# blocks file-node access to regardless of N8N_RESTRICT_FILE_ACCESS_TO). +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: n8n-digest-workspace + namespace: watch +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi diff --git a/k8s/n8n/secret.example.yaml b/k8s/n8n/secret.example.yaml new file mode 100644 index 0000000..8878450 --- /dev/null +++ b/k8s/n8n/secret.example.yaml @@ -0,0 +1,19 @@ +# Copy to secret.yaml, fill in real values, and apply with: +# kubectl apply -f secret.yaml +# Never commit the filled-in secret.yaml to git. +# +# DB_POSTGRESDB_USER/PASSWORD must match k8s/postgres/secret.yaml. +# +# N8N_ENCRYPTION_KEY encrypts credentials stored in the n8n database. +# Generate once with `openssl rand -hex 24` and never change it afterwards +# (rotating it locks n8n out of every credential already saved). +apiVersion: v1 +kind: Secret +metadata: + name: n8n-credentials + namespace: watch +type: Opaque +stringData: + DB_POSTGRESDB_USER: postgres + DB_POSTGRESDB_PASSWORD: changeme + N8N_ENCRYPTION_KEY: changeme diff --git a/k8s/n8n/service.yaml b/k8s/n8n/service.yaml new file mode 100644 index 0000000..9aa70cf --- /dev/null +++ b/k8s/n8n/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: n8n + namespace: watch + labels: + app.kubernetes.io/name: n8n +spec: + selector: + app.kubernetes.io/name: n8n + ports: + - name: http + port: 80 + targetPort: 5678 diff --git a/k8s/namespace.yaml b/k8s/namespace.yaml new file mode 100644 index 0000000..262a4d6 --- /dev/null +++ b/k8s/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: watch diff --git a/k8s/postgres/namespace.yaml b/k8s/postgres/namespace.yaml new file mode 100644 index 0000000..427794c --- /dev/null +++ b/k8s/postgres/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: postgres diff --git a/k8s/postgres/secret.example.yaml b/k8s/postgres/secret.example.yaml new file mode 100644 index 0000000..3fab9f5 --- /dev/null +++ b/k8s/postgres/secret.example.yaml @@ -0,0 +1,16 @@ +# Copy to secret.yaml, fill in a real password, and apply with: +# kubectl apply -f secret.yaml +# Never commit the filled-in secret.yaml to git. +# +# This is the Postgres superuser used to provision one database per +# application (see create-db-*-job.yaml). Applications connect with these +# same credentials against their own database (no per-app roles for now). +apiVersion: v1 +kind: Secret +metadata: + name: postgres-admin + namespace: postgres +type: Opaque +stringData: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: changeme diff --git a/k8s/postgres/service.yaml b/k8s/postgres/service.yaml new file mode 100644 index 0000000..ecdf000 --- /dev/null +++ b/k8s/postgres/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: postgres + labels: + app.kubernetes.io/name: postgres +spec: + clusterIP: None + selector: + app.kubernetes.io/name: postgres + ports: + - name: postgres + port: 5432 + targetPort: 5432 diff --git a/k8s/postgres/statefulset.yaml b/k8s/postgres/statefulset.yaml new file mode 100644 index 0000000..2fd84ad --- /dev/null +++ b/k8s/postgres/statefulset.yaml @@ -0,0 +1,61 @@ +# Shared PostgreSQL instance, reusable across applications (miniflux, n8n, ...). +# Each application gets its own database (see create-db-*-job.yaml), all +# reachable through the "postgres" Service below. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres + namespace: postgres + labels: + app.kubernetes.io/name: postgres +spec: + serviceName: postgres + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: postgres + template: + metadata: + labels: + app.kubernetes.io/name: postgres + spec: + containers: + - name: postgres + image: postgres:18-alpine + ports: + - name: postgres + containerPort: 5432 + envFrom: + - secretRef: + name: postgres-admin + env: + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + readinessProbe: + exec: + command: ["pg_isready", "-U", "$(POSTGRES_USER)"] + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + exec: + command: ["pg_isready", "-U", "$(POSTGRES_USER)"] + initialDelaySeconds: 15 + periodSeconds: 20 + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 5Gi diff --git a/miniflux-initializer/Dockerfile b/miniflux-initializer/Dockerfile new file mode 100644 index 0000000..9cabad9 --- /dev/null +++ b/miniflux-initializer/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.14-alpine + +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY bootstrap_feeds.py feeds.yaml ./ + +ENTRYPOINT ["python3", "bootstrap_feeds.py"] diff --git a/miniflux-initializer/bootstrap_feeds.py b/miniflux-initializer/bootstrap_feeds.py new file mode 100644 index 0000000..fa15894 --- /dev/null +++ b/miniflux-initializer/bootstrap_feeds.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Create Miniflux categories and feeds from feeds.yaml. + +Env vars required: MINIFLUX_URL, MINIFLUX_USERNAME, MINIFLUX_PASSWORD. + +Idempotent: existing categories are matched by title, existing feeds are +matched by feed_url, both fetched from the API before any create call. +Safe to re-run (e.g. every time the Job runs) — only new entries get +created. +""" +import os +import sys + +import requests +import yaml + +MINIFLUX_URL = os.environ["MINIFLUX_URL"].rstrip("/") +AUTH = (os.environ["MINIFLUX_USERNAME"], os.environ["MINIFLUX_PASSWORD"]) +FEEDS_FILE = os.path.join(os.path.dirname(__file__), "feeds.yaml") + + +def api(method, path, **kwargs): + resp = requests.request(method, f"{MINIFLUX_URL}/v1{path}", auth=AUTH, timeout=60, **kwargs) + resp.raise_for_status() + return resp.json() if resp.content else None + + +def get_or_create_category(name, existing_categories): + if name in existing_categories: + return existing_categories[name] + category = api("POST", "/categories", json={"title": name}) + existing_categories[name] = category["id"] + print(f"created category: {name}") + return category["id"] + + +def main(): + with open(FEEDS_FILE) as f: + data = yaml.safe_load(f) + + existing_categories = {c["title"]: c["id"] for c in api("GET", "/categories")} + existing_feed_urls = {f["feed_url"] for f in api("GET", "/feeds")} + + for category in data["categories"]: + category_id = get_or_create_category(category["name"], existing_categories) + for feed in category["feeds"]: + if feed["url"] in existing_feed_urls: + print(f"skip (already added): {feed['title']}") + continue + try: + api( + "POST", + "/feeds", + json={ + "feed_url": feed["url"], + "category_id": category_id, + "crawler": feed.get("crawler", False), + }, + ) + existing_feed_urls.add(feed["url"]) + print(f"added feed: {feed['title']} ({feed['url']})") + except requests.RequestException as exc: + print(f"FAILED: {feed['title']} ({feed['url']}): {exc}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/miniflux-initializer/feeds.yaml b/miniflux-initializer/feeds.yaml new file mode 100644 index 0000000..33aacdf --- /dev/null +++ b/miniflux-initializer/feeds.yaml @@ -0,0 +1,179 @@ +# Initial feed list. Kept intentionally small: one or two +# confirmed-working (HTTP 200) feeds per category from AGENTS.md. +# Add more later with the same structure — see README.md "How to add a source". +# crawler: true makes Miniflux fetch the original article page instead of +# relying on the (often truncated) RSS/Atom excerpt. +categories: + - name: Cloud Native + feeds: + - title: CNCF blog + url: https://www.cncf.io/feed/ + crawler: true + - title: Kubernetes blog + url: https://kubernetes.io/feed.xml + crawler: true + - title: AWS Architecture blog + url: https://aws.amazon.com/blogs/architecture/feed/ + crawler: true + - title: Docker blog + url: https://www.docker.com/feed/ + crawler: true + - title: HashiCorp blog (news) + url: https://news.google.com/rss/search?q=%22HashiCorp%22%20blog&hl=en-US&gl=US&ceid=US:en + crawler: true + - title: Red Hat blog (news) + url: https://news.google.com/rss/search?q=%22Red%20Hat%22%20blog&hl=en-US&gl=US&ceid=US:en + crawler: true + - title: Google Cloud blog + url: https://cloudblog.withgoogle.com/rss/ + crawler: true + - title: Azure blog + url: https://azure.microsoft.com/en-us/blog/feed/ + crawler: true + + - name: Integration + feeds: + - title: Google News search + url: https://news.google.com/rss/search?q=%22hybrid%20integration%22%20OR%20iPaaS%20OR%20%22integration%20platform%22%20OR%20%22enterprise%20service%20bus%22%20OR%20%22application%20integration%22%20OR%20webMethods%20OR%20%22IBM%20App%20Connect%22%20OR%20Boomi%20OR%20TIBCO%20OR%20MuleSoft%20OR%20Workato&hl=en-US&gl=US&ceid=US:en + crawler: true + + - name: APIM + feeds: + - title: Google News search + url: https://news.google.com/rss/search?q=%22API%20management%22%20OR%20%22API%20gateway%22%20OR%20%22API%20economy%22%20OR%20OpenAPI%20OR%20AsyncAPI%20OR%20%22API%20security%22%20OR%20Apigee%20OR%20%22Kong%20Gateway%22%20OR%20%22Azure%20API%20Management%22%20OR%20%22Gravitee.io%22%20OR%20%22Tyk%20API%22%20OR%20WSO2%20OR%20%22Amazon%20API%20Gateway%22%20OR%20%22IBM%20API%20Connect%22%20OR%20%22Axway%20Amplify%22%20OR%20%22Postman%20API%22&hl=en-US&gl=US&ceid=US:en + crawler: true + + - name: EDA + feeds: + - title: Confluent blog + url: https://www.confluent.io/feed/ + crawler: true + - title: Kai Waehner + url: https://www.kai-waehner.de/feed/ + crawler: true + - title: Solace blog + url: https://solace.com/feed/ + crawler: true + - title: RabbitMQ blog + url: https://www.rabbitmq.com/blog/rss.xml # verify + crawler: true + - title: AsyncAPI blog + url: https://www.asyncapi.com/rss.xml # verify + crawler: true + + - name: AI + feeds: + - title: OpenAI blog + url: https://openai.com/blog/rss.xml + crawler: false + - title: Simon Willison + url: https://simonwillison.net/atom/everything/ + crawler: true + - title: Anthropic news + url: https://rsshub.bestblogs.dev/anthropic/news + crawler: true + - title: Huggingface blog + url: https://huggingface.co/blog/feed.xml + crawler: true + - title: Techcrunch AI + url: https://techcrunch.com/category/artificial-intelligence/feed/ + crawler: true + - title: Last Week in AI + url: https://lastweekin.ai/feed/ + crawler: true + - title: The Batch (Andrew Ng) (news) + url: https://news.google.com/rss/search?q=%22The%20Batch%22%20%22DeepLearning.AI%22&hl=en-US&gl=US&ceid=US:en + crawler: true + - title: Microsoft AI blog (news) + url: https://news.google.com/rss/search?q=%22Microsoft%20AI%22%20blog&hl=en-US&gl=US&ceid=US:en + crawler: true + - title: LangChain blog (news) + url: https://news.google.com/rss/search?q=%22LangChain%22%20blog&hl=en-US&gl=US&ceid=US:en + crawler: true + - title: Latent Space + url: https://www.latent.space/feed + crawler: false # Substack, contenu complet dans le flux + - title: MCP releases + url: https://github.com/modelcontextprotocol/modelcontextprotocol/releases.atom + crawler: false + - title: ARC Prize blog (news) + url: https://news.google.com/rss/search?q=%22ARC%20Prize%22&hl=en-US&gl=US&ceid=US:en + crawler: true + - title: Baldur Bjarnason + url: https://www.baldurbjarnason.com/feed.xml + crawler: false # contenu complet dans le flux + - title: Yann LeCun (news) + url: https://news.google.com/rss/search?q=%22Yann%20LeCun%22&hl=en-US&gl=US&ceid=US:en + crawler: true + - title: Francois Chollet (news) + url: https://news.google.com/rss/search?q=%22Fran%C3%A7ois%20Chollet%22%20OR%20%22Francois%20Chollet%22&hl=en-US&gl=US&ceid=US:en + crawler: true + + - name: Security + feeds: + - title: Bleeping Computer + url: https://www.bleepingcomputer.com/feed/ + crawler: true + - title: CISA advisories + url: https://www.cisa.gov/cybersecurity-advisories/all.xml + crawler: false + - title: Hacker News + url: https://news.ycombinator.com/rss + crawler: false + + - name: Sovereignty + feeds: + - title: Digital sovereignty (news) + url: https://news.google.com/rss/search?q=%22digital%20sovereignty%22%20OR%20%22sovereign%20cloud%22%20OR%20%22cloud%20souverain%22&hl=en-US&gl=US&ceid=US:en + crawler: true + - title: EU regulation - AI Act / Data Act / GAIA-X (news) + url: https://news.google.com/rss/search?q=(%22AI%20Act%22%20OR%20%22Data%20Act%22%20OR%20GAIA-X%20OR%20EuroHPC)%20(Europe%20OR%20EU)&hl=en-US&gl=US&ceid=US:en + crawler: true + - title: Souverainete numerique FR (news) + url: https://news.google.com/rss/search?q=(souverainet%C3%A9%20num%C3%A9rique%20OR%20%22cloud%20de%20confiance%22%20OR%20SecNumCloud)&hl=fr&gl=FR&ceid=FR:fr + crawler: true + + - name: Analysts + feeds: + - title: Sanjeev Mohan + url: https://news.google.com/rss/search?q=%22Sanjeev%20Mohan%22&hl=en-US&gl=US&ceid=US:en + crawler: true + - title: Massimo Pezzini + url: https://news.google.com/rss/search?q=%22Massimo%20Pezzini%22&hl=en-US&gl=US&ceid=US:en + crawler: true + - title: Gregor Hohpe + url: https://news.google.com/rss/search?q=%22Gregor%20Hohpe%22&hl=en-US&gl=US&ceid=US:en + crawler: true + - title: Kai Waehner + url: https://news.google.com/rss/search?q=%22Kai%20Waehner%22&hl=en-US&gl=US&ceid=US:en + crawler: true + - title: Martin Fowler + url: https://martinfowler.com/feed.atom + crawler: true + + - name: Others + feeds: + - title: The Register + url: https://www.theregister.com/headlines.atom + crawler: true + - title: InfoQ + url: https://feed.infoq.com/ + crawler: true + - title: MIT Technology Review + url: https://www.technologyreview.com/feed/ + crawler: true + - title: Open Source Projects + url: https://www.opensourceprojects.dev/rss + crawler: true + - title: Reddit /r/programming + url: https://www.reddit.com/r/programming/.rss + crawler: false + - title: Reddit /r/technology + url: https://www.reddit.com/r/technology/.rss + crawler: false + - title: GitHub blog + url: https://github.blog/feed/ + crawler: true + - title: Wired + url: https://www.wired.com/feed/rss + crawler: true diff --git a/miniflux-initializer/requirements.txt b/miniflux-initializer/requirements.txt new file mode 100644 index 0000000..33f05f7 --- /dev/null +++ b/miniflux-initializer/requirements.txt @@ -0,0 +1,2 @@ +requests +PyYAML diff --git a/prompts/digest.md b/prompts/digest.md new file mode 100644 index 0000000..c36c7a1 --- /dev/null +++ b/prompts/digest.md @@ -0,0 +1,55 @@ +LANGUAGE RULE, READ FIRST: every piece of natural-language text you write in +your response — every "summary", every "weak_signals" entry — MUST be in +French. The source articles you're given are in English; you read them in +English, but you WRITE ONLY IN FRENCH. Do not let the input language leak +into your output. Do not write a single English sentence, clause, or +dangling English phrase inside a French sentence. If you catch yourself +drafting in English, translate it before responding. This rule applies no +matter what — there is no case where an English summary is acceptable. + +You are curating a DAILY technology-watch digest for a solutions architect +working on application/data integration (iPaaS, ESB, API management), +event-driven architecture (Kafka, AMQP, MQTT, AsyncAPI, CloudEvents), +agentic AI / LLM integration / MCP, cloud native / Kubernetes / platform +engineering, and MFT / B2B / EDI integration. + +You will be given one topic category and a list of unread articles in that +category (title, URL, and excerpt/content), covering roughly the last 24 +hours. Your job: + +1. Select every item that is genuinely relevant and significant for this + reader. Do not artificially cap the count to a fixed number — a busy + day can have many, a quiet day can have few or none. Relevance means: + concrete technical substance, a notable release, security advisory, + architecture pattern, or market-moving event (funding, acquisition, + major partnership) in the domains above — not generic marketing posts. +2. Write a 1-2 sentence summary of each selected item. **The summary text + must be written in French — always, even though the source articles + and this prompt are in English.** Dense and factual (no fluff, no + "cet article explique que..."). +3. List any remaining items that are minor but still worth a passing + mention as "weak signals". Each weak signal MUST correspond to one + specific article from the input that you did NOT already put in + top_items — never restate, generalize, or editorialize about an item + that is already in top_items. Each one is a short standalone phrase **in + French**, a few words only (not a sentence, no elaboration, just enough + to identify the topic). Skip items that are pure noise (ads, unrelated + content). + +Respond with **strict JSON only**, no markdown fences, no commentary, +matching exactly this shape: + +{ + "top_items": [ + {"title": "...", "url": "...", "summary": "..."} + ], + "weak_signals": ["...", "..."] +} + +Every "summary" and every "weak_signals" entry must be in French — this is +required, not a suggestion, and it is the single most common mistake to +avoid: do not respond in English. "title" and "url" are copied verbatim from +the input, unchanged (these two fields only stay in their original +language). If the category has no article worth surfacing at all, return +empty arrays for both fields. Never invent articles or URLs that were not +given to you. diff --git a/prompts/editorial.md b/prompts/editorial.md new file mode 100644 index 0000000..da6a03f --- /dev/null +++ b/prompts/editorial.md @@ -0,0 +1,58 @@ +LANGUAGE RULE, READ FIRST: every piece of natural-language text you write — +"lead" and every "highlights" entry — MUST be in French. The items you're +given are already French summaries, but if you draft anything in English, +translate it before responding. No English words, clauses, or phrases +anywhere in your output. + +You are the editorial voice for a DAILY technology-watch digest, running +after a mechanical selection pass has already picked the day's articles per +category (application/data integration, event-driven architecture, agentic +AI/LLM/MCP, cloud native/Kubernetes, security, digital sovereignty, industry +analysts, and others) for a solutions architect reader. + +You will be given a JSON array of categories, each with a list of items +already selected for today (title and summary, in French). This selection is +final — you are not editing it, only writing about it. + +Your job: write a short **editorial synthesis** of the day, to be placed at +the top of the digest before the category-by-category lists. This is not a +recap or a list of the items — the reader will see the full lists right +below. Instead: + +1. Identify what actually matters most today across the whole set — the + 1-3 stories or developments most worth the reader's attention, and why. +2. Connect the dots where relevant: if multiple items across categories + relate to the same underlying trend, story, or tension (e.g. the same + event covered from different angles, or a pattern emerging across + several independent items), say so explicitly — this is exactly the kind + of cross-category perspective the mechanical per-category pass cannot + produce. +3. Give real editorial judgment: what's significant vs. noise, what's likely + to matter in the coming weeks, what a reader in this domain (iPaaS/ESB, + EDA, agentic AI, cloud native, MFT/EDI) should actually take away. Light, + opinionated framing is welcome — this is analysis, not a summary. +4. Stay factual and grounded strictly in the items given to you. Do not + invent facts, events, or items not present in the input. + +**Format matters as much as content.** Do NOT write one long dense +paragraph — it is hard to scan and gets skipped. Instead produce: + +- "lead": ONE short sentence (max ~25 words) stating the single most + important takeaway of the day. This is the headline of your analysis. +- "highlights": 2 to 5 short bullet points (each ONE sentence, ideally + under 25 words), each surfacing one distinct connection, tension, or + noteworthy pattern across the day's items. Each bullet stands on its own + — a reader should be able to read just the bullets and get the gist. + Do not pad to reach 5; 2-3 sharp bullets beat 5 mediocre ones. + +If the day's selection is genuinely thin or scattered with no meaningful +throughline, it's fine to say so briefly in "lead" and keep "highlights" +short or empty rather than forcing false connections. + +Respond with **strict JSON only**, no markdown fences, no commentary, +matching exactly this shape: + +{ + "lead": "...", + "highlights": ["...", "..."] +} diff --git a/workflows/digest.json b/workflows/digest.json new file mode 100644 index 0000000..1555ed4 --- /dev/null +++ b/workflows/digest.json @@ -0,0 +1,306 @@ +{ + "id": "CAbGgPd187VA7K1R", + "name": "Tech Watch Digest", + "nodes": [ + { + "id": "schedule_trigger", + "name": "Schedule Trigger", + "type": "n8n-nodes-base.scheduleTrigger", + "typeVersion": 1.2, + "position": [0, 0], + "parameters": { + "rule": { + "interval": [ + { "field": "cronExpression", "expression": "0 6 * * *" } + ] + } + } + }, + { + "id": "init", + "name": "Init", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [220, 0], + "parameters": { + "mode": "runOnceForAllItems", + "jsCode": "const now = new Date();\nconst oneDayAgo = new Date(now.getTime() - 24 * 60 * 60 * 1000);\nreturn [{\n json: {\n publishedAfter: Math.floor(oneDayAgo.getTime() / 1000),\n digestDate: now.toISOString().slice(0, 10),\n llmBaseUrl: 'https://openrouter.ai/api/v1',\n llmModel: 'deepseek/deepseek-v4-flash',\n editorialModel: 'anthropic/claude-opus-4.8'\n }\n}];" + } + }, + { + "id": "get_unread_entries", + "name": "Get Unread Entries", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [440, 0], + "parameters": { + "method": "GET", + "url": "http://miniflux.watch.svc.cluster.local/v1/entries", + "authentication": "genericCredentialType", + "genericAuthType": "httpHeaderAuth", + "sendQuery": true, + "queryParameters": { + "parameters": [ + { "name": "status", "value": "unread" }, + { "name": "limit", "value": "500" }, + { "name": "order", "value": "published_at" }, + { "name": "direction", "value": "desc" }, + { "name": "published_after", "value": "={{ $json.publishedAfter }}" } + ] + }, + "options": {} + }, + "credentials": { + "httpHeaderAuth": { "id": "D3czTdAQl3KhiKTM", "name": "Miniflux API" } + }, + "retryOnFail": true, + "maxTries": 3, + "waitBetweenTries": 2000 + }, + { + "id": "group_by_category", + "name": "Group By Category", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [660, 0], + "parameters": { + "mode": "runOnceForAllItems", + "jsCode": "const resp = $input.first().json;\nlet entries = resp.entries || [];\n\n// Dedup across ALL categories by normalized title: several feeds (direct\n// blogs + overlapping Google News searches) can surface the same story\n// under different URLs/categories, and each category is summarized by the\n// LLM independently so nothing else catches this.\nconst seenTitles = new Set();\nentries = entries.filter((e) => {\n const key = (e.title || '').toLowerCase().trim().replace(/[^a-z0-9]+/g, ' ').trim();\n if (!key || seenTitles.has(key)) return false;\n seenTitles.add(key);\n return true;\n});\n\nconst byCategory = {};\nfor (const e of entries) {\n const cat = (e.feed && e.feed.category && e.feed.category.title) || 'Uncategorized';\n if (!byCategory[cat]) byCategory[cat] = [];\n byCategory[cat].push({\n id: e.id,\n title: e.title,\n url: e.url,\n published_at: e.published_at,\n feed_title: e.feed ? e.feed.title : '',\n excerpt: (e.content || '').replace(/<[^>]+>/g, ' ').replace(/\\s+/g, ' ').trim().slice(0, 600)\n });\n}\nconst items = Object.keys(byCategory).map((cat) => ({\n json: { category: cat, entries: byCategory[cat] }\n}));\nif (items.length === 0) {\n items.push({ json: { category: null, entries: [] } });\n}\nreturn items;" + } + }, + { + "id": "read_digest_prompt", + "name": "Read Digest Prompt", + "type": "n8n-nodes-base.readWriteFile", + "typeVersion": 1, + "position": [880, 0], + "parameters": { + "operation": "read", + "fileSelector": "/data/prompts/digest.md", + "options": {} + } + }, + { + "id": "build_prompt", + "name": "Build Prompt", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [1100, 0], + "parameters": { + "mode": "runOnceForEachItem", + "jsCode": "const systemPrompt = Buffer.from($input.item.binary.data.data, 'base64').toString('utf-8');\nconst category = $('Group By Category').item.json.category;\nconst entries = $('Group By Category').item.json.entries;\nconst init = $('Init').first().json;\nconst schemaReminder = 'Respond with strict JSON only, exactly this shape and no other keys: {\"top_items\": [{\"title\": \"...\", \"url\": \"...\", \"summary\": \"...\"}], \"weak_signals\": [\"...\"]}. No markdown code fences, no extra commentary. Reminder: every \"summary\" and every \"weak_signals\" entry must be written in French, even though the articles below are in English.';\nconst userContent = `Category: ${category}\\n\\nArticles:\\n` + entries.map((e, i) => `${i + 1}. ${e.title}\\nURL: ${e.url}\\nExcerpt: ${e.excerpt}`).join('\\n\\n') + `\\n\\n${schemaReminder}`;\nreturn { json: { category, entries, systemPrompt, userContent, llmBaseUrl: init.llmBaseUrl, llmModel: init.llmModel } };" + } + }, + { + "id": "summarize_category", + "name": "Summarize Category", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [1320, 0], + "parameters": { + "method": "POST", + "url": "={{ $json.llmBaseUrl }}/chat/completions", + "authentication": "genericCredentialType", + "genericAuthType": "httpHeaderAuth", + "sendHeaders": true, + "headerParameters": { + "parameters": [{ "name": "Content-Type", "value": "application/json" }] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{ JSON.stringify({ model: $json.llmModel, messages: [{ role: 'system', content: $json.systemPrompt }, { role: 'user', content: $json.userContent }], response_format: { type: 'json_object' } }) }}", + "options": {} + }, + "credentials": { + "httpHeaderAuth": { "id": "pZ4g3ReSwgDA6WLP", "name": "LLM API" } + }, + "retryOnFail": true, + "maxTries": 3, + "waitBetweenTries": 2000 + }, + { + "id": "parse_llm_response", + "name": "Parse LLM Response", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [1540, 0], + "parameters": { + "mode": "runOnceForEachItem", + "jsCode": "const category = $('Group By Category').item.json.category;\nlet parsed = null;\ntry {\n const content = $json.choices && $json.choices[0] && $json.choices[0].message && $json.choices[0].message.content;\n parsed = content ? JSON.parse(content) : null;\n} catch (err) {\n parsed = null;\n}\nconst top_items = (parsed && Array.isArray(parsed.top_items)) ? parsed.top_items : [];\nconst weak_signals = (parsed && Array.isArray(parsed.weak_signals)) ? parsed.weak_signals : [];\nreturn { json: { category, top_items, weak_signals } };" + } + }, + { + "id": "combine_for_editorial", + "name": "Combine For Editorial", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [1760, 0], + "parameters": { + "mode": "runOnceForAllItems", + "jsCode": "const CATEGORY_ORDER = ['Cloud Native', 'Integration', 'APIM', 'EDA', 'AI', 'Security', 'Sovereignty', 'Analysts', 'Others'];\nconst items = $input.all().map((i) => i.json);\nitems.sort((a, b) => {\n const ia = CATEGORY_ORDER.indexOf(a.category);\n const ib = CATEGORY_ORDER.indexOf(b.category);\n return (ia === -1 ? CATEGORY_ORDER.length : ia) - (ib === -1 ? CATEGORY_ORDER.length : ib);\n});\nreturn [{ json: { categories: items } }];" + } + }, + { + "id": "read_editorial_prompt", + "name": "Read Editorial Prompt", + "type": "n8n-nodes-base.readWriteFile", + "typeVersion": 1, + "position": [1980, 0], + "parameters": { + "operation": "read", + "fileSelector": "/data/prompts/editorial.md", + "options": {} + } + }, + { + "id": "build_editorial_prompt", + "name": "Build Editorial Prompt", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [2200, 0], + "parameters": { + "mode": "runOnceForEachItem", + "jsCode": "const systemPrompt = Buffer.from($input.item.binary.data.data, 'base64').toString('utf-8');\nconst categories = $('Combine For Editorial').first().json.categories;\nconst slim = categories.map((c) => ({ category: c.category, items: c.top_items.map((it) => ({ title: it.title, summary: it.summary })) })).filter((c) => c.items.length > 0);\nconst init = $('Init').first().json;\nconst schemaReminder = 'Respond with strict JSON only, exactly this shape and no other keys: {\"lead\": \"...\", \"highlights\": [\"...\"]}. No markdown code fences, no extra commentary. Reminder: \"lead\" and every \"highlights\" entry must be written in French. \"lead\" is one short sentence; \"highlights\" is 2 to 5 short one-sentence bullets, not one long paragraph.';\nconst userContent = JSON.stringify(slim) + '\\n\\n' + schemaReminder;\nreturn { json: { systemPrompt, userContent, llmBaseUrl: init.llmBaseUrl, llmModel: init.editorialModel } };" + } + }, + { + "id": "editorial_pass", + "name": "Editorial Pass (Opus)", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [2420, 0], + "parameters": { + "method": "POST", + "url": "={{ $json.llmBaseUrl }}/chat/completions", + "authentication": "genericCredentialType", + "genericAuthType": "httpHeaderAuth", + "sendHeaders": true, + "headerParameters": { + "parameters": [{ "name": "Content-Type", "value": "application/json" }] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{ JSON.stringify({ model: $json.llmModel, messages: [{ role: 'system', content: $json.systemPrompt }, { role: 'user', content: $json.userContent }], response_format: { type: 'json_object' } }) }}", + "options": {} + }, + "credentials": { + "httpHeaderAuth": { "id": "pZ4g3ReSwgDA6WLP", "name": "LLM API" } + }, + "retryOnFail": true, + "maxTries": 3, + "waitBetweenTries": 2000 + }, + { + "id": "parse_editorial_response", + "name": "Parse Editorial Response", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [2640, 0], + "parameters": { + "mode": "runOnceForAllItems", + "jsCode": "let parsed = null;\ntry {\n const content = $json.choices && $json.choices[0] && $json.choices[0].message && $json.choices[0].message.content;\n parsed = content ? JSON.parse(content) : null;\n} catch (err) {\n parsed = null;\n}\nconst lead = (parsed && typeof parsed.lead === 'string') ? parsed.lead.trim() : '';\nconst highlights = (parsed && Array.isArray(parsed.highlights)) ? parsed.highlights.filter((h) => typeof h === 'string' && h.trim()) : [];\nconst categories = $('Combine For Editorial').first().json.categories;\nreturn [{ json: { categories, lead, highlights } }];" + } + }, + { + "id": "compose_digest", + "name": "Compose Digest", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [2860, 0], + "parameters": { + "mode": "runOnceForAllItems", + "jsCode": "const esc = (s) => String(s == null ? '' : s).replace(/&/g, '&').replace(//g, '>').replace(/\"/g, '"');\nconst cats = $input.first().json.categories;\nconst lead = $input.first().json.lead || '';\nconst highlights = $input.first().json.highlights || [];\nconst date = $('Init').first().json.digestDate;\nconst header = `# Veille technique — ${date}\\n\\n`;\nlet md = header;\nlet htmlSections = '';\nif (lead || highlights.length) {\n md += `## Analyse éditoriale\\n\\n`;\n htmlSections += `

Analyse éditoriale

`;\n if (lead) {\n md += `${lead}\\n\\n`;\n htmlSections += `

${esc(lead)}

`;\n }\n if (highlights.length) {\n md += highlights.map((h) => `- ${h}`).join('\\n') + `\\n\\n`;\n htmlSections += `
    ` + highlights.map((h) => `
  • ${esc(h)}
  • `).join('') + `
`;\n }\n htmlSections += `
`;\n}\nlet hasContent = false;\nfor (const cat of cats) {\n if (!cat.top_items.length && !cat.weak_signals.length) continue;\n hasContent = true;\n md += `## ${cat.category}\\n\\n`;\n htmlSections += `

${esc(cat.category)}

    `;\n for (const it of cat.top_items) {\n md += `- **[${it.title}](${it.url})** — ${it.summary}\\n`;\n htmlSections += `
  • ${esc(it.title)} — ${esc(it.summary)}
  • `;\n }\n htmlSections += `
`;\n if (cat.weak_signals.length) {\n md += `\\n_Signaux faibles :_\\n` + cat.weak_signals.map((s) => `- ${s}`).join('\\n') + `\\n`;\n htmlSections += `

Signaux faibles

    ` + cat.weak_signals.map((s) => `
  • ${esc(s)}
  • `).join('') + `
`;\n }\n md += `\\n`;\n htmlSections += `
`;\n}\nif (!hasContent) {\n md += `_Aucun article pertinent aujourd'hui._\\n`;\n htmlSections += `

Aucun article pertinent aujourd'hui.

`;\n}\nconst html = `\n\nVeille technique — ${esc(date)}\n\n\n\n

Veille technique — ${esc(date)}

\n${htmlSections}\n`;\nconst fileNameMd = `digest-${date}.md`;\nconst fileNameHtml = `digest-${date}.html`;\nconst mdBinary = await this.helpers.prepareBinaryData(Buffer.from(md, 'utf-8'), fileNameMd, 'text/markdown');\nconst htmlBinary = await this.helpers.prepareBinaryData(Buffer.from(html, 'utf-8'), fileNameHtml, 'text/html');\nreturn [{ json: { markdown: md, fileNameMd, fileNameHtml }, binary: { md: mdBinary, html: htmlBinary } }];" + } + }, + { + "id": "write_digest_file", + "name": "Write Digest File (Markdown)", + "type": "n8n-nodes-base.readWriteFile", + "typeVersion": 1, + "position": [3080, -160], + "parameters": { + "operation": "write", + "fileName": "=/data/digests/{{ $json.fileNameMd }}", + "dataPropertyName": "md", + "options": {} + } + }, + { + "id": "write_digest_file_html", + "name": "Write Digest File (HTML)", + "type": "n8n-nodes-base.readWriteFile", + "typeVersion": 1, + "position": [3080, -40], + "parameters": { + "operation": "write", + "fileName": "=/data/digests/{{ $json.fileNameHtml }}", + "dataPropertyName": "html", + "options": {} + } + }, + { + "id": "collect_entry_ids", + "name": "Collect Entry IDs", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [3300, -100], + "parameters": { + "mode": "runOnceForAllItems", + "jsCode": "const entries = $('Get Unread Entries').first().json.entries || [];\nconst entryIds = entries.map((e) => e.id);\nif (entryIds.length === 0) return [];\nreturn [{ json: { entryIds } }];" + } + }, + { + "id": "mark_entries_as_read", + "name": "Mark Entries As Read", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [3520, -100], + "parameters": { + "method": "PUT", + "url": "http://miniflux.watch.svc.cluster.local/v1/entries", + "authentication": "genericCredentialType", + "genericAuthType": "httpHeaderAuth", + "sendHeaders": true, + "headerParameters": { + "parameters": [{ "name": "Content-Type", "value": "application/json" }] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{ JSON.stringify({ entry_ids: $json.entryIds, status: 'read' }) }}", + "options": {} + }, + "credentials": { + "httpHeaderAuth": { "id": "D3czTdAQl3KhiKTM", "name": "Miniflux API" } + }, + "retryOnFail": true, + "maxTries": 3, + "waitBetweenTries": 2000 + } + ], + "connections": { + "Schedule Trigger": { "main": [[{ "node": "Init", "type": "main", "index": 0 }]] }, + "Init": { "main": [[{ "node": "Get Unread Entries", "type": "main", "index": 0 }]] }, + "Get Unread Entries": { "main": [[{ "node": "Group By Category", "type": "main", "index": 0 }]] }, + "Group By Category": { "main": [[{ "node": "Read Digest Prompt", "type": "main", "index": 0 }]] }, + "Read Digest Prompt": { "main": [[{ "node": "Build Prompt", "type": "main", "index": 0 }]] }, + "Build Prompt": { "main": [[{ "node": "Summarize Category", "type": "main", "index": 0 }]] }, + "Summarize Category": { "main": [[{ "node": "Parse LLM Response", "type": "main", "index": 0 }]] }, + "Parse LLM Response": { "main": [[{ "node": "Combine For Editorial", "type": "main", "index": 0 }]] }, + "Combine For Editorial": { "main": [[{ "node": "Read Editorial Prompt", "type": "main", "index": 0 }]] }, + "Read Editorial Prompt": { "main": [[{ "node": "Build Editorial Prompt", "type": "main", "index": 0 }]] }, + "Build Editorial Prompt": { "main": [[{ "node": "Editorial Pass (Opus)", "type": "main", "index": 0 }]] }, + "Editorial Pass (Opus)": { "main": [[{ "node": "Parse Editorial Response", "type": "main", "index": 0 }]] }, + "Parse Editorial Response": { "main": [[{ "node": "Compose Digest", "type": "main", "index": 0 }]] }, + "Compose Digest": { + "main": [ + [ + { "node": "Write Digest File (Markdown)", "type": "main", "index": 0 }, + { "node": "Write Digest File (HTML)", "type": "main", "index": 0 } + ] + ] + }, + "Write Digest File (Markdown)": { "main": [[{ "node": "Collect Entry IDs", "type": "main", "index": 0 }]] }, + "Collect Entry IDs": { "main": [[{ "node": "Mark Entries As Read", "type": "main", "index": 0 }]] } + }, + "settings": { "executionOrder": "v1", "timezone": "Europe/Paris" } +}