Initial checkin of code for managing homelab

.forgejo/workflows/deploy.yml

@@ -0,0 +1,116 @@
+name: deploy
+
+on:
+  push:
+    branches: [main]
+    paths:
+      # No build/config/terraform/** — this workflow only deploys to Unraid,
+      # and `terraform apply` is run by hand. Add it if that ever changes.
+      - "src/**"
+      - "build/config/ansible/**"
+
+jobs:
+  deploy:
+    # Requires a self-hosted Forgejo Actions runner on your LAN (labelled
+    # `unraid-deploy`) with: network access to Vault and to the Unraid hosts'
+    # SSH + Postgres ports, and Python + pip available. See README.md
+    # "Bootstrapping" — this workflow can only run once that runner exists,
+    # which is why the first deploy has to happen by hand. The SSH key
+    # `ansible_user` (root) authenticates with is fetched from Vault below,
+    # not stored as a runner or repo secret — see README.md "SSH access".
+    runs-on: unraid-deploy
+    env:
+      VAULT_ADDR: ${{ secrets.VAULT_ADDR }}
+      VAULT_AUTH_METHOD: approle
+      VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID }}
+      VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # need history to diff — a shallow clone can't compute this
+
+      - name: Install Ansible and Python dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install ansible hvac psycopg2-binary
+
+      - name: Install Ansible collections
+        working-directory: build/config/ansible
+        run: ansible-galaxy collection install -r requirements.yml
+
+      - name: Fetch SSH deploy key from Vault
+        run: |
+          set -eu
+          python3 - <<'PY'
+          import os
+          import hvac
+
+          client = hvac.Client(url=os.environ["VAULT_ADDR"])
+          client.auth.approle.login(
+              role_id=os.environ["VAULT_ROLE_ID"],
+              secret_id=os.environ["VAULT_SECRET_ID"],
+          )
+          secret = client.secrets.kv.v2.read_secret_version(
+              path="homelab/ci/ssh", mount_point="kv"
+          )["data"]["data"]
+
+          key_path = os.path.join(os.environ["RUNNER_TEMP"], "unraid_ansible_key")
+          with open(key_path, "w") as f:
+              f.write(secret["PRIVATE_KEY"].rstrip() + "\n")
+          os.chmod(key_path, 0o600)
+          PY
+          echo "ANSIBLE_PRIVATE_KEY_FILE=$RUNNER_TEMP/unraid_ansible_key" >> "$GITHUB_ENV"
+
+      - name: Determine changed stacks
+        id: changed
+        run: |
+          set -eu
+          # Three possible outcomes, and they are not the same thing:
+          #   all=true          deploy every stack on every host
+          #   only_stacks=a,b   deploy just those
+          #   skip=true         nothing here affects the Unraid deployment
+          # An empty only_stacks must never be read as "deploy everything" —
+          # that is exactly what a Proxmox-only or Terraform-only push
+          # produces.
+          #
+          # A change under build/config/ansible/ can affect how every stack
+          # is deployed (the role, inventory, playbook), so don't try to
+          # narrow it. Same if the diff can't be computed at all (force-push,
+          # or a first push where `before` is all-zeros): fall back to a full
+          # deploy rather than silently deploying nothing.
+          if ! changed_files=$(git diff --name-only "${{ github.event.before }}" "${{ github.sha }}" 2>/dev/null); then
+            echo "all=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          if echo "$changed_files" | grep -q '^build/config/ansible/'; then
+            echo "all=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          # Only changes that affect the Unraid deployment count, since that
+          # is all this job runs. A stack's common/ and ansible/unraid/ do;
+          # its terraform/ and ansible/proxmox/ don't.
+          stacks=$(echo "$changed_files" \
+            | { grep '^src/' || true; } \
+            | { grep -Ev '/(terraform|ansible/proxmox)/' || true; } \
+            | awk -F/ '{ print ($2 == "shared") ? $3 : $2 }' \
+            | sort -u | paste -sd, -)
+          if [ -z "$stacks" ]; then
+            echo "skip=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "only_stacks=$stacks" >> "$GITHUB_OUTPUT"
+          fi
+
+      # Unraid only. Deploying to Proxmox from CI would need the runner to
+      # reach the Proxmox API and the guests, and `terraform apply` is
+      # deliberately kept manual — creating and destroying LXCs on a push is
+      # a bigger blast radius than restarting a Compose stack. Run
+      # playbooks/proxmox.yml by hand for now.
+      - name: Run deploy playbook
+        if: steps.changed.outputs.skip != 'true'
+        working-directory: build/config/ansible
+        run: |
+          if [ "${{ steps.changed.outputs.all }}" = "true" ]; then
+            ansible-playbook playbooks/unraid.yml
+          else
+            ansible-playbook playbooks/unraid.yml -e only_stacks="${{ steps.changed.outputs.only_stacks }}"
+          fi

.gitignore

@@ -0,0 +1,33 @@
+# Ansible
+*.retry
+build/config/ansible/inventory/*.retry
+
+# Rendered secrets — only .env.example is ever committed
+**/.env
+!**/.env.example
+
+# Terraform
+# .terraform/ holds the provider cache and, after
+# `terraform init -backend-config=...`, the Postgres connection string with
+# its password — never commit it. .terraform.lock.hcl is deliberately NOT
+# ignored: it pins provider versions and belongs in git.
+.terraform/
+*.tfstate
+*.tfstate.*
+*.tfvars
+!*.tfvars.example
+# A saved plan (`terraform plan -out=…`) is not just a diff: it embeds a copy
+# of the state it was made against *and* the values of every input variable,
+# `sensitive` ones included — so build/config/terraform/deploy.plan contains
+# the Proxmox API token in the clear. Ignored so that a `git add -A` can't
+# publish it.
+*.plan
+
+# OS cruft
+.DS_Store
+Thumbs.db
+
+# Scratch output that isn't state and isn't secret-free — e.g. the k3s
+# kubeconfig playbooks/k3s.yml fetches to local/k3s/<cluster>.kubeconfig,
+# which carries a client cert.
+local/

CLAUDE.md

@@ -0,0 +1,597 @@
+# Homelab IaC — Project Notes
+
+Manages apps as infrastructure-as-code across three platforms — Unraid
+server(s), Proxmox, and a bare-metal k3s cluster — with secrets sourced from
+HashiCorp Vault and CI/CD via Forgejo Actions (self-hosted, on the Forgejo
+instance this repo deploys).
+
+Ansible is the deployment tool on **all three**; what differs is the
+deployment primitive and who creates the host:
+
+| | Unraid | Proxmox | k3s |
+|---|---|---|---|
+| Provision the host | already exists | Terraform (`bpg/proxmox`) creates an LXC | already exists (4 Raspberry Pis) |
+| Install the app | Ansible + `compose_stack` → `docker compose up -d` | Ansible + `lxc_app` → native install + systemd | Ansible + `k3s_app` → manifests into k3s's auto-deploy dir |
+| App config source | `common/vars.yml` + `ansible/unraid/vars.yml` | `common/vars.yml` + `ansible/proxmox/vars.yml` | `common/vars.yml` + `ansible/kubernetes/vars.yml` |
+| App list lives in | `host_vars/<host>.yml` `stacks:` | `host_vars/<guest>.yml` `apps:` | `group_vars/k3s_cluster.yml` `k3s_apps:` |
+
+The k3s column is the newest and reached that shape late: `homelab-utils`
+started as cluster-bootstrap-only, with no `src/<app>/` entry, and the third
+column was added when Authentik moved onto it (see
+`docs/authentik-migration.md`). Two things about it don't reduce to the
+table:
+
+- **The cluster itself has no `src/` entry.** Terraform never provisions it
+  and it isn't an app, so it gets an inventory group (`k3s_cluster`) and
+  `roles/k3s_node`, the same way the Proxmox *node* gets `roles/pve_backup`
+  while its guests get `lxc_app`.
+- **Cluster services aren't apps.** `k3s_metallb`, `k3s_monitoring`,
+  `k3s_postgres` and `k3s_cert_manager` are the cluster's own
+  infrastructure — one hard-coded role each, config in `group_vars`. Apps go
+  through the generic `k3s_app` and live in `src/`. Both end up as
+  `HelmChart` CRs, so they look alike on disk; the distinction is whether
+  anything would still need it if no app were deployed. See "K3s cluster
+  (homelab-utils)" below.
+
+### Hosts
+
+Two Unraid entries are gone from the original plan: `nas2` was retired, and its
+address (192.168.50.2) was reassigned to `turtle-proxmox-01` — so a stale copy
+of it fails by reaching the wrong machine rather than by timing out. What
+remains: `nas01` (Unraid, 192.168.50.1), `turtle-proxmox-01`/`-02` (Proxmox,
+.2/.3), the guests on them from the API inventory, and the four k3s Pis
+(.60–.63, MetalLB pool .80–.99). Gateway is 192.168.50.254.
+
+**Proxmox LXC guests live in 192.168.50.50–.59** — Forgejo .52, shared
+Postgres .54. Addresses here are assigned by hand from that band, not derived
+from anything, so a new guest's `ip_address` is a question for whoever owns
+the network rather than a value to pick.
+
+The shared Postgres for everything outside the cluster is now the Proxmox LXC
+at 192.168.50.54 — see `docs/postgres-proxmox.md`. The Unraid Compose
+deployment under `src/shared/postgres/ansible/unraid/` is kept for the
+platform-shape reason every unused platform folder is kept, not because it is
+running anywhere.
+
+This repo supersedes the planning done in `../homelab/CLAUDE.md` — that file
+has the original decision log if you want the "why" behind the architecture
+in more detail. Multi-host support, the shared-Postgres pattern, and the
+Forgejo-Actions-hosts-itself bootstrap problem were decided after that doc
+was written; this file and `README.md` are the current source of truth.
+
+## Layout
+
+- `src/<app>/common/vars.yml` — `env_defaults:` that hold regardless of where
+  the app runs (version, ports, domain, DB name/user).
+- `src/<app>/ansible/unraid/` — the Compose deployment: `docker-compose.yml`,
+  `.env.example`, a `vars.yml` of Unraid-specific overrides (appdata paths,
+  the `shared-postgres` Docker network alias), and optionally an `icon.png`
+  for the Unraid Docker page.
+- `src/<app>/ansible/proxmox/` — the native install: `vars.yml` of
+  Proxmox-specific overrides, `install.yml` of app-specific install steps,
+  and templates for the app's own config file and systemd unit. Two apps
+  ship one: `src/forgejo/` (binary + systemd unit, the shape the role was
+  written for) and `src/shared/postgres/` (distro packaging, so the unit
+  comes from the package and `install.yml` only reconfigures the cluster).
+- `src/<app>/ansible/kubernetes/` — the k3s deployment: `vars.yml` of
+  cluster-specific overrides, and one or more `*.yaml.j2` manifests (normally
+  a `HelmChart` CR, plus anything the chart won't create — see
+  `src/authentik/`). "kubernetes" rather than "k3s": the contents are plain
+  Kubernetes apart from the `HelmChart` CR.
+- `src/<app>/terraform/` — a module defining this app's LXC (template, cores,
+  memory, disk, IP), called from `build/config/terraform/main.tf`.
+- `src/shared/<service>/` — shared services (databases, etc.) used by
+  multiple apps, same shape.
+- `build/config/ansible/` — inventory, playbooks, and the roles. Three of
+  them deploy *apps*, one per platform, and share the same layering and
+  `state: present|absent` contract: `compose_stack` (Unraid), `lxc_app`
+  (Proxmox guests), `k3s_app` (the cluster). `compose_stack` and `k3s_app`
+  each split into `deploy.yml`/`remove.yml`, dispatched by `main.yml` on the
+  entry's `state:`. The rest configure infrastructure rather than apps:
+  `k3s_node` (bootstraps a Pi as k3s server or agent, picked by inventory
+  group), `k3s_metallb`, `k3s_monitoring`, `k3s_postgres`,
+  `k3s_cert_manager` (cluster services), `k3s_maintenance` (drain/reboot/
+  uncordon), `unattended_upgrades`, and `pve_backup` (the Proxmox node's
+  backup storage and vzdump schedule).
+- `build/config/terraform/` — Proxmox provider + `backend "pg"` state config,
+  and the module calls saying what infrastructure exists.
+- `.forgejo/workflows/` — CI/CD, runs once Forgejo + a self-hosted runner
+  exist (see "Bootstrapping" in README.md — chicken-and-egg on the first
+  deploy).
+
+## Key decisions
+
+- Compose files are static and reference `${VAR}`; only the rendered `.env`
+  changes per deploy. Nothing secret is ever committed.
+- Config is layered, not duplicated: `common/vars.yml` merged with
+  `ansible/<platform>/vars.yml` (platform wins), then Vault secrets on top
+  (secrets win). Both roles do this the same way, so moving an app between
+  platforms means writing a new platform vars file, not restating its config.
+- One Vault KV path per app (`homelab/<app>`), fetched as a whole dict via
+  `community.hashi_vault.vault_kv2_get`. The prefix is `homelab/`, not
+  `unraid/`, because an app's secrets don't change with the platform it lands
+  on.
+- Unraid draws a Compose deployment's stack row and its containers from
+  unrelated sources, so one committed `src/<app>/ansible/unraid/icon.png` is
+  copied to two places. The **stack row** is Compose Manager serving
+  `<project dir>/icon.png` off disk — a file or nothing, no label and no URL,
+  and only in the maintained fork (Compose Manager Plus); on the original
+  plugin the copy is inert. The **containers** get icon, WebUI link and
+  console shell from the dockerMan template that created them, which a Compose
+  stack doesn't have, so each service carries `net.unraid.docker.*` labels
+  (6.10+ falls back to them). The label path is a second copy under
+  `/mnt/user/appdata/icons/` rather than the project-folder one, because the
+  webgui resolves it on every Docker page load and that shouldn't hit the
+  flash drive.
+- Of those labels only the icon is role-supplied: `compose_stack` injects
+  `STACK_ICON` *underneath* both vars.yml layers, so an app can override it
+  with a hosted URL. WebUI and shell stay literal in the compose file — the
+  WebUI port has to be the published one and the shell is a property of the
+  base image, so both belong next to `ports:` and `image:` rather than in a
+  variable. Details, including where SVG does and doesn't work, in README.md →
+  "Icons on the Unraid Docker page".
+- Shared Postgres is provisioned once per host; per-app DB/user creation
+  is idempotent via `community.postgresql`, not `docker-entrypoint-initdb.d`
+  (which only runs once on an empty volume — wrong fit for a shared service
+  apps get added to over time). The Unraid instance was pinned to 13 because
+  its data directory predated this repo; the Proxmox one is 17 from the PGDG
+  archive, a fresh cluster with nothing to stay compatible with.
+- The Proxmox shared Postgres lives on the `AppData` ZFS pool with a
+  `proxmox_replication` job to the second node, and **everything is on one
+  volume** — no `mount_point` splitting PGDATA from the WAL. Unlike Forgejo's
+  "no bind mounts", which is about what `vzdump` includes, this is a
+  correctness requirement: replication snapshots each of a guest's volumes
+  independently, so two volumes replicate as two snapshots taken at different
+  instants, and the far node can end up with a data directory and a WAL that
+  disagree. One volume means one atomic snapshot — exactly the
+  crash-consistent image WAL replay is designed for. It also means running
+  out of space has one fix and not two: raise `disk_size` on the module call
+  (256 GiB as of this writing, up from the original 32 once Forgejo's
+  database made the OS-sized rootfs look optimistic), never add a second
+  disk. Cheap to do — on ZFS the resize is a refquota change applied to a
+  running guest — but one-way, since shrinking is a replacement and
+  `prevent_destroy` blocks it. Runbook in `docs/postgres-proxmox.md`. Two
+  further consequences worth keeping straight: replication is configured *per guest*, not per pool, so a
+  container created on `AppData` without a job is a single point of failure
+  that looks identical in the storage view (hence Terraform owning the job
+  rather than a hand-run `pvesr`); and it's asynchronous, so a failover loses
+  up to `replication_schedule` — five minutes — of commits. It is not a
+  backup; `pve_backup`'s vzdump is.
+- `full_page_writes = off` on the Proxmox instance, and only there. It guards
+  against a torn page, which copy-on-write storage cannot produce, so on ZFS
+  the full-page images are pure write amplification that also becomes
+  replication traffic. The same setting on ext4 or xfs risks unrecoverable
+  corruption — which is why it's in `ansible/proxmox/vars.yml` and must never
+  migrate to `common/vars.yml`.
+- Terraform provisions the guest and stops there — it has no way to deploy an
+  app into an LXC, so Ansible picks up from the point the container exists.
+- Proxmox guests keep all their state on their own rootfs; no bind mounts, no
+  NFS shares mounted into a container. Counter-intuitive if the goal is "back
+  the repos up to the NAS", but `vzdump` excludes bind mounts by design, so a
+  bind-mounted repo directory is the one thing *missing* from the archive.
+  Everything local means one archive is a complete filesystem restore point.
+  Getting a copy off-box is the node's job (`pve_backup` → NFS storage on the
+  NAS + a vzdump schedule), not the container's.
+- State an app keeps *outside* its container is that app's problem. Forgejo's
+  database is on the shared Postgres LXC — a different guest, and so a
+  different vzdump archive — so a systemd timer in the
+  container `pg_dump`s it onto the container's own disk, and the vzdump archive
+  carries both halves. The dump schedule and the vzdump window are a pair —
+  move one, move the other.
+- Proxmox hosts come from the `community.proxmox` dynamic inventory plugin
+  querying the Proxmox API, so `terraform apply` creating an LXC doesn't also
+  require an inventory edit.
+- The `terraform` tag is what marks a Proxmox guest as this repo's to manage.
+  The dynamic inventory returns every LXC on the cluster, hand-made ones
+  included, so `inventory/proxmox.yml` filters on that tag — both Terraform
+  modules set it (`tags` in their `variables.tf`), nothing else does. It
+  filters rather than narrowing the `proxmox_guests` condition, so an
+  unmanaged guest is absent from `all` as well and no future play can reach
+  it by accident. The failure mode is silent in one direction: a new module
+  that forgets the tag creates a container Ansible simply never visits, with
+  no error anywhere — same shape as an omitted `stacks:` entry, and the first
+  thing to check if a freshly-applied guest is never deployed to.
+- Terraform state lives in Postgres (`backend "pg"`), not a local file, so CI
+  and a workstation see the same state — specifically the **CloudNativePG
+  cluster on k3s**, not the shared Postgres this configuration provisions on
+  Proxmox. That would be circular: Terraform would need the database to exist
+  in order to create the container the database runs in. State goes somewhere
+  Terraform has no hand in building, which breaks the cycle outright at the
+  cost of a dependency the rest of the repo avoids — `terraform apply` now
+  needs the Pis up. Worth it over the alternative (local state, apply,
+  `init -migrate-state`), which has to be got right exactly once. Bootstrap
+  ordering is `docs/postgres-proxmox.md`.
+- Every app keeps the same `common/` + `ansible/` + `terraform/` shape even
+  when only one platform is in use, so adding the other later doesn't require
+  restructuring.
+- Removal on Unraid is explicit (`state: absent` on the stack entry), never
+  implied by deleting the entry. Ansible stores no record of the last run, so
+  an omitted stack isn't removed — it's just never visited again, and keeps
+  running unmanaged. The entry stays as a tombstone documenting that the app
+  was deliberately removed. Volumes, images and the app's database are
+  further opt-ins (`remove_volumes`, `remove_images`, `remove_database`)
+  rather than being implied by `state: absent`, and the Vault path is never
+  touched — a default teardown is meant to be reversible. `lxc_app` has no
+  counterpart yet; it would need a per-app `uninstall.yml`.
+- The k3s cluster's join token is a fixed value from Vault
+  (`homelab/k3s-homelab-utils` → `K3S_TOKEN`), not the one `k3s server`
+  generates for itself on first install. Both the control-plane play and the
+  worker play look it up independently, so a full rebuild — wipe both SD
+  cards, reinstall — reproduces the same cluster identity instead of needing
+  a freshly-generated token fetched off the server and redistributed by hand.
+  Same "generate once, store it, never let a redeploy invent a new one"
+  reasoning as Forgejo's fixed secrets, applied to cluster identity instead
+  of app state.
+- The SSH key the k3s plays connect with is fetched from Vault
+  (`homelab/ci/ssh-k3s`) by `playbooks/k3s_ssh_key.yml` and written to
+  `local/`, which both `k3s.yml` and `k3s_maintenance.yml` import as their
+  first play — rather than the operator loading it into `ssh-agent` first, as
+  the Unraid manual flow still expects. Same arrangement CI already uses for
+  the Unraid key, just moved into the playbook because these two are
+  manual-only and there's no workflow to hold the fetch step. It's a separate
+  playbook, not a `pre_tasks` block, because both entry points need it and
+  neither can assume the other ran. It targets `k3s_cluster` rather than
+  `localhost` for a non-obvious reason: the implicit localhost isn't a member
+  of `all`, so it wouldn't inherit `vault_addr`/`vault_kv_mount`/
+  `vault_auth_method` from `group_vars/all.yml`. Nothing in it connects to a
+  Pi — `gather_facts: false` plus `delegate_to: localhost`, necessarily, since
+  the key isn't on disk yet at that point. Falling back to a local key takes
+  *two* overrides (`-e k3s_ssh_key_fetch=false` alongside
+  `-e ansible_ssh_private_key_file=...`): pointing at another key alone
+  doesn't help if Vault is what's broken, because the fetch play fails before
+  any other play runs.
+- `playbooks/k3s.yml` is deliberately **not** wired into `deploy.yml` or
+  `.forgejo/workflows/deploy.yml`. Converging 4 physical nodes on every push
+  is a bigger blast radius than restarting a Compose stack — the same
+  reasoning that keeps `terraform apply` manual for Proxmox. Run it by hand.
+- `playbooks/k3s.yml`'s plays are tagged so one piece can be converged on its
+  own — `nodes`, `metallb`, `monitoring`, `postgres`, `services` (those three
+  cluster services together), `upgrades`, `kubeconfig`. Play-level tags, one
+  per play, not per-task tags inside the roles: the plays already are the
+  units, and tagging tasks would mean every role carrying tags for a
+  selectivity only this playbook needs. This is the k3s equivalent of
+  `only_stacks` for Unraid — a cluster service isn't a `stacks:` entry, so
+  `only_stacks` can't reach it. The SSH-key import play is tagged `always`
+  rather than getting a name of its own, since it's the prerequisite for
+  anything else connecting and has to survive every `--tags` filter. Note
+  what a tagged service run does *not* cover: the roles only drop manifests
+  for k3s's controllers to reconcile, so a green playbook means the files
+  landed, not that the workload came up.
+- `k3s_node` is one role, not two, even though a server and an agent install
+  differently. Both need the same OS prep (cgroups, swap), and which install
+  path runs is a group var (`k3s_node_role: server`/`agent`, set per
+  inventory group) rather than a second role — so the shared prep can't drift
+  between the two nodes types the way copy-pasted tasks eventually would.
+- Extra `INSTALL_K3S_EXEC` flags split across two vars, not one:
+  `k3s_extra_args` (both roles) and `k3s_server_extra_args` (control-plane
+  only). `k3s agent` doesn't understand server-only flags like `--disable`
+  and fails to start if handed one, so a flag like `--disable=servicelb`
+  (which `k3s_metallb` needs to avoid fighting k3s's bundled ServiceLB for
+  the same IPs) can't safely live in the shared list.
+- `k3s_node`'s install tasks track the exec line they last installed a node
+  with (a small marker file next to k3s's own config) and reinstall —
+  restarting the `k3s`/`k3s-agent` service, not the node — whenever it
+  drifts from what `k3s_extra_args`/`k3s_server_extra_args`/`k3s_api_tls_san`
+  currently say, even if `k3s_version` hasn't changed. Deliberately folded
+  into the ordinary install task rather than a separate maintenance
+  playbook: `playbooks/k3s.yml` is already manual-only and documented as
+  safe to re-run, and unlike the reboots `k3s_maintenance` guards, bouncing
+  the k3s process doesn't take pods down — containerd keeps them running
+  underneath it. The worker play still runs with `serial: 1` so at most one
+  node's kubelet is ever mid-restart at once.
+- `k3s server`/`k3s agent` are installed with an explicit `--node-name
+  {{ inventory_hostname }}`, not left to default to the OS hostname. This is
+  what lets `roles/k3s_maintenance` address a node by
+  `inventory_hostname` when draining/uncordoning — the k8s node object and
+  the Ansible host are guaranteed to be the same string.
+- Patching a k3s Pi is split into two roles that don't know about each other
+  directly, only through a file: `unattended_upgrades` installs updates
+  hands-off but with `Automatic-Reboot "false"`, and `k3s_maintenance` is the
+  only thing that actually reboots a node — triggered by the presence of
+  `/var/run/reboot-required`, which is the OS's own signal, not something
+  either role invents. Splitting it this way means the risky half (taking a
+  node out of the cluster) is exactly one thing (`playbooks/k3s_maintenance.yml`,
+  `serial: 1`), not entangled with the routine half (installing packages),
+  which runs unattended every day on every node without anyone thinking about
+  it.
+- `k3s_maintenance`'s kubectl calls (`drain`, `wait`, `uncordon`) are
+  delegated to the control-plane host and run as `k3s kubectl`, k3s's own
+  bundled client, rather than requiring a kubectl install or a kubeconfig
+  anywhere else — including on whatever eventually runs this on a schedule.
+  This also has to work when the node currently being processed *is* the
+  control plane: delegating a host to itself is just a normal SSH connection,
+  used right up until the moment that connection reboots out from under it.
+- `k3s_monitoring` (a lean kube-prometheus-stack, for OpenLens and similar
+  tools) is installed as a `HelmChart` custom resource dropped into k3s's own
+  auto-deploying manifests directory, not run through a `helm` binary or the
+  `kubernetes.core.helm` collection — k3s ships a helm-controller that
+  reconciles anything found there, the same mechanism it uses to install its
+  own bundled Traefik and ServiceLB. That keeps the pattern this repo already
+  uses everywhere else (Ansible renders a file, something else converges on
+  it) instead of adding a second, unrelated way to reach the cluster from the
+  controller. Grafana and Alertmanager are left disabled — OpenLens brings
+  its own dashboards and this cluster doesn't page anyone — and the
+  control-plane scrape targets (`kubeControllerManager`, `kubeScheduler`,
+  `kubeProxy`, `kubeEtcd`) are disabled too, since k3s bundles those into one
+  static binary instead of exposing them the way the chart expects; leaving
+  them on just produces permanently-"down" targets, not a working scrape.
+- `k3s_metallb` (MetalLB, L2 mode) follows the same `HelmChart` CR pattern as
+  `k3s_monitoring`, plus a second, plain manifest for its
+  `IPAddressPool`/`L2Advertisement` config dropped in the same
+  auto-deploying directory — k3s's deploy controller applies any manifest it
+  finds there, not only `HelmChart` CRs, and retries one referencing CRDs
+  that don't exist yet rather than failing outright, so the config doesn't
+  need to wait on the chart install finishing first. It replaces k3s's
+  bundled ServiceLB rather than running next to it — both would otherwise
+  hand out IPs for the same `LoadBalancer` Services — so
+  `k3s_server_extra_args` in `inventory/group_vars/k3s_cluster.yml`
+  carries `--disable=servicelb` (server-only — `k3s_extra_args`, applied to
+  both server and agent, can't carry a server-only flag like `--disable`
+  without breaking agent installs). The IP pool (`k3s_metallb_address_range`, same
+  file) has no built-in default; the role fails fast rather than silently
+  advertising an empty pool.
+- `k3s_postgres` (CloudNativePG) is the k3s cluster's own shared Postgres —
+  a third platform for `src/shared/postgres/`, alongside Unraid and Proxmox,
+  deployed via Helm rather than Compose or a native install. Chosen over
+  Bitnami's postgresql-ha (repmgr+pgpool, and Bitnami's free chart/image
+  catalog was restructured into a "legacy" repo in 2025) and the Zalando
+  operator (Patroni-based, heavier) because it fits the same `HelmChart` CR
+  pattern already established: the operator installs as a chart CR, same
+  shape as `k3s_metallb`/`k3s_monitoring`, and the actual cluster is a plain
+  `Cluster` CR manifest — same "config manifest that outlives the CRDs it
+  references" trick as `k3s_metallb`'s `IPAddressPool`. It's a separate
+  physical instance from the Docker-based shared/postgres (pinned to 13),
+  but reuses the same Vault path (`homelab/shared/postgres`) rather than a
+  k3s-specific one — one superuser identity for the "shared postgres"
+  concept regardless of which platform it's running on, consistent with how
+  every host already draws from that same path. 1 primary + 1 replica,
+  scheduled on worker Pis only via `nodeAffinity` — the control plane stays
+  free of app pods, same boundary `k3s_metallb`/`k3s_monitoring` already
+  keep. HA replication only for now, no backups.
+- The CNPG cluster is published on the LAN through MetalLB at a pinned
+  `k3s_postgres_loadbalancer_ip`, on top of the three `ClusterIP` Services
+  CNPG makes for every `Cluster`. Declared inside the `Cluster` CR as a
+  managed service (`.spec.managed.services.additional`, `selectorType: rw`)
+  rather than as a Service manifest of this repo's own, so the operator owns
+  the selector and the address follows a failover instead of needing to be
+  re-pointed by hand. Pinned rather than auto-assigned because the address is
+  meant to be written down in config elsewhere; it comes out of the bottom of
+  `k3s_metallb_address_range` and would need a second `autoAssign: false`
+  pool if anything else on this cluster ever wants a fixed IP. This is what
+  closes the reachability half of per-app DB provisioning — the
+  `community.postgresql` tasks `compose_stack`/`lxc_app` use can now reach
+  this instance the same way they reach the Unraid one. `roles/k3s_app` is
+  the other half, and uses exactly those tasks; note the asymmetry it
+  relies on, which is easy to misread as a mistake. The *provisioning* runs
+  from the Ansible controller and therefore uses the LoadBalancer address
+  (`k3s_postgres_loadbalancer_ip`), because `delegate_to: localhost` is off
+  the cluster and can't route to a ClusterIP. The *app* uses the in-cluster
+  `shared-postgres-rw` DNS name, because sending pod traffic out to the LAN
+  and back would put MetalLB's L2 speaker in the path of every query for no
+  benefit. Two addresses for one database, each correct for its caller.
+- `roles/k3s_app` is the cluster's counterpart to `compose_stack` and
+  `lxc_app`: the generic role that deploys an *app*, driven by `k3s_apps:`
+  in `group_vars/k3s_cluster.yml`, as distinct from the `k3s_*` roles that
+  each deploy one cluster service. It renders whatever `*.yaml.j2` the app
+  ships under `src/<app>/ansible/kubernetes/` into k3s's auto-deploying
+  manifests directory, so adding a manifest to an app is dropping a file
+  next to the others rather than a role change. Secrets go into a separate
+  Kubernetes Secret rendered from Vault, never interpolated into the
+  manifests — the same committed-config/generated-secrets split as Unraid's
+  static compose file plus rendered `.env`, which is why the manifests can
+  stay 0644 on the node while the Secret is 0600.
+- `k3s_apps:` lives in `group_vars/k3s_cluster.yml`, not `host_vars/`, unlike
+  both other platforms. An app is deployed to the cluster, not to a node:
+  the play runs against `k3s_control_plane` only because that's where the
+  manifests directory is, which is an implementation detail of how k3s is
+  reached rather than a statement about where the app runs.
+- Removal on k3s is one pass, not two. `compose_stack` needs the compose file
+  still on disk to run `docker compose down`, so `state: absent` and deleting
+  `src/<app>/` have to be separate commits (see "Removing an app" below).
+  k3s's deploy controller tracks which resources each manifest file created,
+  so deleting the file *is* the teardown and `src/<app>/` can go in the same
+  commit. The tombstone convention still applies for the same reason as
+  everywhere else — an entry dropped from `k3s_apps:` is never visited again
+  and keeps running unmanaged.
+- `k3s_cert_manager` is the fourth cluster service and the first that exists
+  purely for apps rather than for the cluster — nothing in the cluster needs
+  a certificate, `k3s_app`'s tenants do. It's still a cluster service rather
+  than an app, on the same reasoning as the shared Postgres: one
+  `ClusterIssuer` that every app's Ingress annotates itself against, rather
+  than each app carrying its own ACME account and DNS credentials. DNS-01,
+  not HTTP-01, because HTTP-01 needs Let's Encrypt to reach the cluster from
+  the internet on port 80 and this LAN deliberately isn't reachable — which
+  has the useful side effect that a certificate can be issued *before* DNS
+  points at the cluster, so a migration's TLS is settled before its cutover.
+  The solver stanza itself has no default and the role fails fast without
+  one: it depends on who runs the DNS, which the repo can't know. Same
+  fail-fast-on-unset treatment as `k3s_metallb_address_range`.
+- Apps on k3s reach the outside through the Traefik k3s already bundles,
+  on the MetalLB address its Service holds, rather than each getting its own
+  LoadBalancer IP. One entrypoint and host-based routing means adding an app
+  costs a DNS record, not a pool address — the opposite of the choice made
+  for `k3s_postgres`, which is pinned to its own IP precisely because
+  Postgres isn't HTTP and can't be name-routed.
+- `k3s_traefik` is the one cluster-service role that installs nothing: k3s
+  installs Traefik itself, so the role only adjusts it, via a
+  `HelmChartConfig` merged over k3s's own `HelmChart` rather than an edit to
+  the `traefik.yaml` k3s rewrites on every server start. Its scope is the
+  dashboard, which a stock k3s 404s — Traefik still builds it, but the chart
+  stopped shipping the router that reaches it in v28. The role restores that
+  router on Traefik's internal `traefik` entrypoint (port 9000, unpublished →
+  port-forward only), and optionally publishes the dashboard on a hostname
+  behind an Authentik forward-auth middleware when
+  `k3s_traefik_dashboard_host` is set. Three things worth keeping straight:
+  the internal route stays on even when the hostname one exists, because
+  authenticated access depends on Authentik → CNPG → a healthy cluster,
+  exactly what you'd open the dashboard to diagnose; the published route is
+  an `IngressRoute` with an explicit cert-manager `Certificate` rather than
+  an annotated `Ingress`, because the dashboard is `api@internal` and has no
+  Service to point an `Ingress` at; and the role fails fast if a hostname is
+  set without an auth address, since the failure mode is publishing every
+  router, service and middleware on the cluster to the LAN unauthenticated.
+  Authentik's provider must be **forward auth (domain level)** —
+  single-application mode needs `/outpost.goauthentik.io/` routed on the
+  dashboard's own host, a cross-namespace service reference Traefik rejects
+  unless `allowCrossNamespace` is on.
+
+## Adding a new app
+
+1. `src/<app>/common/vars.yml` — `env_defaults:` for the portable config.
+2. Whichever platform(s) it targets:
+   - **Unraid:** `src/<app>/ansible/unraid/` with `docker-compose.yml`
+     (static, `${VAR}`-driven), `vars.yml` of overrides, `.env.example`
+     documenting every var.
+   - **Proxmox:** `src/<app>/ansible/proxmox/` with `vars.yml`,
+     `install.yml`, and config/systemd templates; plus
+     `src/<app>/terraform/` defining its LXC, wired into
+     `build/config/terraform/main.tf`.
+   - **k3s:** `src/<app>/ansible/kubernetes/` with `vars.yml` and at least
+     one `*.yaml.j2` manifest — normally a `HelmChart` CR, plus anything the
+     chart won't create for you. See `src/authentik/` for a worked example,
+     including the two things a chart usually leaves out: a PVC, and secrets
+     referenced by `secretKeyRef` rather than set as values.
+3. Populate its Vault path (`homelab/<app>`) with real secret values.
+4. Declare it on the target:
+   - Unraid → `host_vars/<host>.yml` `stacks:` list.
+   - Proxmox → `host_vars/<guest>.yml` `apps:` list.
+   - k3s → `group_vars/k3s_cluster.yml` `k3s_apps:` list.
+   All three take the same entry shape (name, src, vault_path, optional
+   `db:`, optional `state:`). Never declare the same app on more than one at
+   once — they share a Vault path and a database, so two live deployments
+   corrupt each other's state. Moving an app *between* platforms is
+   therefore a cutover with a verification step in the middle, not an edit;
+   `docs/authentik-migration.md` is the worked example.
+5. Leave the unused platforms' folders as README placeholders rather than
+   deleting them — the shape is the point.
+
+## Removing an app
+
+**Unraid.** Set `state: absent` on its `stacks:` entry and run
+`playbooks/unraid.yml` — don't delete the entry, which removes nothing. Then
+delete `src/<app>/` in a *second* pass, never before: `docker compose down`
+needs the compose file to know what it's tearing down. Full detail, including
+the opt-in flags for volumes/images/database, is in README.md → "Removing an
+app".
+
+**k3s.** Same `state: absent` tombstone rule, but one pass — deleting the
+manifests is the teardown, because k3s's deploy controller garbage-collects
+what each file created, so `src/<app>/` can go in the same commit. The
+database, PVCs and Vault path deliberately survive; there's no
+`remove_volumes`/`remove_database` equivalent yet, and adding one is a
+"when it's actually wanted" job rather than a guess at the shape.
+
+**Proxmox.** No teardown path at all — `lxc_app` would need a per-app
+`uninstall.yml`, and neither app that ships an `install.yml` has one. Note
+this is worse for the shared Postgres than for an ordinary app: tearing it
+down means every other app's database as well.
+
+## K3s cluster (homelab-utils)
+
+4 Raspberry Pis, `inventory/hosts.yml` → `k3s_cluster` (`k3s_control_plane`:
+1 host, `k3s_workers`: 3). No Terraform — the Pis already exist. The cluster
+itself has no `src/<app>/` entry (it isn't an app), but apps now land on it:
+`k3s_apps:` in `group_vars/k3s_cluster.yml`, deployed by `roles/k3s_app` from
+`src/<app>/ansible/kubernetes/`. `playbooks/k3s.yml` is manual-only (see "Key
+decisions" above); run it with:
+
+```sh
+cd build/config/ansible
+ansible-playbook playbooks/k3s.yml
+
+# or one piece of it — see "Key decisions" above for the tag list
+ansible-playbook playbooks/k3s.yml --tags postgres
+
+# apps only, or one app
+ansible-playbook playbooks/k3s.yml --tags apps
+ansible-playbook playbooks/k3s.yml --tags apps -e only_apps=authentik
+```
+
+`--tags services` covers the cluster's own infrastructure and `--tags apps`
+what runs on top of it; neither implies the other, so "everything except
+reinstalling k3s" is `--tags services,apps`.
+
+Prerequisites, once, before the first run: the `ansible` user exists on every
+Pi with NOPASSWD sudo and the **public** half of this repo's k3s SSH key
+installed, and `homelab/k3s-homelab-utils` → `K3S_TOKEN` plus
+`homelab/ci/ssh-k3s` → `PRIVATE_KEY` are set in Vault. Nothing needs setting
+up on the controller beyond `VAULT_ADDR`/`VAULT_TOKEN` — the private key comes
+from Vault at run time (see "Key decisions" above). Full detail in README.md →
+"K3s (Raspberry Pi)" and `docs/vault-secrets.md`.
+
+**Rebuilding**: reimage the SD card(s), redo the `ansible` user + SSH key
+step, then re-run the playbook — the fixed Vault token means the rebuilt
+node(s) rejoin the same cluster identity rather than needing a token hunted
+down from a live server. **Adding a fifth Pi**: add it under
+`k3s_control_plane` or `k3s_workers` in `hosts.yml`; no `host_vars/` entry
+needed, `k3s_node_role` comes from the group. This repo's `k3s_node/server.yml`
+assumes a single control-plane node — adding a second one for HA needs
+changes there, not just an inventory edit.
+
+**Patching**: `playbooks/k3s.yml` also applies `roles/unattended_upgrades` to
+every node, so updates install themselves daily with no login required — but
+never reboot themselves (`Automatic-Reboot "false"`). Run
+`playbooks/k3s_maintenance.yml` to actually apply a pending reboot: it checks
+`/var/run/reboot-required` per node and, only where set, drains → reboots →
+waits for `Ready` → uncordons, `serial: 1` so only one node is ever down.
+Manual-only for now, same as `playbooks/k3s.yml` — meant to eventually run on
+a schedule from a self-hosted Forgejo Actions runner rather than by hand.
+
+**Metrics**: `playbooks/k3s.yml` also applies `roles/k3s_monitoring`, a lean
+kube-prometheus-stack (no Grafana/Alertmanager) so OpenLens and similar tools
+can show node/pod metrics — see "Key decisions" above for why it's a
+`HelmChart` CR rather than a `helm` install, and README.md → "Metrics
+(Prometheus)" for pointing OpenLens at it.
+
+**LoadBalancer IPs**: `playbooks/k3s.yml` also applies `roles/k3s_metallb`,
+MetalLB in L2 mode, so `type: LoadBalancer` Services get real LAN IPs instead
+of staying `ClusterIP`-only. Set the pool it hands out from —
+`k3s_metallb_address_range` in `inventory/group_vars/k3s_cluster.yml` — to a
+range your DHCP scope and every static assignment in `hosts.yml` steer clear
+of before the first run; see "Key decisions" above for why it replaces
+ServiceLB rather than running alongside it, and README.md → "LoadBalancer
+IPs (MetalLB)" for more.
+
+There's no *node* teardown path, unlike `state: absent` for apps — removing a
+Pi today means wiping its SD card and deleting its inventory entry by hand.
+
+**Shared Postgres**: `playbooks/k3s.yml` also applies `roles/k3s_postgres`,
+CloudNativePG — the cluster's own shared Postgres, a third platform for
+`src/shared/postgres/` alongside Unraid and Proxmox. See "Key decisions"
+above for why CloudNativePG and the topology, and README.md → "Postgres
+(CloudNativePG)" for connecting to it. Per-app database provisioning works
+now (`roles/k3s_app`, `db:` on a `k3s_apps:` entry) — note the two-address
+asymmetry described in "Key decisions". Still **no backups**: HA replication
+only, which protects against a dead SD card and not against a dropped table.
+That gap matters more now that Authentik's database lives here rather than
+inside whatever covers Unraid's `/mnt/user/appdata`; a CNPG `ScheduledBackup`
+is the obvious next piece of work.
+
+**TLS**: `playbooks/k3s.yml` also applies `roles/k3s_cert_manager` —
+cert-manager plus one `ClusterIssuer` that every app's Ingress annotates
+itself against. Two values have no default and the role refuses to run
+without them: `k3s_cert_manager_acme_email` and `k3s_cert_manager_solver`
+(the DNS-01 stanza, which depends on your DNS provider — a commented
+Cloudflare example is in `group_vars/k3s_cluster.yml`). Use the Let's Encrypt
+staging directory while working a solver out; production allows five failed
+validations per hostname per hour.
+
+**Traefik dashboard**: `playbooks/k3s.yml` also applies `roles/k3s_traefik` —
+the only cluster-service role that installs nothing, since k3s installs
+Traefik itself. Out of the box the dashboard is reachable only by
+port-forward (`kubectl -n kube-system port-forward deploy/traefik 9000:9000`,
+then `http://127.0.0.1:9000/dashboard/` — trailing slash required; without
+this role a stock k3s 404s there). Publishing it on a hostname behind
+Authentik is one commented-out line, `k3s_traefik_dashboard_host` in
+`group_vars/k3s_cluster.yml`, plus a domain-level forward-auth Proxy Provider
+on the Authentik side. See "Key decisions" above and README.md → "Dashboard
+(Traefik)".
+
+**Apps**: `k3s_apps:` in `group_vars/k3s_cluster.yml`, deployed by
+`roles/k3s_app`. Authentik is the first and currently only one — it moved off
+a hand-made Unraid container rather than being deployed fresh, so
+`src/authentik/ansible/unraid/` is a README explaining why there's no compose
+file there. The cutover procedure is `docs/authentik-migration.md`; the part
+worth knowing without reading it is that the deployed `AUTHENTIK_VERSION`
+must match the version a restored database was dumped from, because
+Authentik's migrations run on startup and are one-way.

README.md

@@ -0,0 +1,966 @@
+# Homelab IaC
+
+Deploy apps as code across two platforms: Unraid server(s), where apps run as
+Docker Compose stacks, and Proxmox, where Terraform creates an LXC and the
+app is installed into it natively. Ansible is the deployment tool on both.
+Secrets are pulled from HashiCorp Vault at deploy time; CI/CD is Forgejo
+Actions once Forgejo itself is running.
+
+There's also a third piece of infrastructure alongside those two: a bare-metal
+k3s cluster (`homelab-utils`) across 4 Raspberry Pis, bootstrapped by Ansible
+too. It doesn't fit the `src/<app>/` table above the way Unraid/Proxmox apps
+do — Terraform never provisions it, and no `stacks:`/`apps:` list drives
+what's on it — so it gets its own inventory group and role instead. It does
+run shared services deployed via Helm, though: `k3s_monitoring`, `k3s_metallb`,
+and the cluster's own shared Postgres (`k3s_postgres`, a third platform for
+`src/shared/postgres/`). See `playbooks/k3s.yml` and "K3s (Raspberry Pi)"
+below.
+
+See `CLAUDE.md` for the short version of the architecture decisions.
+
+## Layout
+
+```
+src/
+  <app>/
+    common/vars.yml        config that holds on either platform
+    ansible/
+      unraid/              docker-compose.yml + .env.example + platform overrides
+                           (+ optional icon.png for the Unraid Docker page)
+      proxmox/             install.yml + config/systemd templates + platform overrides
+      kubernetes/          vars.yml + *.yaml.j2 manifests (normally a HelmChart CR)
+    terraform/             this app's LXC definition (Proxmox only)
+  shared/<service>/        same shape, for services multiple apps use
+build/config/
+  ansible/
+    inventory/
+      hosts.yml            static: unraid_servers, proxmox_nodes, k3s_cluster
+      proxmox.yml          dynamic: proxmox_guests, from the Proxmox API
+      group_vars/all.yml   Vault connection, Compose Manager project path
+      group_vars/k3s_*.yml k3s cluster name/version, per-role vars (server/agent)
+      host_vars/<host>.yml which stacks/apps deploy there
+                           (k3s apps are in group_vars/k3s_cluster.yml instead)
+    roles/compose_stack/   Unraid: sync compose + render .env + provision DB + up
+                           (deploy.yml / remove.yml, picked by the entry's state:)
+    roles/lxc_app/         Proxmox: install natively + render config + systemd
+    roles/k3s_app/         k3s: render app manifests + Secret + provision DB
+                           (deploy.yml / remove.yml, picked by the entry's state:)
+    roles/pve_backup/      Proxmox node: NAS backup storage + vzdump schedule
+    roles/k3s_node/        k3s Pi: cgroups/swap prep + install server or agent
+    roles/k3s_monitoring/  k3s control plane: lean kube-prometheus-stack via HelmChart CR
+    roles/k3s_metallb/     k3s control plane: MetalLB (LoadBalancer IPs) via HelmChart CR
+    roles/k3s_postgres/    k3s control plane: shared Postgres (CloudNativePG) via HelmChart CR
+    roles/k3s_cert_manager/ k3s control plane: cert-manager + ClusterIssuer via HelmChart CR
+    roles/unattended_upgrades/  any apt host: hands-off patching, no auto-reboot
+    roles/k3s_maintenance/ k3s Pi: drain + reboot-if-required + uncordon, one at a time
+    playbooks/
+      deploy.yml           everything (Unraid + Proxmox — not k3s, see below)
+      unraid.yml           just Unraid
+      proxmox.yml          just the Proxmox guests
+      pve_host.yml         just the Proxmox node (backups)
+      k3s.yml               bootstraps the k3s cluster (manual-only, not in deploy.yml)
+      k3s_maintenance.yml   rolls pending reboots across it, serial: 1 (manual-only)
+  terraform/               Proxmox provider, pg state backend, LXC module calls
+.forgejo/workflows/      CI/CD (needs a self-hosted runner, see below)
+docs/vault-secrets.md    Vault path/key reference
+docs/authentik-migration.md  hand-run cutover: Authentik from Unraid to k3s
+```
+
+Terraform's scope stops at the guest — it creates the LXC and nothing more,
+because the Proxmox provider has no way to install an application. Ansible
+picks up from there, which is why it's the common tool and Terraform is
+Proxmox-only.
+
+## How config is layered
+
+An app's settings are split by whether they survive a change of platform:
+
+- `src/<app>/common/vars.yml` — version, ports, domain, database name/user.
+- `src/<app>/ansible/<platform>/vars.yml` — the rest. Data paths differ
+  (`/mnt/user/appdata/forgejo` vs `/var/lib/forgejo`), and so does how the
+  database is reached (the `shared-postgres` Docker network alias only exists
+  on Unraid).
+
+Both roles merge them the same way: common, then platform overrides, then
+Vault secrets last. Nothing is restated between platforms.
+
+## How a deploy works
+
+**Unraid.** For each stack in a host's `stacks:` list, `compose_stack`:
+
+1. Copies `src/<stack>/ansible/unraid/docker-compose.yml` to the host's
+   Compose Manager project folder, unmodified — it's static and only
+   references `${VAR}`.
+2. Merges the two `vars.yml` layers with the stack's Vault KV path and
+   renders the project's `.env`. Nothing secret ever touches git.
+3. If the stack declares a `db:` block, ensures its database/role exist on
+   the shared Postgres instance (idempotent — safe to add new apps to the
+   shared service over time without touching existing data).
+4. Copies `icon.png`, if the stack ships one, to both the project folder and
+   `/mnt/user/appdata/icons/` (see "Icons on the Unraid Docker page" below).
+5. Runs `docker compose up -d` for the stack.
+
+**Proxmox.** `terraform apply` creates the LXC first. Then, for each app in a
+guest's `apps:` list, `lxc_app` does the same config merge and database
+provisioning, runs that app's `ansible/proxmox/install.yml`, and manages its
+systemd unit instead of a Compose project.
+
+**k3s.** For each app in `k3s_apps:` (`group_vars/k3s_cluster.yml`, not
+host_vars — an app belongs to the cluster, not a node), `k3s_app`:
+
+1. Does the same two-layer config merge, using
+   `src/<app>/ansible/kubernetes/vars.yml` as the platform layer.
+2. Provisions the app's database/role on the CloudNativePG cluster if it
+   declares a `db:` block — the same `community.postgresql` tasks the other
+   two platforms use, pointed at MetalLB's LoadBalancer address for the CNPG
+   primary.
+3. Renders the app's Vault path into a Kubernetes `Secret` (plus the
+   `Namespace`), keys passed through verbatim.
+4. Renders every `*.yaml.j2` the app ships into
+   `/var/lib/rancher/k3s/server/manifests/`, where k3s's own deploy and helm
+   controllers pick them up.
+
+Nothing here talks to the Kubernetes API — no `helm`, no kubeconfig, no
+`kubernetes.core`. Ansible writes files; k3s reconciles them. The consequence
+worth remembering: **a green run means the manifests landed, not that the
+workload came up.** Check with `kubectl -n <namespace> get pods`.
+
+Secrets land on disk in the rendered `.env` (root-readable, on the Unraid
+box) or, on k3s, in a 0600 manifest and then in etcd as an ordinary
+Kubernetes Secret — this is "fetch from Vault at deploy time," not a
+zero-secrets-at-rest model. Fine for a homelab; revisit if that changes.
+
+## Icons on the Unraid Docker page
+
+There are two icons per stack, and they come from completely different places.
+Commit **one** `src/<app>/ansible/unraid/icon.png` and the deploy feeds both.
+A stack running more than one container can commit
+`icon-<service>.png` alongside it for the extra services — see "More than one
+container" below.
+
+### The stack row
+
+Compose Manager draws the collapsible header row for the whole project, and
+serves its icon straight off disk from `icon.png` in the project folder —
+`/boot/config/plugins/compose.manager/projects/<stack>/icon.png`, the same
+directory the compose file and rendered `.env` land in. No label, no
+template, no URL: the file is there or the row has no icon. `.jpg`, `.gif`,
+`.svg` and an extensionless `icon` also work, since the plugin serves the file
+itself rather than handing a path to the webgui — but stick to PNG so the same
+file can serve the labels below.
+
+This is a feature of **Compose Manager Plus** (`mstrhakr/compose_plugin`), the
+maintained fork. The original Docker Compose Manager plugin (`dcflachs`,
+archived April 2026) has no icon support at all — on that one the copied file
+is simply ignored, so the deploy is safe either way, you just won't see it.
+
+### The containers under it
+
+Unraid's Docker page gets each *container's* icon, WebUI link and console
+shell out of the dockerMan template that created it, under
+`/boot/config/plugins/dockerMan/templates-user/`. A Compose stack never
+creates one, which is why compose-deployed containers show a question mark and
+no WebUI entry. Unraid 6.10+ falls back to Docker labels when there's no
+template, so each service sets them itself:
+
+```yaml
+labels:
+  net.unraid.docker.icon: ${STACK_ICON}
+  net.unraid.docker.webui: "http://[IP]:${FORGEJO_HTTP_PORT}/"
+  net.unraid.docker.shell: sh
+```
+
+They're per **service**, not per stack — a stack with two containers labels
+each one separately, and can point them at different icons.
+
+- **icon** — `${STACK_ICON}` is set by the role, not by hand. The same
+  committed `icon.png` is copied a second time to
+  `/mnt/user/appdata/icons/<stack>.png` and the label points there; ship no
+  `icon.png` and the variable renders empty, leaving the placeholder. To use a
+  hosted image instead, set `STACK_ICON` in the app's
+  `ansible/unraid/vars.yml` — it's merged over the role's value.
+
+  A second copy rather than reusing the project-folder one because this label
+  is a path the *webgui* resolves on every Docker page load, and that path
+  shouldn't run through the flash drive.
+
+  **PNG only here** — unlike the stack row, this goes through the webgui's own
+  icon handling, where SVG renders as the question-mark fallback and WebP
+  renders nothing at all. A remote URL is fetched when the page renders, so an
+  unreachable host breaks the icon — the reason a committed file is the
+  default.
+
+- **webui** — `[IP]` is substituted with the host's address. The port must be
+  the *published* one, so interpolate the stack's own port variable rather
+  than the container port. Omit the label entirely for something with no web
+  interface (as `shared/postgres` does) — an empty value still draws a WebUI
+  entry that goes nowhere.
+
+- **shell** — `sh` or `bash`, whichever the image actually has. Alpine-based
+  images (Forgejo) need `sh`; Debian-based ones (Postgres, Shelfarr) can take
+  `bash`.
+
+One caveat: editing a compose container's labels through the Unraid UI writes
+a template for it, and the template then wins over the labels until you clear
+it. Change these in the compose file and redeploy, not in the webgui.
+
+### More than one container
+
+`${STACK_ICON}` is a single value — the stack's `icon.png` — so a stack with
+two services would otherwise have to give both the same image. Since the
+labels are per-service, commit `icon-<service>.png` next to `icon.png` for
+each additional container, named for the **compose service**:
+
+```
+src/arr/ansible/unraid/
+  icon.png              → ${STACK_ICON}           → /mnt/user/appdata/icons/arr.png
+  icon-prowlarr.png     → ${STACK_ICON_PROWLARR}  → /mnt/user/appdata/icons/arr-prowlarr.png
+```
+
+The variable is the service name uppercased with `-` folded to `_`
+(`icon-shelfarr-libation.png` → `${STACK_ICON_SHELFARR_LIBATION}`). These get
+the appdata copy only — there is one Compose Manager stack row and `icon.png`
+already has it — and, like `STACK_ICON`, each can be overridden with a hosted
+URL by setting the same name in the app's `ansible/unraid/vars.yml`.
+
+The variable only exists when the file does. Interpolating
+`${STACK_ICON_FOO}` with no `icon-foo.png` committed gets Compose's
+unset-variable warning and the question-mark placeholder — so if the icon is
+meant to be a URL, set it in `vars.yml` rather than relying on the file.
+
+## Backups (Proxmox)
+
+Proxmox guests keep **all** their state on their own rootfs. Nothing is
+bind-mounted in from the NAS, which is the opposite of the obvious instinct —
+the reason is that `vzdump` deliberately excludes bind mounts, so a
+bind-mounted repository directory would be the one thing missing from the
+backup that was made to protect it. Keeping everything local means one archive
+is a complete copy of the guest's filesystem.
+
+`playbooks/pve_host.yml` (role: `pve_backup`) configures the off-box copy on
+the node: an NFS storage pointing at the NAS, and a scheduled `vzdump` job
+writing to it. Retention is the storage's `prune-backups` settings. Point it at
+a Proxmox Backup Server instead by setting `pve_backup_storage_type: pbs` in
+`host_vars/<node>.yml`.
+
+**Data that lives outside a container is that app's problem, not vzdump's.**
+Forgejo's database is on the shared Postgres over on Unraid, so restoring its
+container alone would give back every repository with no issues, pull requests,
+users or permissions. The fix is a `pg_dump` on a systemd timer *inside* the
+container, writing into its own filesystem — which puts the dump inside the
+same archive as the repositories it belongs to. That only works if it finishes
+before the backup window, so the two schedules are a pair:
+
+| | Set in | Default |
+|---|---|---|
+| Forgejo database dump | `src/forgejo/ansible/proxmox/vars.yml` → `FORGEJO_DB_DUMP_ONCALENDAR` | 01:30 |
+| vzdump job | `inventory/host_vars/pve.yml` → `pve_backup_schedule` | 02:00 |
+
+Any future app with external state should follow the same pattern. Restore
+steps are in `src/forgejo/ansible/proxmox/README.md`.
+
+None of this covers the Unraid side, which has its own backup arrangements
+outside this repo.
+
+## Prerequisites
+
+### Both platforms
+
+- Ansible control node needs to be Linux — **on Windows, run it from WSL2**,
+  not natively.
+- If this repo is checked out on a Windows mount (e.g. under `/mnt/c/...`,
+  which is where a Seafile-synced folder ends up) rather than the native WSL
+  filesystem, Ansible will warn that `ansible.cfg` is being ignored because
+  the directory looks world-writable — that's DrvFs reporting `777` on
+  everything, not a real permissions problem. Either `export
+  ANSIBLE_CONFIG="$(pwd)/ansible.cfg"` before running (from the
+  `build/config/ansible/` dir), or fix it permanently by adding to
+  `/etc/wsl.conf` inside WSL:
+  ```ini
+  [automount]
+  options = "metadata,umask=22,fmask=11"
+  ```
+  then `wsl --shutdown` from PowerShell and reopen the shell.
+- `pip install ansible hvac psycopg2-binary`
+- `ansible-galaxy collection install -r build/config/ansible/requirements.yml`
+- A reachable Vault instance, with secrets populated per
+  `docs/vault-secrets.md`.
+
+### Unraid
+
+- Root SSH enabled on the Unraid box(es) (Settings → Management Access).
+- Compose Manager (Docker Compose plugin) installed on Unraid — already
+  done.
+- **Python3 on each Unraid host itself**, not just the controller: the
+  `compose_stack` role's tasks (`file`, `copy`, `template`,
+  `docker_compose_v2`) run on the target, and Ansible modules — unlike
+  `raw`/ad hoc shell — need a Python interpreter there to execute at all.
+  Stock Unraid doesn't ship one. Installing it by hand into the running
+  system doesn't survive a reboot, same reason as the SSH-key gotcha above —
+  Unraid rebuilds its root filesystem from the flash drive on every boot.
+  Install the **NerdTools** plugin (Community Applications → search
+  "NerdTools", formerly "NerdPack") and enable `python3` in its package
+  list — it reinstalls whatever you've selected on every boot itself, same
+  mechanism Compose Manager already relies on. `ansible.cfg` already sets
+  `interpreter_python = auto_silent`, so Ansible finds it automatically
+  wherever NerdTools puts it; no path to hardcode. Verify after installing:
+  ```sh
+  ansible unraid_servers -m ping
+  ```
+
+### Proxmox
+
+- Terraform CLI (>= 1.6) on whatever runs `terraform apply`.
+- A Proxmox API token with rights to create containers, stored in Vault at
+  `homelab/ci/proxmox` (see `docs/vault-secrets.md`). Both the Terraform
+  provider and Ansible's dynamic inventory authenticate with it.
+- A `terraform_state` database and `terraform` role on the **CloudNativePG
+  cluster on k3s** (192.168.50.81), for `backend "pg"` — not on either shared
+  Postgres this repo deploys, which would be circular. See "Bootstrapping"
+  below.
+- Container templates downloaded on the Proxmox node for whatever OS the
+  LXCs are built from.
+- An NFS export on the NAS for backup archives, reachable from the Proxmox
+  node — set it in `inventory/host_vars/pve.yml`. See "Backups (Proxmox)".
+- Root SSH from the Ansible controller to the Proxmox node itself, not just
+  the guests: `playbooks/pve_host.yml` configures the node over SSH.
+- Unlike Unraid, nothing special is needed *inside* the guests: they're
+  ordinary Linux containers with Python already present, so Ansible works
+  without the NerdTools workaround above.
+
+### K3s (Raspberry Pi)
+
+- 4 Raspberry Pis running Ubuntu Server (64-bit), already imaged and on the
+  network at the addresses in `inventory/hosts.yml` → `k3s_cluster`.
+- An `ansible` user on each Pi with **NOPASSWD sudo** and this repo's k3s SSH
+  key installed as an authorized key — see "SSH access" below. Not automated
+  by this repo; create it by hand (or via cloud-init at image time) before the
+  first run of `playbooks/k3s.yml`.
+- `homelab/k3s-homelab-utils` → `K3S_TOKEN` set in Vault *before* the first
+  run — see `docs/vault-secrets.md`. Both the server and every agent read
+  this same fixed value rather than one generating it and handing it to the
+  other, which is what makes a rebuild reproducible.
+- Nothing else special: `python3` ships with Ubuntu Server, and the k3s
+  install script (`get.k3s.io`) handles containerd, the systemd unit, and
+  everything else that isn't Pi-specific. The one Pi-specific thing —
+  ensuring the memory cgroup controller is on — is handled by
+  `roles/k3s_node` itself, not a prerequisite here.
+
+#### Keeping it patched
+
+`playbooks/k3s.yml` also applies `roles/unattended_upgrades` to every node —
+apt updates install themselves on their own daily schedule, no login
+required. It sets `Unattended-Upgrade::Automatic-Reboot "false"` though, so a
+kernel or containerd update that needs a reboot to take effect just sits
+applied-but-inactive until one happens; blindly auto-rebooting a k3s node
+takes its pods down with no warning.
+
+`playbooks/k3s_maintenance.yml` is the other half: it checks
+`/var/run/reboot-required` on each node and, only where it's set, cordons and
+drains the node, reboots it, waits for it to report `Ready` again, then
+uncordons it — one node at a time (`serial: 1`), so patching the cluster never
+means dropping every workload at once. Manual-only for now, same as
+`playbooks/k3s.yml`:
+
+```sh
+ansible-playbook playbooks/k3s_maintenance.yml
+```
+
+Eventually this is meant to run on a schedule from a self-hosted Forgejo
+Actions runner rather than by hand — not wired up yet, see the header comment
+in the playbook.
+
+#### Metrics (Prometheus)
+
+`playbooks/k3s.yml` also deploys `roles/k3s_monitoring`: a lean
+kube-prometheus-stack (Prometheus + prometheus-operator + node-exporter +
+kube-state-metrics — no Grafana, no Alertmanager) so tools like OpenLens can
+show node/pod metrics. It's installed as a `HelmChart` custom resource
+dropped into k3s's own auto-deploying manifests directory
+(`/var/lib/rancher/k3s/server/manifests/`) rather than run through a `helm`
+binary — k3s's bundled helm-controller reconciles it the same way it
+installs its own Traefik and ServiceLB, so this needs no extra Ansible
+collection or kubeconfig on the controller. See
+`roles/k3s_monitoring/defaults/main.yml` for the resource sizing (tuned for
+a Raspberry Pi 4, not a real node) and why the control-plane component
+scrapers (`kubeControllerManager`, `kubeScheduler`, `kubeProxy`, `kubeEtcd`)
+are disabled — k3s doesn't expose those the way kube-prometheus-stack
+expects, so leaving them on just produces permanently-"down" targets.
+
+To point OpenLens at it: open the cluster's Settings → Metrics, set
+Prometheus to "Auto detect" or explicitly to the `Operator` provider — it
+should find the `kube-prometheus-stack-prometheus` service in the
+`monitoring` namespace via the API server proxy, the same path OpenLens
+already uses to reach the cluster, so nothing needs to be exposed outside
+it.
+
+#### LoadBalancer IPs (MetalLB)
+
+`playbooks/k3s.yml` also deploys `roles/k3s_metallb`: MetalLB in L2 mode,
+handing out real LAN IPs to `type: LoadBalancer` Services instead of the
+`ClusterIP`-only world k3s would otherwise leave homelab-utils apps in. It's
+installed the same way as monitoring — a `HelmChart` CR for k3s's
+helm-controller to reconcile — plus a plain `IPAddressPool`/`L2Advertisement`
+manifest dropped in the same directory; k3s's deploy controller applies both
+kinds of file and retries the config manifest until the HelmChart's CRDs
+exist.
+
+MetalLB replaces k3s's bundled ServiceLB (Klipper), it doesn't run alongside
+it — both would otherwise compete to satisfy the same Services. That's why
+`inventory/group_vars/k3s_cluster.yml` sets `k3s_server_extra_args:
+['--disable=servicelb']` — a separate, server-only var from `k3s_extra_args`,
+since `k3s agent` doesn't understand `--disable` and would fail to start if
+it were passed there too. The IP pool itself is
+`k3s_metallb_address_range` in the same file — a range on the cluster's LAN
+(`192.168.50.0/24`) that DHCP and every static assignment in
+`inventory/hosts.yml` steer clear of. There's no built-in default: the role
+fails fast if it's still empty.
+
+Enabling `--disable=servicelb` on an already-running cluster doesn't need a
+separate step: `roles/k3s_node` notices its exec line has changed and
+reinstalls (restarting just the `k3s` service, not the node) the next time
+`playbooks/k3s.yml` runs — see `CLAUDE.md` → "Key decisions".
+
+#### Postgres (CloudNativePG)
+
+`playbooks/k3s.yml` also deploys `roles/k3s_postgres`: CloudNativePG (CNPG),
+a Postgres operator, installed the same way as monitoring and MetalLB — a
+`HelmChart` CR for k3s's helm-controller to reconcile — plus a plain
+`Cluster` CR (the actual database) and a `Secret` (superuser credentials),
+dropped in the same auto-deploying manifests directory. This is a third
+platform for `src/shared/postgres/`, alongside the Unraid Compose stack and
+the Proxmox placeholder: `roles/k3s_postgres` layers
+`src/shared/postgres/common/vars.yml` with
+`src/shared/postgres/ansible/kubernetes/vars.yml` the same way
+`compose_stack`/`lxc_app` layer an app's config, and pulls the superuser
+password from the same `homelab/shared/postgres` Vault path those platforms
+already use (see `docs/vault-secrets.md`) — a separate physical instance,
+not a shared login across platforms, just the same identity.
+
+1 primary + 1 replica, kept off the control-plane Pi via `nodeAffinity` (see
+`roles/k3s_postgres/templates/postgres-cluster.yaml.j2`) — CNPG's own pod
+anti-affinity then spreads the two across the 3 worker Pis. Storage is k3s's
+default `local-path` StorageClass; that's node-local, but resilience here
+comes from CNPG's own streaming replication between instances, not from
+shared storage, so losing one instance's disk doesn't lose the data as long
+as the other instance is up. HA replication only for now — no
+`ScheduledBackup` to the NAS or anywhere else yet.
+
+To connect: `kubectl -n shared-postgres get svc` lists four Services. Three
+are CNPG's own, all `ClusterIP` — `shared-postgres-rw` (the current primary),
+`-ro` (replicas only) and `-r` (any instance). An app running on the cluster
+uses the first of those by DNS and needs nothing else:
+
+```
+shared-postgres-rw.shared-postgres.svc.cluster.local:5432
+```
+
+The fourth, `shared-postgres-lb`, is a `LoadBalancer` on
+`k3s_postgres_loadbalancer_ip` (`inventory/group_vars/k3s_cluster.yml` —
+inside the MetalLB pool above), which is how the cluster is reached from
+outside it:
+
+```sh
+psql -h 192.168.50.81 -U postgres          # password: homelab/shared/postgres
+```
+
+It's declared in the `Cluster` CR as a CNPG *managed service* rather than as
+a Service manifest of our own, so the operator keeps its selector pointed at
+whichever instance is currently primary — a hand-written Service would need
+re-pointing by hand after a failover. Credentials are the
+`shared-postgres-superuser-vault` Secret Ansible renders; CNPG separately
+auto-creates `shared-postgres-app` for the default `app` database it
+bootstraps, which nothing uses yet.
+
+The `-vault` suffix keeps that Secret clear of `<cluster>-superuser`, the
+name CNPG uses for the superuser secret it generates itself. Name our own
+object that and the operator treats it as one it already authored, so a
+rotated password reaches the Secret and never reaches the database — the
+Secret reads correctly, `psql` from the LAN keeps failing authentication, and
+only `select rolpassword is null from pg_authid where rolname='postgres'`
+inside the pod shows the disagreement.
+
+That LoadBalancer is what makes the controller-reachable `host:port` the
+`community.postgresql` tasks need — the same ones `compose_stack` and
+`lxc_app` use to create an app's database and role — available here too, and
+`roles/k3s_app` now uses exactly those tasks for any app declaring a `db:`
+block.
+
+Note the deliberate asymmetry, which reads like a mistake until you know why:
+provisioning uses `192.168.50.81` (the LoadBalancer) because it runs on the
+Ansible controller, which is off-cluster and can't route to a ClusterIP;
+apps use `shared-postgres-rw.shared-postgres.svc.cluster.local` because
+sending pod traffic out to the LAN and back would put MetalLB's L2 speaker in
+the path of every query for nothing. Two addresses, one database, each
+correct for its caller.
+
+Without `kubectl` to hand, or if MetalLB is having a bad day, the in-cluster
+routes still work:
+
+```sh
+kubectl -n shared-postgres port-forward svc/shared-postgres-rw 5432:5432
+kubectl -n shared-postgres exec -it shared-postgres-1 -- psql -U postgres
+```
+
+**No backups.** HA replication only — a streaming replica on another Pi,
+which covers a dead SD card and not a dropped table, a bad migration, or a
+mistyped `DROP`. This mattered less when the cluster held nothing; it matters
+now that Authentik's database lives here rather than on Unraid. A CNPG
+`ScheduledBackup` to the NAS is the obvious next piece of work and isn't
+built yet.
+
+#### TLS (cert-manager)
+
+`playbooks/k3s.yml` also applies `roles/k3s_cert_manager`: cert-manager plus
+one `ClusterIssuer` named `letsencrypt`, both via the usual `HelmChart` CR +
+plain manifest pair. An app's Ingress opts in with a single annotation —
+
+```yaml
+annotations:
+  cert-manager.io/cluster-issuer: letsencrypt
+```
+
+— and cert-manager creates and renews the certificate on its own.
+
+Two values have no default and the role refuses to run without them, both in
+`inventory/group_vars/k3s_cluster.yml`:
+
+- `k3s_cert_manager_acme_email` — an ACME account is registered against it.
+- `k3s_cert_manager_solver` — the DNS-01 stanza for your DNS provider,
+  rendered into the issuer as-is. A commented Cloudflare example is in that
+  file; for anything else take the stanza from
+  [cert-manager's docs](https://cert-manager.io/docs/configuration/acme/dns01/).
+  Its API token goes in Vault at `homelab/k3s-cert-manager`.
+
+DNS-01 rather than HTTP-01 because HTTP-01 needs Let's Encrypt to reach this
+cluster from the internet on port 80, which it can't. That has a useful
+consequence: a certificate can be issued **before** DNS points at the
+cluster, so a migration's TLS is settled before its cutover rather than
+after.
+
+While working out a solver, point `k3s_cert_manager_acme_server` at Let's
+Encrypt staging — production allows five failed validations per hostname per
+hour, and exhausting it means waiting, not retrying. Check it registered:
+
+```sh
+kubectl get clusterissuer letsencrypt -o jsonpath='{.status.conditions[*].message}'
+```
+
+#### Dashboard (Traefik)
+
+Traefik is the one chart here this repo doesn't install — k3s installs it
+itself. `roles/k3s_traefik` only adjusts it, through a `HelmChartConfig`
+merged over k3s's own `HelmChart`. Editing
+`/var/lib/rancher/k3s/server/manifests/traefik.yaml` on the node instead
+works until the next server restart rewrites it.
+
+A stock k3s answers **404** on the dashboard, which looks like a broken
+install and isn't: Traefik still builds the dashboard, but the Traefik chart
+stopped creating the router that reaches it in v28. The role puts that router
+back on Traefik's internal `traefik` entrypoint, which isn't published on the
+Service — so it's reachable by port-forward and nothing else:
+
+```sh
+kubectl -n kube-system port-forward deploy/traefik 9000:9000
+# then http://127.0.0.1:9000/dashboard/  — the trailing slash is required
+```
+
+Two things to check if that still 404s. `curl -i http://127.0.0.1:9000/ping`
+returning 200 means the port-forward is fine and only the router is missing
+(so the role hasn't run, or its `HelmChartConfig` hasn't reconciled yet —
+`kubectl -n kube-system get ingressroute` should list `traefik-dashboard`).
+And forward to `deploy/traefik`, not `svc/traefik`: the Service only publishes
+80/443, so going through it lands you on the `web` entrypoint, where an
+unmatched request also returns 404.
+
+**Publishing it on a hostname** is one commented line in
+`inventory/group_vars/k3s_cluster.yml`:
+
+```yaml
+k3s_traefik_dashboard_host: traefik.turtlesystems.uk
+```
+
+That renders an `IngressRoute` on `websecure` with a cert-manager certificate
+and an Authentik forward-auth middleware in front. `k3s_traefik_dashboard_auth_address`
+must be set too — the role refuses to run otherwise rather than publishing an
+unauthenticated view of every router, service and middleware on the cluster.
+It's already set in that file, so in practice this is the single line above
+plus a DNS record pointing at Traefik's MetalLB address
+(`kubectl -n kube-system get svc traefik`).
+
+On the Authentik side, first: a Proxy Provider in **forward auth (domain
+level)** mode, assigned to an application, added to the embedded outpost.
+Domain level rather than single-application because the latter needs
+`/outpost.goauthentik.io/` routed to Authentik on the dashboard's own
+hostname, which from `kube-system` is a cross-namespace service reference
+Traefik rejects by default.
+
+An `IngressRoute` rather than an `Ingress`, unlike every app here, because
+the dashboard is served by Traefik's internal `api@internal` and has no
+Kubernetes Service for an `Ingress` to point at. That's also why its
+certificate is an explicit `Certificate` resource — cert-manager watches
+`Ingress`, not `IngressRoute`.
+
+Clearing the hostname again is a real teardown: the role deletes the
+manifest, and k3s's deploy controller garbage-collects the route, middleware
+and certificate it created.
+
+The port-forward stays enabled alongside the published route on purpose. Once
+the dashboard is behind Authentik it depends on Authentik, which depends on
+CNPG, which depends on a healthy cluster — the things you'd open the
+dashboard to diagnose. Port-forward talks to the pod and traverses none of
+it, so it's the break-glass path, not a leftover.
+
+#### Apps on the cluster
+
+Apps go in `k3s_apps:` in `inventory/group_vars/k3s_cluster.yml` — the k3s
+equivalent of a host's `stacks:`/`apps:` list, in group_vars because an app
+is deployed to the cluster rather than to a node. Same entry shape as the
+other platforms (name, src, vault_path, optional `db:`, optional `state:`).
+
+```sh
+ansible-playbook playbooks/k3s.yml --tags apps
+ansible-playbook playbooks/k3s.yml --tags apps -e only_apps=authentik
+```
+
+Ingress goes through the Traefik k3s already bundles, on the MetalLB address
+its Service holds (`kubectl -n kube-system get svc traefik`) — so adding an
+app costs a DNS record pointed at that one address, not a pool IP each.
+
+Removal is `state: absent` on the entry, same tombstone rule as Unraid: keep
+the entry, don't delete it, or nothing is torn down and the workload keeps
+running unmanaged. Unlike Unraid it's a single pass — deleting the manifests
+*is* the teardown, because k3s's deploy controller garbage-collects what each
+file created — so `src/<app>/` can go in the same commit. The database, PVCs
+and Vault path deliberately survive.
+
+**Authentik** is currently the only app, and it got here by migrating off a
+hand-made Unraid container rather than being deployed fresh. If you're
+repeating that for something else, `docs/authentik-migration.md` is the
+worked procedure; the part worth knowing up front is that the deployed
+version must match the version a restored database was dumped from, because
+Authentik runs its migrations on startup and they don't go backwards.
+
+## SSH access
+
+Ansible connects as `ansible_user: root` (set in
+`build/config/ansible/inventory/hosts.yml`) over SSH. Password auth would mean either an interactive prompt every run —
+which doesn't work from CI at all — or a plaintext password somewhere, so
+use a dedicated key pair instead:
+
+```sh
+ssh-keygen -t ed25519 -f ~/.ssh/unraid_ansible -C "ansible@unraid-iac" -N ""
+```
+
+Install the public half as an authorized key on **every** Unraid host this
+repo targets. Don't just `ssh-copy-id` it into `~/.ssh/authorized_keys` —
+Unraid boots from the flash drive and `/root` lives on a RAM-backed overlay,
+so anything written there is gone on the next reboot. Persist it through
+`/boot/config/ssh/root.authorized_keys` instead, which Unraid copies into
+place at every boot:
+
+```sh
+cat ~/.ssh/unraid_ansible.pub | ssh root@<unraid-host> \
+  'cat >> /boot/config/ssh/root.authorized_keys'
+```
+
+For manual/bootstrap runs, load the key into `ssh-agent` and Ansible picks it
+up automatically — no config file changes needed:
+
+```sh
+eval "$(ssh-agent -s)"
+ssh-add ~/.ssh/unraid_ansible
+```
+
+For CI, rather than adding yet another place secrets live, the private key
+is stored in Vault (`homelab/ci/ssh`, see `docs/vault-secrets.md`) and the
+workflow fetches it at the start of each run:
+
+```sh
+vault kv put kv/homelab/ci/ssh PRIVATE_KEY=@~/.ssh/unraid_ansible
+```
+
+**K3s Pis** work the same way but as a non-root `ansible` user, not root, and
+with their own key pair (`~/.ssh/k3s_ansible`, Vault path
+`homelab/ci/ssh-k3s`) — see `docs/vault-secrets.md` for the full commands.
+Three differences from the Unraid steps above. The public key goes in the
+ordinary `~/.ssh/authorized_keys` for the `ansible` user (Ubuntu's root
+filesystem isn't rebuilt from flash on every boot, so there's no persistence
+quirk to work around), and that user needs NOPASSWD sudo configured in
+`/etc/sudoers.d/` — `playbooks/k3s.yml` installs k3s via `become: true`.
+
+And the private half doesn't need loading into `ssh-agent` at all, even for a
+manual run: `playbooks/k3s_ssh_key.yml` fetches it from Vault and writes it to
+`local/k3s/homelab-utils.key`, and both `playbooks/k3s.yml` and
+`playbooks/k3s_maintenance.yml` import that as their first play. So a k3s run
+needs `VAULT_ADDR`/`VAULT_TOKEN` in the environment and nothing else — the
+same "Vault is the only place secrets live" arrangement CI already uses for
+the Unraid key, rather than a second setup step to remember. To fall back to a
+local copy (Vault down, say), skip the fetch **and** point at the key —
+overriding the path alone isn't enough, since the fetch play would still fail
+before anything else ran:
+
+```sh
+ansible-playbook playbooks/k3s.yml \
+  -e k3s_ssh_key_fetch=false \
+  -e ansible_ssh_private_key_file=~/.ssh/k3s_ansible
+```
+
+## Bootstrapping (chicken-and-egg on the very first deploy)
+
+CI/CD runs on Forgejo Actions — but Forgejo doesn't exist yet on the first
+run, and Actions needs a self-hosted runner besides. So the first deploy is
+manual:
+
+1. Fill in `build/config/ansible/inventory/hosts.yml` with nas2's real
+   address.
+2. Set up the SSH key pair per "SSH access" above and install the public key
+   on nas2 (and nas1, once you migrate anything to it).
+3. `vault kv put` the secrets in `docs/vault-secrets.md` for
+   `homelab/shared/postgres`, `homelab/forgejo`, and `homelab/ci/ssh`.
+4. From WSL, with `VAULT_ADDR` / `VAULT_TOKEN` exported (`vault login` first)
+   and the key loaded in `ssh-agent`:
+   ```sh
+   cd build/config/ansible
+   ansible-playbook playbooks/unraid.yml --limit nas2
+   ```
+   This brings up shared Postgres, provisions the `forgejo` database/role,
+   and brings up Forgejo. Use `unraid.yml` rather than `deploy.yml` at this
+   point — no Proxmox guests exist yet, so the Proxmox play has nothing to
+   target.
+5. Log into Forgejo, create this repo there, push it.
+6. Register a self-hosted Forgejo Actions runner reachable to Vault and to
+   both Unraid hosts (labelled `unraid-deploy` — see
+   `.forgejo/workflows/deploy.yml`). Give it an AppRole (`VAULT_ROLE_ID` /
+   `VAULT_SECRET_ID`) scoped to read the `kv/homelab/*` paths it needs
+   (including `homelab/ci/ssh`), stored as Forgejo Actions secrets alongside
+   `VAULT_ADDR`.
+7. From then on, pushes to `main` touching `src/**` or `build/config/ansible/**`
+   deploy automatically — and only the stack(s) whose `src/<stack>/` folder
+   actually changed (see "Deploying one platform, one host, one stack"
+   below); a change under `build/config/ansible/` still triggers a full
+   deploy of every stack on every host, since that's a change to how *all* of
+   them get deployed.
+
+### Bringing Proxmox online later
+
+The shared Postgres and its LXC are the first thing to stand up here, and the
+ordering is fiddly enough to have its own runbook: **`docs/postgres-proxmox.md`**.
+The short version, and the reason it isn't just "run Terraform":
+
+Terraform stores its state in Postgres, so provisioning the Postgres LXC with
+Terraform would be circular. It's broken by keeping state on the
+CloudNativePG cluster on the k3s Pis instead — something this configuration
+has no hand in building — which means `terraform apply` now depends on the
+cluster being up. That's a cross-platform dependency the rest of the repo
+avoids, and it's worth it because the alternative is a local-state-then-migrate
+dance that has to be got right exactly once.
+
+After that, `docs/postgres-proxmox.md` walks through the template download,
+`terraform apply -target=module.postgres`, checking the ZFS replication job
+actually exists, installing Postgres with
+`ansible-playbook playbooks/proxmox.yml -e only_stacks=postgres`, and
+`playbooks/pve_host.yml` for node backups. It also covers failing over between
+the two nodes and repointing clients afterwards.
+
+Forgejo comes after all of that, on **turtle-proxmox-02** — its database is
+the shared Postgres the runbook above creates, so none of it works until that
+one is finished. `docs/forgejo-proxmox.md` walks through the extra Vault key
+the native install needs, `terraform apply -target=module.forgejo`,
+`ansible-playbook playbooks/proxmox.yml -e only_stacks=forgejo`, and the parts
+Ansible deliberately does not do — moving repository data, and the DNS/SSH-port
+cutover.
+
+## Deploying one platform, one host, one stack
+
+`playbooks/deploy.yml` runs everything; `playbooks/unraid.yml`,
+`playbooks/proxmox.yml` and `playbooks/pve_host.yml` do one slice each
+(Unraid stacks, Proxmox guests, and the Proxmox node's backup config
+respectively). Within any of them, `--limit <host>`
+narrows by host and `-e only_stacks=<name>[,<name>...]` narrows by stack or
+app (name = the `name:` in the host's `stacks:`/`apps:` list, e.g. `forgejo`
+or `postgres` — not the `src:` path):
+
+```sh
+# everything, both platforms
+ansible-playbook playbooks/deploy.yml
+
+# everything declared for nas2
+ansible-playbook playbooks/unraid.yml --limit nas2
+
+# just forgejo on nas2
+ansible-playbook playbooks/unraid.yml --limit nas2 -e only_stacks=forgejo
+
+# forgejo and its database, skipping any other stack on nas2
+ansible-playbook playbooks/unraid.yml --limit nas2 -e only_stacks=forgejo,postgres
+```
+
+Note `only_stacks` doesn't resolve dependencies for you — if you limit to
+`forgejo` before `postgres` has ever been deployed, the DB provisioning step
+will fail because there's nothing to connect to. CI doesn't hit this: it
+computes `only_stacks` from which `src/**` paths actually changed in the
+push, so an unrelated app's push never touches Postgres or Forgejo.
+
+`deploy.yml` runs Unraid before Proxmox by convention rather than necessity —
+the two platforms no longer depend on each other. Proxmox-side apps provision
+their databases against the Proxmox shared Postgres (192.168.50.54), and
+Terraform keeps its state on the CloudNativePG cluster on k3s; neither goes
+through Unraid. The node's backup config goes last, since a vzdump job pinned
+to specific VMIDs needs those guests to exist. Note that `only_stacks` has no
+meaning for `pve_host.yml` — it configures a node, not an app, and ignores it.
+
+`playbooks/k3s.yml` is separate from all of the above — it isn't included in
+`deploy.yml` and doesn't take `only_stacks`, because a cluster service there
+isn't a `stacks:` entry to filter down to. It has two selectors of its own
+instead: `--limit` narrows *which nodes*, and `--tags` narrows *which piece*.
+
+```sh
+# the whole cluster: nodes, then the cluster services, then the kubeconfig
+ansible-playbook playbooks/k3s.yml
+
+# re-converge one worker, e.g. after reimaging it
+ansible-playbook playbooks/k3s.yml --limit k3s-node-02
+
+# just one cluster service, no node install across 4 Pis
+ansible-playbook playbooks/k3s.yml --tags postgres
+
+# every cluster service, still no node install
+ansible-playbook playbooks/k3s.yml --tags services
+```
+
+The tags, one per play (see the playbook's header comment): `nodes` (both
+node plays), `metallb`, `monitoring`, `postgres`, `cert-manager`, `traefik`,
+`services` (all five of those together), `apps`, `upgrades`, `kubeconfig`.
+`services` and `apps` don't imply each other, so "everything except
+reinstalling k3s" is `--tags services,apps`. The SSH-key fetch play is tagged
+`always` rather than named, so it survives every filter — it's what any of
+the others connect with, not something you'd pick.
+
+Deploying a single service this way skips the `kubeconfig` play too, so
+`local/`'s copy isn't refreshed. That only matters on a cluster rebuild;
+add `--tags postgres,kubeconfig` if you want both.
+
+Worth knowing what a tagged run does and doesn't tell you: every service role
+just templates manifests into `/var/lib/rancher/k3s/server/manifests/`
+for k3s's own controllers to reconcile, so the playbook finishing means the
+files landed, not that the workload is up. Watch that separately, e.g.
+`kubectl -n shared-postgres get cluster,pods -w`.
+
+`playbooks/k3s_maintenance.yml` is separate again — see "Keeping it patched"
+above. It ignores `--limit` grouping in one sense worth knowing: `serial: 1`
+still applies to whatever `--limit` narrows the run to, so limiting to two
+nodes still patches them one at a time, not together.
+
+## Adding a new app
+
+1. `src/<app>/common/vars.yml` — `env_defaults:` for the config that doesn't
+   depend on where it runs.
+2. Build the platform side(s) you need:
+
+   **Unraid** — `src/<app>/ansible/unraid/` containing
+   `docker-compose.yml` (static, `${VAR}`-driven, joining `unraid_shared`
+   with `external: true` if it needs the shared Postgres), `vars.yml` of
+   overrides, and `.env.example` documenting every var with secrets blank.
+   Optionally an `icon.png` and the `net.unraid.docker.*` labels, so the
+   Docker page shows something other than a question mark.
+
+   **Proxmox** — `src/<app>/ansible/proxmox/` containing `vars.yml`,
+   `install.yml`, and templates for the app's config file and systemd unit;
+   plus `src/<app>/terraform/` defining its LXC, called from
+   `build/config/terraform/main.tf`.
+
+3. Add its `homelab/<app>` path to Vault (`docs/vault-secrets.md`).
+4. Declare it on the target host — `stacks:` in an Unraid host's
+   `host_vars/<host>.yml`, or `apps:` in a Proxmox guest's. Same entry shape
+   either way (name, src, vault_path, optional `db:` block).
+5. Push — or run the relevant playbook manually before CI exists.
+6. Leave the platform you didn't build as a README placeholder rather than
+   deleting the folder.
+
+## Removing an app (Unraid)
+
+**Deleting the entry from `stacks:` does not remove anything.** Ansible keeps
+no record of what it deployed last run, so an app that disappears from the
+list is simply never visited again — its containers keep running on the host,
+now unmanaged and invisible to the playbook. This is the one place the Unraid
+side doesn't behave like the Terraform side, where deleting a module call does
+destroy the LXC.
+
+Removal is therefore an instruction, not an absence. Set `state: absent` on
+the entry and leave it in place:
+
+```yaml
+stacks:
+  - name: arr
+    src: arr
+    vault_path: homelab/arr
+    state: absent
+```
+
+The next run of `playbooks/unraid.yml` runs `docker compose down`, then
+deletes the project folder (and with it the rendered `.env`). Because a
+`host_vars/` edit is a change under `build/config/ansible/`, CI treats it as a
+full converge — so pushing that change is enough to action the teardown.
+
+Keep the entry as a tombstone until you're sure. It's the only record that
+the app was deliberately removed rather than never deployed, and flipping
+`state:` back to `present` redeploys it.
+
+### What survives, and how to remove the rest
+
+Bind mounts are untouched, so `/mnt/user/appdata/<app>` — which for a
+SQLite-backed app like Shelfarr *is* the application — survives a teardown and
+makes it reversible. External networks (`caddy-net`, `unraid_shared`) are left
+alone too; they belong to Unraid or to another stack. Both copies of the
+stack's icon *are* deleted without needing to be asked — the project-folder
+one goes with the folder, and the one under `/mnt/user/appdata/icons/` is
+removed explicitly. Neither is state; they're copies of a file in the repo,
+and a redeploy puts them back.
+
+Three things are destructive enough to stay opt-in, per entry:
+
+| Key | Effect |
+|---|---|
+| `remove_volumes: true` | Also delete the project's named volumes. Bind mounts are unaffected either way. |
+| `remove_images: local` | Also delete its images (`local` or `all`, as `docker compose down --rmi`). |
+| `remove_database: true` | Drop the Postgres database **and** its role. Only for stacks with a `db:` block. |
+
+Nothing removes the app's `homelab/<app>` path from Vault — that has its own
+lifecycle, and destroying it would make the teardown irreversible. Delete it
+by hand with `vault kv metadata delete` once you're finished with the app.
+
+### Order of operations
+
+`docker compose down` reads the compose file to know what it's removing, so
+the teardown has to run **before** you delete `src/<app>/` or the host's
+project folder. Delete those first and there's nothing left to tell Docker
+what belonged to the project — you're cleaning up by hand with `docker rm` and
+`docker network rm` instead. So: set `state: absent`, run the playbook, then
+delete files in a second pass.
+
+There's no equivalent on the Proxmox side yet. `lxc_app` only ever installs,
+and a symmetric teardown needs a per-app `uninstall.yml` convention that no
+app implements — the role is still a skeleton. For now, removing a Proxmox app
+means stopping and disabling its systemd unit by hand, or destroying the
+container via Terraform.
+
+## Adding a host
+
+**Unraid:** add it under `unraid_servers` in
+`build/config/ansible/inventory/hosts.yml`, then create
+`build/config/ansible/inventory/host_vars/<name>.yml` with its own `stacks:`
+list — each host only runs what it's assigned.
+
+**Proxmox:** define the LXC as a module in `src/<app>/terraform/`, call it
+from `build/config/terraform/main.tf`, and `terraform apply`. The guest
+appears in inventory automatically via the API — no `hosts.yml` edit — but
+it still needs a `host_vars/<guest>.yml` declaring its `apps:` list.
+
+**K3s:** add it as a new host under `k3s_control_plane` or `k3s_workers` (per
+`inventory/hosts.yml` → `k3s_cluster`) — no `terraform apply`, the Pi has to
+physically exist and have SSH access set up first (see "K3s (Raspberry Pi)"
+above). No `host_vars/` entry needed beyond that: `roles/k3s_node` reads
+`k3s_node_role` off the group, not the host, so which group a new Pi joins is
+the only thing that decides whether it becomes another agent or a second
+control-plane node — and this repo's role doesn't support the latter (single
+server, no HA etcd) without changes to `k3s_node/tasks/server.yml`.

build/config/ansible/ansible.cfg

@@ -0,0 +1,18 @@
+[defaults]
+# A directory, not a single file: static Unraid hosts (hosts.yml) and the
+# Proxmox API-backed dynamic inventory (proxmox.yml) are both sources.
+inventory = inventory/
+roles_path = roles
+host_key_checking = False
+retry_files_enabled = False
+interpreter_python = auto_silent
+
+[inventory]
+# `auto` must come before `yaml`: it reads the `plugin:` key in
+# inventory/proxmox.yml and dispatches to the named plugin. With `yaml`
+# first, that file gets parsed as a static host list instead, and `plugin:`
+# is read as a group name.
+enable_plugins = auto, yaml, ini, community.proxmox.proxmox
+
+[ssh_connection]
+pipelining = True

build/config/ansible/inventory/group_vars/all.yml

@@ -0,0 +1,18 @@
+---
+# HashiCorp Vault connection. VAULT_ADDR/VAULT_TOKEN (or VAULT_ROLE_ID /
+# VAULT_SECRET_ID for AppRole) are read from the environment of whatever is
+# running ansible-playbook — a local shell during bootstrap, or Forgejo
+# Actions secrets once CI is wired up. Never hardcode a token here.
+vault_addr: "{{ lookup('env', 'VAULT_ADDR') }}"
+vault_kv_mount: kv
+
+# AppRole auth (used by CI). Leave unset for local token-based auth
+# (`vault login` + VAULT_TOKEN env var) during bootstrap.
+vault_auth_method: "{{ lookup('env', 'VAULT_AUTH_METHOD') | default('token', true) }}"
+vault_role_id: "{{ lookup('env', 'VAULT_ROLE_ID') | default(omit) }}"
+vault_secret_id: "{{ lookup('env', 'VAULT_SECRET_ID') | default(omit) }}"
+
+# Where Compose Manager (the Unraid Docker Compose plugin) expects project
+# folders. Verify this against your plugin version — override per-host in
+# host_vars if yours differs.
+compose_projects_root: /boot/config/plugins/compose.manager/projects

build/config/ansible/inventory/group_vars/k3s_cluster.yml

@@ -0,0 +1,205 @@
+---
+# Shared config for the whole k3s_cluster group (both k3s_control_plane and
+# k3s_workers) — see roles/k3s_node and playbooks/k3s.yml.
+
+# Used as the kubeconfig context/cluster name and as the Vault path suffix
+# below, so the cluster has one name in every place it shows up.
+k3s_cluster_name: homelab-utils
+
+# Pinned, not "latest" — a rebuild months from now should produce the same
+# cluster it does today, same reasoning as pinning an app's image tag in
+# common/vars.yml. Bump deliberately; check the current stable release at
+# https://github.com/k3s-io/k3s/releases first.
+k3s_version: v1.31.4+k3s1
+
+# One Vault path for the whole cluster, same "one path per stack" convention
+# as everything under homelab/<app> — see docs/vault-secrets.md.
+k3s_vault_path: "homelab/k3s-{{ k3s_cluster_name }}"
+
+# The SSH key this group is reached with. A different path from k3s_vault_path
+# above because it's a different kind of secret: that one is cluster identity
+# (the join token), this one is access to the hosts, and it lives alongside
+# the Unraid key under homelab/ci/ rather than with the cluster's own data.
+# Fetched by playbooks/k3s_ssh_key.yml, which both k3s playbooks import.
+k3s_ssh_key_vault_path: homelab/ci/ssh-k3s
+
+# Where that play writes it, and what every play here then connects with — so
+# a manual run needs Vault credentials in the environment and nothing else,
+# rather than the key pre-loaded into ssh-agent. Override on the command line
+# (-e ansible_ssh_private_key_file=~/.ssh/k3s_ansible) to use a local copy
+# instead, e.g. if Vault itself is down.
+#
+# Under local/, which .gitignore already excludes — same as the kubeconfig
+# below, and for the same reason: generated credential, never committed. Built
+# from playbook_dir rather than repo_root, unlike k3s_kubeconfig_local_path:
+# repo_root is a *play* var set in k3s.yml, so it only resolves inside the two
+# plays that set it, whereas this is a connection var and has to resolve in
+# every play in the file.
+k3s_ssh_key_local_path: "{{ playbook_dir }}/../../../../local/k3s/{{ k3s_cluster_name }}.key"
+ansible_ssh_private_key_file: "{{ k3s_ssh_key_local_path }}"
+
+# Extra INSTALL_K3S_EXEC args appended on every node, both roles. Empty by
+# default. Only flags valid on *both* `k3s server` and `k3s agent` belong
+# here — see k3s_server_extra_args below for server-only flags like
+# `--disable`, which `k3s agent` doesn't understand and will fail to start
+# with.
+k3s_extra_args: []
+
+# Extra INSTALL_K3S_EXEC args appended on the control-plane node only.
+# --disable=servicelb turns off k3s's bundled LoadBalancer controller
+# (Klipper) in favour of roles/k3s_metallb — the two would otherwise both
+# try to satisfy the same LoadBalancer Services. Add e.g. '--disable=traefik'
+# here too if homelab-utils apps end up wanting their own ingress instead.
+#
+# Changing this (or k3s_extra_args) on an already-installed cluster is picked
+# up on the next `ansible-playbook playbooks/k3s.yml` run — roles/k3s_node
+# tracks the exec line it last installed with and reinstalls (restarting the
+# k3s/k3s-agent service, not the node) whenever it drifts from what's
+# configured here. See tasks/server.yml and tasks/agent.yml.
+k3s_server_extra_args:
+  - "--disable=servicelb"
+
+# MetalLB's LoadBalancer IP pool — see roles/k3s_metallb. Must be addresses
+# on the LAN (192.168.50.0/24 — see inventory/hosts.yml) that nothing else,
+# DHCP included, will ever hand out. Adjust to a range your DHCP scope
+# doesn't cover before the first run.
+k3s_metallb_address_range: "192.168.50.80-192.168.50.99"
+
+# The LAN address the shared Postgres primary answers on — see
+# roles/k3s_postgres, which declares a LoadBalancer Service for it alongside
+# CNPG's built-in ClusterIP ones. Pinned rather than auto-assigned because
+# the address gets written down elsewhere (an app's vars.yml, a connection
+# string in Vault) instead of only being looked up at runtime, so it mustn't
+# move when the Service is recreated.
+#
+# Must be inside the pool above — MetalLB only assigns from its own pools,
+# and a request for an address outside them leaves the Service pending
+# forever rather than failing loudly. The caveat of pinning out of the same
+# pool MetalLB auto-assigns from: it hands unpinned Services the lowest free
+# address, so if some other LoadBalancer Service is created before this one
+# and takes .80, this Service is the one left pending. Nothing else here
+# requests an IP today. If that changes, the fix is a second
+# IPAddressPool with autoAssign: false in roles/k3s_metallb reserved for
+# pinned addresses, not a different number here.
+k3s_postgres_loadbalancer_ip: "192.168.50.81"
+
+# Where playbooks/k3s.yml fetches the kubeconfig to, on the Ansible
+# controller. Under local/, which .gitignore already excludes — it's a
+# generated credential, never committed.
+k3s_kubeconfig_local_path: "{{ repo_root }}/local/k3s/{{ k3s_cluster_name }}.kubeconfig"
+
+# --- cert-manager (roles/k3s_cert_manager) ----------------------------------
+#
+# TLS for anything this cluster serves over Traefik. See that role's
+# defaults/main.yml for why DNS-01 rather than HTTP-01, and what each of
+# these does.
+
+# Where Let's Encrypt sends expiry warnings. An ACME account is registered
+# against it.
+k3s_cert_manager_acme_email: russell.seymour@turtlesystems.co.uk
+
+# >>> SET BEFORE THE FIRST RUN <<<
+#
+# How cert-manager proves control of the domain, rendered into the
+# ClusterIssuer's solvers list as-is. Left empty deliberately — it depends on
+# who runs DNS for turtlesystems.uk, which this repo has no way to know.
+# roles/k3s_cert_manager fails fast while it's empty, rather than creating an
+# issuer that can never satisfy an order.
+#
+# Cloudflare, as the most common case — the token needs Zone:DNS:Edit on the
+# zone, and is stored in Vault (see k3s_cert_manager_vault_path below and
+# docs/vault-secrets.md) rather than written here:
+#
+#   k3s_cert_manager_solver:
+#     dns01:
+#       cloudflare:
+#         apiTokenSecretRef:
+#           name: cert-manager-dns-credentials
+#           key: CLOUDFLARE_API_TOKEN
+#     selector:
+#       dnsZones:
+#         - turtlesystems.uk
+#
+# For another provider, take the stanza from
+# https://cert-manager.io/docs/configuration/acme/dns01/ and point its secret
+# ref at whichever key you stored at that Vault path.
+#
+# Use the Let's Encrypt staging directory while working this out — override
+# k3s_cert_manager_acme_server. Production allows 5 failed validations per
+# hostname per hour, and exhausting it means waiting, not retrying.
+k3s_cert_manager_solver:
+  dns01:
+    cloudflare:
+      apiTokenSecretRef:
+        name: cert-manager-dns-credentials
+        key: CLOUDFLARE_API_TOKEN
+  selector:
+    dnsZones:
+      - turtlesystems.uk
+
+# --- Traefik (roles/k3s_traefik) --------------------------------------------
+#
+# The ingress controller k3s installs for itself; that role installs nothing,
+# it only adjusts what's already there. Its defaults are enough for the
+# dashboard over `kubectl port-forward` — see that role's defaults/main.yml
+# and README.md → "Dashboard (Traefik)".
+
+# >>> SET WHEN AUTHENTIK IS READY TO PROTECT IT <<<
+#
+# Uncomment to publish the dashboard on this hostname, behind the forward-auth
+# middleware below. Until then the port-forward is the only way in, which is
+# the safe default rather than a limitation. Needs a DNS record pointing at
+# Traefik's MetalLB address (`kubectl -n kube-system get svc traefik`).
+#
+# Before uncommenting, in Authentik: create a Proxy Provider in **forward auth
+# (domain level)** mode covering turtlesystems.uk, assign it to an
+# application, and add that application to the embedded outpost. Domain level
+# rather than single-application for a Kubernetes-specific reason — see
+# roles/k3s_traefik/defaults/main.yml.
+#
+# k3s_traefik_dashboard_host: traefik.turtlesystems.uk
+
+# Authentik's embedded outpost, by in-cluster DNS — the `authentik-server`
+# Service in the `authentik` namespace, i.e. the chart's own service, not a
+# separately deployed outpost. Unused while the host above is commented out;
+# set here anyway so enabling the dashboard is one line rather than two.
+# roles/k3s_traefik refuses to run if the host is set and this isn't.
+k3s_traefik_dashboard_auth_address: "http://authentik-server.authentik.svc.cluster.local/outpost.goauthentik.io/auth/traefik"
+
+# --- Apps on the cluster (roles/k3s_app) ------------------------------------
+#
+# The k3s equivalent of `stacks:` in host_vars/nas01.yml and `apps:` in a
+# Proxmox guest's host_vars — the list of *applications* on this cluster, as
+# opposed to the k3s_* cluster services above, which are the cluster's own
+# infrastructure.
+#
+# In group_vars rather than host_vars because an app is deployed to the
+# cluster, not to a node: playbooks/k3s.yml runs roles/k3s_app against
+# k3s_control_plane only (that's where the manifests directory is), but
+# nothing about an app belongs to that Pi specifically.
+#
+# Same entry shape as the other two platforms — name, src, vault_path, an
+# optional `db:`, and an optional `state:`. And the same removal rule as
+# Unraid: set `state: absent` to tear an app down and leave the entry here as
+# a tombstone; deleting the entry removes nothing, it just stops Ansible
+# visiting it. See roles/k3s_app/tasks/remove.yml.
+k3s_apps:
+  - name: authentik
+    src: authentik
+    vault_path: homelab/authentik
+    db:
+      name: authentik
+      user: authentik
+      # The key at homelab/authentik holding this role's password. Named for
+      # the environment variable Authentik itself reads, because
+      # roles/k3s_app passes Vault keys through to the app's Secret verbatim
+      # — so one value serves both the provisioning step and the running app.
+      password_vault_key: AUTHENTIK_POSTGRESQL__PASSWORD
+      admin_vault_path: homelab/shared/postgres
+      # The CNPG LoadBalancer, not the in-cluster -rw Service: these tasks
+      # run on the Ansible controller (delegate_to: localhost), which is off
+      # the cluster and can't resolve or route to a ClusterIP. Authentik
+      # itself uses the ClusterIP — see
+      # src/authentik/ansible/kubernetes/vars.yml.
+      provision_host: "{{ k3s_postgres_loadbalancer_ip }}"
+      provision_port: "5432"

build/config/ansible/inventory/group_vars/k3s_control_plane.yml

@@ -0,0 +1,9 @@
+---
+# Selects the server branch of roles/k3s_node/tasks/main.yml.
+k3s_node_role: server
+
+# The API server's cert needs this IP in its SAN list so kubectl can reach it
+# at the same address agents join through. Defaults to the control-plane
+# host's own address; override here if you later put a DNS name or VIP in
+# front of it.
+k3s_api_tls_san: "{{ ansible_host }}"

build/config/ansible/inventory/group_vars/k3s_workers.yml

@@ -0,0 +1,3 @@
+---
+# Selects the agent branch of roles/k3s_node/tasks/main.yml.
+k3s_node_role: agent

build/config/ansible/inventory/group_vars/proxmox_guests.yml

@@ -0,0 +1,19 @@
+---
+# Connection settings for the LXCs on the Proxmox nodes — the group
+# inventory/proxmox.yml composes from the API, targeted by
+# playbooks/proxmox.yml. As distinct from proxmox_nodes.yml, which is the
+# hosts those guests run on.
+#
+# In group_vars rather than repeated per guest because it is a property of
+# how every container this repo creates is built, not of any one app: the
+# Terraform modules install `ssh_public_keys` for root (see
+# build/config/terraform/variables.tf) and create no other account, so root
+# is the only user Ansible can authenticate as.
+#
+# Needed explicitly because the other three groups set this in hosts.yml
+# (`root` for unraid_servers and proxmox_nodes, `ansible` for k3s_cluster)
+# and the guests have no static entry to set it in. Without it Ansible falls
+# back to the *controller's* login name, which no container has an account
+# for — a "Permission denied (publickey)" that names a user nothing in the
+# repo mentions, so there is no obvious thread to pull.
+ansible_user: root

build/config/ansible/inventory/group_vars/proxmox_nodes.yml

@@ -0,0 +1,46 @@
+---
+# Configuration for the Proxmox nodes themselves — as distinct from the guests
+# on them, which come from the API-backed dynamic inventory and are configured
+# by playbooks/proxmox.yml. Applied by playbooks/pve_host.yml.
+#
+# In group_vars rather than a host_vars file per node: both nodes want the
+# same backup storage and the same schedule, and each backs up whichever
+# guests are running on it, so there is nothing here that differs between
+# them. A node that ever does need to differ can still override in
+# host_vars/<node>.yml.
+
+# Where archives go. An export on the NAS, added to Proxmox as a storage so
+# vzdump can write to it — the node mounts it, no guest ever sees it.
+#
+# nas01 (192.168.50.1). This used to point at nas2, which no longer exists —
+# and note the address it used, 192.168.50.2, now belongs to
+# turtle-proxmox-01, so a stale copy of this value would have had each node
+# quietly trying to write its backups to itself.
+pve_backup_storage: nas-backup
+pve_backup_storage_type: nfs
+pve_backup_nfs_server: 192.168.50.1
+pve_backup_nfs_export: /mnt/user/backups/proxmox
+
+# Every guest on the node, so anything added later is covered without an edit
+# here. Set false and list VMIDs (postgres is 161, forgejo 160 — see the
+# `backup_vmids` output in build/config/terraform/main.tf) to be selective.
+#
+# Applied per node, so with two nodes each one archives whatever is currently
+# running on it. That matters for a replicated guest: it is a real guest on
+# one node and an inactive replica volume on the other, so it is backed up
+# once, by whichever node currently owns it — and after a failover, by the
+# other one, without an edit here.
+pve_backup_all: true
+
+# 02:00, half an hour after Forgejo's database dump at 01:30
+# (src/forgejo/ansible/proxmox/vars.yml). The archive is only a complete
+# restore point if the dump inside it is from the same night — move one and
+# move the other.
+pve_backup_schedule: "02:00"
+
+# Archive retention, applied by Proxmox. Distinct from the retention inside a
+# guest: Proxmox prunes archives, not their contents.
+pve_backup_prune:
+  keep-daily: 7
+  keep-weekly: 4
+  keep-monthly: 6

build/config/ansible/inventory/host_vars/forgejo.yml

@@ -0,0 +1,52 @@
+---
+# The Proxmox LXC created by src/forgejo/terraform (VMID 160), on
+# turtle-proxmox-02. Named for the container's hostname, which is how it
+# appears in the API-backed dynamic inventory.
+#
+# This was empty for a long time while Forgejo also ran as a Compose stack on
+# nas2 — two deployments sharing one Vault path and one database means the
+# second to start wins and the other quietly serves stale state. nas2 has
+# since been retired, so there is no longer a second copy to collide with and
+# the `apps:` list below is live.
+#
+# WHAT THIS DOES NOT DO: nothing here moves repository data. Ansible
+# provisions the database and installs the app; a Forgejo with no
+# /var/lib/forgejo/data/forgejo-repositories is a working, empty forge. If the
+# nas2 repositories still exist somewhere, restore them into
+# `FORGEJO_DATA_PATH` and `chown -R git:git /var/lib/forgejo/data` before
+# pointing DNS at this host — the original instructions here were an rsync
+# off nas2, which is no longer a machine that can be read from.
+#
+# Also note SSH clone URLs move from port 2222 to 22 (see FORGEJO_SSH_PORT in
+# src/forgejo/ansible/proxmox/vars.yml): existing remotes need editing, and
+# git.turtlesystems.uk has to resolve to this container.
+
+# The address Terraform assigns this guest (build/config/terraform/main.tf).
+# Pinned here rather than left to DNS: the API-backed dynamic inventory
+# supplies no `ansible_host`, so without this Ansible SSHes to the bare
+# inventory name `forgejo`, which the LAN's resolver has no record for at
+# all. These addresses are hand-assigned from the 192.168.50.50-.59 band
+# anyway, so restating one here duplicates nothing that was ever derived.
+ansible_host: 192.168.50.52
+
+apps:
+  - name: forgejo
+    src: forgejo
+    vault_path: homelab/forgejo
+    db:
+      # The shared Postgres LXC, reached over the LAN. Unlike the Unraid
+      # host_vars this can't be `{{ ansible_host }}` — that would be this
+      # container's own address, not the database's. Keep in step with
+      # DB_HOST in src/forgejo/ansible/proxmox/vars.yml; the authoritative
+      # copy of both is the `postgres_address` output in
+      # build/config/terraform/main.tf.
+      #
+      # Provisioning runs from the Ansible controller (`delegate_to:
+      # localhost` in roles/lxc_app), so this address has to be reachable
+      # from wherever you run the playbook, not just from this container.
+      provision_host: 192.168.50.54
+      provision_port: 5432
+      name: forgejo
+      user: forgejo
+      password_vault_key: DB_PASSWORD
+      admin_vault_path: homelab/shared/postgres

build/config/ansible/inventory/host_vars/nas01.yml

@@ -0,0 +1,19 @@
+---
+# Existing Unraid server. Stacks are applied in list order.
+#
+# Each entry takes an optional `state:` — `present` (the default) deploys it,
+# `absent` tears it down. To remove a stack, set `state: absent` and leave the
+# entry here; deleting it instead removes nothing, it just stops Ansible
+# managing the containers. See README.md → "Removing an app".
+stacks:
+  # The arr ecosystem as one stack: Shelfarr (library) plus Prowlarr (indexer
+  # manager) in a single Compose project, so they come up together and share a
+  # network, a rendered .env and a Vault path.
+  - name: arr
+    src: arr
+    vault_path: homelab/arr
+    # No `db:` block — neither container uses an external database. Shelfarr
+    # is SQLite-only (config/database.yml pins production to storage/*.sqlite3
+    # with no DATABASE_URL support) and Prowlarr keeps its own SQLite under
+    # /config, so there is nothing to provision on the shared Postgres. Both
+    # databases live in the appdata bind mounts along with everything else.

build/config/ansible/inventory/host_vars/postgres.yml

@@ -0,0 +1,28 @@
+---
+# The shared Postgres LXC created by src/shared/postgres/terraform (VMID 161,
+# 192.168.50.54). Named for the container's hostname, which is how it appears
+# in the API-backed dynamic inventory.
+#
+# This is the one app with no `db:` block, because it *is* the database — the
+# per-app databases are created against it by whichever compose_stack /
+# lxc_app / k3s_app run declares them, using homelab/shared/postgres as
+# `admin_vault_path`.
+
+# The address Terraform assigns this guest (build/config/terraform/main.tf).
+# Pinned here rather than left to DNS: the API-backed dynamic inventory
+# supplies no `ansible_host`, so without this Ansible SSHes to the bare
+# inventory name and whatever the LAN's resolver happens to return. For
+# `postgres` that resolver answers 192.168.50.53 — the *Unraid* Postgres
+# container, a different instance of the same service — which fails as a
+# connection refused rather than as anything that names the real problem.
+# These addresses are hand-assigned from the 192.168.50.50-.59 band anyway,
+# so restating one here duplicates nothing that was ever derived.
+ansible_host: 192.168.50.54
+
+apps:
+  - name: postgres
+    src: shared/postgres
+    vault_path: homelab/shared/postgres
+    # The packaging's umbrella unit, not postgresql@17-main — see
+    # POSTGRES_SERVICE_NAME in src/shared/postgres/ansible/proxmox/vars.yml.
+    service_name: postgresql

build/config/ansible/inventory/hosts.yml

@@ -0,0 +1,58 @@
+---
+all:
+  children:
+    unraid_servers:
+      hosts:
+        nas01:
+          ansible_host: 192.168.50.1
+      vars:
+        ansible_user: root
+
+    # The Proxmox host itself, not the guests on it — those come from the
+    # API-backed dynamic inventory (proxmox.yml) as `proxmox_guests`. Node-level
+    # configuration (backup storage, the vzdump schedule) has to be applied over
+    # SSH to the node, which the API inventory gives no way to address.
+    #
+    # VERIFY: depending on its configuration, the community.proxmox inventory
+    # plugin can also emit the node as a host. If it does, this static entry and
+    # the dynamic one merge under the same name — check with
+    # `ansible-inventory --graph` before assuming which vars won.
+    proxmox_nodes:
+      hosts:
+        turtle-proxmox-01:
+          ansible_host: 192.168.50.2
+        turtle-proxmox-02:
+          ansible_host: 192.168.50.3
+      vars:
+        ansible_user: root
+
+    # Bare-metal K3s cluster (4 Raspberry Pis) — a third platform alongside
+    # Unraid and Proxmox, not an app deployed onto either of them. No
+    # Terraform: the hosts already exist. See playbooks/k3s.yml and
+    # roles/k3s_node. Split into two groups, not one, so the playbook can
+    # bootstrap the control plane before any agent tries to join it.
+    #
+    # ansible_user is `ansible`, not root: create this user on every Pi with
+    # NOPASSWD sudo and install the public half of `homelab/ci/ssh-k3s`
+    # (docs/vault-secrets.md) as its authorized key before the first run —
+    # see README.md "SSH access". The private half doesn't need setting up on
+    # the controller: group_vars/k3s_cluster.yml points
+    # ansible_ssh_private_key_file at a path playbooks/k3s_ssh_key.yml fetches
+    # from that same Vault entry.
+    k3s_cluster:
+      vars:
+        ansible_user: ansible
+        ansible_become: true
+      children:
+        k3s_control_plane:
+          hosts:
+            k3s-ctrl-01:
+              ansible_host: 192.168.50.60
+        k3s_workers:
+          hosts:
+            k3s-wkr-01:
+              ansible_host: 192.168.50.61
+            k3s-wkr-02:
+              ansible_host: 192.168.50.62
+            k3s-wkr-03:
+              ansible_host: 192.168.50.63

build/config/ansible/inventory/proxmox.yml

@@ -0,0 +1,56 @@
+---
+# Dynamic inventory: asks the Proxmox API which guests exist rather than
+# listing them by hand, so LXCs created by `terraform apply` show up without
+# a second edit here.
+#
+# VERIFY BEFORE FIRST USE — two things this config asserts that depend on
+# your Proxmox and collection versions:
+#
+#   1. Collection name. Proxmox content was split out of `community.general`
+#      into its own `community.proxmox` collection. If your installed
+#      Ansible predates that split, this is `community.general.proxmox`
+#      instead — here, in requirements.yml, and in ansible.cfg's
+#      `enable_plugins`. Check with:
+#        ansible-doc -t inventory community.proxmox.proxmox
+#   2. Auto-generated group names. The plugin builds groups from guest type
+#      and state; `proxmox_all_lxc` is the conventional one but confirm
+#      against your own output before relying on it:
+#        ansible-inventory -i inventory/proxmox.yml --graph
+#
+plugin: community.proxmox.proxmox
+
+url: "{{ lookup('env', 'PROXMOX_URL') }}"
+user: "{{ lookup('env', 'PROXMOX_USER') }}"
+
+# API token, not a password — same reasoning as the Unraid SSH key: a
+# password would mean an interactive prompt (impossible from CI) or a
+# plaintext secret on disk. Store these in Vault under `homelab/ci/proxmox`
+# and export them before running; see docs/vault-secrets.md.
+token_id: "{{ lookup('env', 'PROXMOX_TOKEN_ID') }}"
+token_secret: "{{ lookup('env', 'PROXMOX_TOKEN_SECRET') }}"
+
+# Homelab Proxmox typically has a self-signed cert. Flip to true once you
+# put a real one on it.
+validate_certs: false
+
+want_facts: true
+
+# Manually-created guests are not this repo's to touch. Both Terraform
+# modules tag the LXCs they create with `terraform` (see
+# src/*/terraform/variables.tf), so that tag is the line between "managed
+# here" and "made by hand on the node". Filtering rather than narrowing
+# the group below keeps hand-made guests out of the inventory entirely,
+# so no future play can target them by accident either.
+#
+# `filters` needs a reasonably recent plugin version. If yours rejects the
+# option, delete this block and put the same condition on the group:
+#   proxmox_guests: >-
+#     proxmox_vmtype == 'lxc'
+#     and 'terraform' in (proxmox_tags_parsed | default([]))
+filters:
+  - "'terraform' in (proxmox_tags_parsed | default([]))"
+
+groups:
+  # The group playbooks/proxmox.yml targets. Containers only — VMs, if you
+  # ever add any, are not what `lxc_app` knows how to install into.
+  proxmox_guests: "proxmox_vmtype == 'lxc'"

build/config/ansible/playbooks/deploy.yml

@@ -0,0 +1,21 @@
+---
+# Everything, both platforms. Unraid first only by convention now — the two
+# platforms no longer depend on each other. Proxmox-side apps provision their
+# databases against the Proxmox shared Postgres (192.168.50.54), and Terraform
+# keeps its state on the CloudNativePG cluster on k3s, neither of which is on
+# Unraid.
+#
+# Run a single platform with `ansible-playbook playbooks/unraid.yml` or
+# `playbooks/proxmox.yml` instead. `--limit <host>` and `-e only_stacks=...`
+# work the same way against any of the three.
+- name: Deploy Unraid stacks
+  ansible.builtin.import_playbook: unraid.yml
+
+- name: Deploy Proxmox apps
+  ansible.builtin.import_playbook: proxmox.yml
+
+# Last, because the vzdump job can be pinned to specific VMIDs and those guests
+# have to exist first. Harmless to re-run; skipped entirely if no node is
+# defined in the `proxmox_nodes` group.
+- name: Configure Proxmox node backups
+  ansible.builtin.import_playbook: pve_host.yml

build/config/ansible/playbooks/k3s.yml

@@ -0,0 +1,241 @@
+---
+# Bootstraps the homelab-utils k3s cluster from bare Pis: no Terraform, the
+# hosts already exist (inventory/hosts.yml → k3s_cluster). Manual-only for
+# now — not wired into .forgejo/workflows/deploy.yml, since converging 4
+# physical nodes on every push is a bigger blast radius than restarting a
+# Compose stack (same reasoning the repo already applies to `terraform
+# apply`). Run by hand:
+#
+#   ansible-playbook playbooks/k3s.yml
+#
+# Every play below the SSH-key import is tagged so a single piece can be
+# converged without touching the rest — deploying one cluster service
+# shouldn't mean re-running the node install across 4 Pis:
+#
+#   --tags nodes        k3s install/join + exec-line drift (both node plays)
+#   --tags metallb      MetalLB chart + IPAddressPool
+#   --tags monitoring   kube-prometheus-stack chart
+#   --tags postgres     CloudNativePG operator + Cluster
+#   --tags cert-manager cert-manager chart + ClusterIssuer
+#   --tags traefik      dashboard route on k3s's bundled Traefik
+#   --tags services     all five of the above, no node install
+#   --tags apps         the k3s_apps: list (roles/k3s_app) — apps, not
+#                       cluster services; narrow further with -e only_apps=…
+#   --tags upgrades     unattended-upgrades config
+#   --tags kubeconfig   re-fetch the admin kubeconfig
+#
+# e.g. `ansible-playbook playbooks/k3s.yml --tags postgres`. The SSH-key
+# import is tagged `always` rather than given a name of its own: it's not a
+# thing you'd select, it's the prerequisite for any of these connecting at
+# all, so it has to survive every --tags filter.
+#
+# `services` covers the cluster's own infrastructure and `apps` covers what
+# runs on top of it; neither implies the other, so a run meaning "everything
+# except reinstalling k3s" is `--tags services,apps`.
+#
+# Prerequisites (see README.md "K3s (Raspberry Pi)"): the `ansible` user
+# exists on every Pi with NOPASSWD sudo and this repo's SSH key installed,
+# and homelab/k3s-homelab-utils/K3S_TOKEN is already set in Vault — see
+# docs/vault-secrets.md. Both plays fetch that same fixed token independently
+# rather than one generating it and handing it to the other, which is what
+# makes a full rebuild (wipe both SD cards, reinstall) reproduce the same
+# cluster identity.
+
+# First, before anything tries to connect: pull the SSH key the plays below
+# authenticate with out of Vault and onto the controller. See
+# k3s_ssh_key.yml — it's a separate file because k3s_maintenance.yml imports
+# it too.
+- ansible.builtin.import_playbook: k3s_ssh_key.yml
+  tags: always
+
+- name: Bootstrap the k3s control plane
+  hosts: k3s_control_plane
+  gather_facts: true
+  tags: nodes
+  roles:
+    - k3s_node
+
+# serial: 1 not for joining itself (idempotent, safe in parallel) but for
+# roles/k3s_node's exec-line drift detection: if k3s_extra_args changes and
+# every worker's k3s-agent restarts to pick it up, one at a time keeps more
+# than one node's kubelet from bouncing simultaneously. See tasks/agent.yml.
+- name: Join k3s worker nodes
+  hosts: k3s_workers
+  gather_facts: true
+  serial: 1
+  tags: nodes
+  roles:
+    - k3s_node
+
+# Against the control plane only — it just drops manifests for k3s's own
+# helm-controller and deploy controller to reconcile, so it doesn't need
+# every node like the plays above. Runs after workers have joined so the
+# resulting speaker DaemonSet schedules across the whole cluster on first
+# reconcile, though this doesn't strictly matter: the controller picks up
+# new nodes on its own regardless of ordering. Before monitoring so a
+# LoadBalancer Service (e.g. exposing Prometheus/Grafana later) has
+# somewhere to get an IP from as soon as it's requested.
+- name: Deploy MetalLB (LoadBalancer IPs)
+  hosts: k3s_control_plane
+  gather_facts: false
+  tags:
+    - services
+    - metallb
+  roles:
+    - k3s_metallb
+
+# Against the control plane only — it just drops a HelmChart manifest for
+# k3s's own helm-controller to reconcile, so it doesn't need every node like
+# the plays above. Runs after workers have joined so the resulting
+# node-exporter DaemonSet schedules across the whole cluster on first
+# reconcile, though this doesn't strictly matter: the controller picks up
+# new nodes on its own regardless of ordering.
+- name: Deploy cluster monitoring (Prometheus)
+  hosts: k3s_control_plane
+  gather_facts: false
+  tags:
+    - services
+    - monitoring
+  roles:
+    - k3s_monitoring
+
+# Against the control plane only, same reasoning as monitoring/MetalLB above
+# — it just drops manifests for k3s's own controllers to reconcile. Needs
+# repo_root (unlike the other k3s roles) because it reads config from
+# src/shared/postgres/, the same shared-service config Unraid/Proxmox
+# already deploy from — see roles/k3s_postgres/tasks/main.yml.
+- name: Deploy shared Postgres (CloudNativePG)
+  hosts: k3s_control_plane
+  gather_facts: false
+  tags:
+    - services
+    - postgres
+  vars:
+    repo_root: "{{ playbook_dir }}/../../../.."
+  roles:
+    - k3s_postgres
+
+# Against the control plane only, same reasoning as the services above.
+# Before the apps play because an app's Ingress annotates itself against the
+# ClusterIssuer this creates — not that ordering is load-bearing (k3s's
+# deploy controller retries, and cert-manager picks up an Ingress whenever it
+# appears), but an app deployed first would sit without a certificate until
+# this ran, which reads as a broken deploy rather than a pending one.
+- name: Deploy cert-manager (TLS certificates)
+  hosts: k3s_control_plane
+  gather_facts: false
+  tags:
+    - services
+    - cert-manager
+  roles:
+    - k3s_cert_manager
+
+# Against the control plane only, same reasoning as the services above. The
+# odd one out among them: it installs nothing, because k3s installs Traefik
+# itself — it only adjusts what k3s already put there, via a HelmChartConfig.
+# After cert-manager because publishing the dashboard on a hostname asks for a
+# Certificate from the ClusterIssuer that play creates; ordering isn't
+# load-bearing (the deploy controller retries), it just avoids a route sitting
+# without a certificate in between.
+- name: Configure Traefik (ingress + dashboard)
+  hosts: k3s_control_plane
+  gather_facts: false
+  tags:
+    - services
+    - traefik
+  roles:
+    - k3s_traefik
+
+# Apps, as opposed to the cluster services above — see roles/k3s_app for the
+# distinction and inventory/group_vars/k3s_cluster.yml for the list. Against
+# the control plane because that's where the manifests directory is; nothing
+# about an app belongs to that Pi in particular.
+#
+# Note what a green run here does and doesn't mean, the same caveat every
+# service play above carries: the role renders manifests for k3s's
+# controllers to reconcile, so success means the files landed and any
+# database was provisioned — not that the workload came up. Check with
+# `kubectl -n <namespace> get pods`.
+- name: Converge apps on the cluster
+  hosts: k3s_control_plane
+  gather_facts: false
+  tags: apps
+  vars:
+    repo_root: "{{ playbook_dir }}/../../../.."
+    # Comma-separated app names to restrict this run to, e.g.
+    # `-e only_apps=authentik`. Empty (the default) converges every app in
+    # k3s_apps. Same knob as only_stacks in playbooks/unraid.yml — --tags
+    # can select the apps play as a whole, but not one app within it.
+    only_apps: ""
+
+  tasks:
+    # "Converge", not "deploy": an entry carrying `state: absent` is torn
+    # down rather than brought up. Removals have to stay in the list to be
+    # acted on — see roles/k3s_app/tasks/main.yml.
+    - name: Converge each declared app
+      ansible.builtin.include_role:
+        name: k3s_app
+      loop: >-
+        {{ k3s_apps | default([]) if only_apps == ''
+           else k3s_apps | default([])
+                | selectattr('name', 'in', only_apps.split(',')) | list }}
+      loop_control:
+        loop_var: app
+        label: "{{ app.name }}"
+
+# Every node, both roles — hands-off patching so the Pis don't need logging
+# into just to stay updated. Reboots are deliberately not part of this: see
+# roles/unattended_upgrades and playbooks/k3s_maintenance.yml.
+- name: Configure unattended upgrades
+  hosts: k3s_cluster
+  gather_facts: true
+  tags: upgrades
+  roles:
+    - unattended_upgrades
+
+# Last, and against the control plane specifically (there's only one) — pulls
+# the admin kubeconfig k3s wrote for itself back to the controller so
+# `kubectl` works from your workstation.
+- name: Fetch the cluster kubeconfig
+  hosts: k3s_control_plane
+  gather_facts: false
+  tags: kubeconfig
+  vars:
+    repo_root: "{{ playbook_dir }}/../../../.."
+
+  tasks:
+    - name: Read the cluster's kubeconfig
+      ansible.builtin.slurp:
+        src: /etc/rancher/k3s/k3s.yaml
+      register: k3s_kubeconfig_raw
+
+    - name: Ensure the local kubeconfig directory exists
+      ansible.builtin.file:
+        path: "{{ k3s_kubeconfig_local_path | dirname }}"
+        state: directory
+        mode: "0700"
+      delegate_to: localhost
+      become: false
+
+    # k3s.yaml points at 127.0.0.1 and names everything "default" — both
+    # correct only on the node itself. Rewritten so the file is usable
+    # straight off the controller: the server address becomes reachable from
+    # off-box, and the cluster/context/user names become this cluster's own
+    # rather than colliding with every other "default" in ~/.kube/config.
+    - name: Write the rewritten kubeconfig to the controller
+      ansible.builtin.copy:
+        dest: "{{ k3s_kubeconfig_local_path }}"
+        content: >-
+          {{ (k3s_kubeconfig_raw.content | b64decode)
+             | replace('127.0.0.1', ansible_host)
+             | regex_replace('\\bdefault\\b', k3s_cluster_name) }}
+        mode: "0600"
+      delegate_to: localhost
+      become: false
+
+    - name: Show how to use the fetched kubeconfig
+      ansible.builtin.debug:
+        msg: >-
+          Kubeconfig written to {{ k3s_kubeconfig_local_path }}. Use it with
+          `export KUBECONFIG={{ k3s_kubeconfig_local_path }}`, or merge it
+          into ~/.kube/config by hand.

build/config/ansible/playbooks/k3s_maintenance.yml

@@ -0,0 +1,37 @@
+---
+# Rolls a "reboot required" flag left by unattended-upgrades
+# (roles/unattended_upgrades, Automatic-Reboot disabled) into an actual
+# reboot, one node at a time — cordon, drain, reboot, wait for Ready,
+# uncordon — so patching the OS never means dropping every workload on the
+# cluster at once. A no-op on any node that isn't carrying a pending reboot.
+#
+# serial: 1 is the point of this playbook: exactly one node is ever down for
+# maintenance at a time, whatever `--limit` narrows the run to.
+#
+# Host order matters here and isn't left to inventory.yml's own group
+# order (control plane, then workers) — reversed below so workers are
+# rebooted while the API server is still up on the control plane, and the
+# control plane's own reboot (which drops the API for everyone, single-server
+# cluster, no HA) happens last rather than first.
+#
+# Manual-only for now, like playbooks/k3s.yml — run by hand:
+#
+#   ansible-playbook playbooks/k3s_maintenance.yml
+#
+# Intended to eventually run on a schedule from a self-hosted Forgejo Actions
+# runner (a `schedule:`-triggered workflow, same runner as
+# .forgejo/workflows/deploy.yml) rather than by hand — not wired up yet, but
+# the SSH key import below is what makes that possible without also handing
+# the runner an ssh-agent.
+
+# Same first play as k3s.yml — the key comes from Vault, and this playbook is
+# routinely run on its own, so it can't rely on a k3s.yml run having fetched
+# it. See k3s_ssh_key.yml.
+- ansible.builtin.import_playbook: k3s_ssh_key.yml
+
+- name: Roll pending reboots across the k3s cluster
+  hosts: k3s_workers:k3s_control_plane
+  gather_facts: true
+  serial: 1
+  roles:
+    - k3s_maintenance

build/config/ansible/playbooks/k3s_ssh_key.yml

@@ -0,0 +1,82 @@
+---
+# Materializes the SSH key the k3s plays authenticate with, from Vault
+# (homelab/ci/ssh-k3s) onto the controller. Imported as the first play of both
+# playbooks/k3s.yml and playbooks/k3s_maintenance.yml — a separate file rather
+# than copied into each, since both target k3s_cluster and both would
+# otherwise depend on a file only the other one creates.
+#
+# This is the same shape CI already uses for the Unraid key:
+# .forgejo/workflows/deploy.yml fetches homelab/ci/ssh, writes it to disk and
+# points ANSIBLE_PRIVATE_KEY_FILE at it. Vault is the one place secrets live,
+# so a manual k3s run shouldn't need the key hand-loaded into ssh-agent first
+# — which was the only way it worked before.
+#
+# hosts: k3s_cluster, not localhost, and deliberately so: the implicit
+# localhost is not a member of `all`, so it doesn't inherit group_vars/all.yml
+# — where vault_addr, vault_kv_mount and vault_auth_method live. Targeting the
+# group picks those up along with k3s_ssh_key_vault_path/_local_path from
+# group_vars/k3s_cluster.yml. Nothing here connects to a Pi (gather_facts is
+# off and every task is delegated or connectionless), which is the whole
+# point: at this stage the key isn't on disk yet.
+#
+# become: false on the delegated tasks because the k3s_cluster group sets
+# ansible_become: true for the Pis (inventory/hosts.yml) — without it these
+# would try to sudo on the controller.
+
+- name: Fetch the k3s SSH deploy key from Vault
+  hosts: k3s_cluster
+  gather_facts: false
+
+  tasks:
+    # The escape hatch, and the reason it's a flag rather than just pointing
+    # ansible_ssh_private_key_file somewhere else: if Vault is unreachable,
+    # overriding the key path alone doesn't help — this play would still fail
+    # before the first real one runs. Skipping it is the only thing that lets
+    # a run proceed on a local key. Both overrides together:
+    #
+    #   ansible-playbook playbooks/k3s.yml \
+    #     -e k3s_ssh_key_fetch=false \
+    #     -e ansible_ssh_private_key_file=~/.ssh/k3s_ansible
+    - name: Fetch and write the key
+      when: k3s_ssh_key_fetch | default(true) | bool
+      block:
+        # run_once because the key is per-cluster, not per-host: one Vault
+        # read for the whole group. Facts set by a run_once task apply to
+        # every host in the play, but nothing outside this play needs it.
+        - name: Look up the k3s SSH deploy key from Vault
+          ansible.builtin.set_fact:
+            k3s_ssh_key_secret: >-
+              {{ lookup('community.hashi_vault.vault_kv2_get',
+                         k3s_ssh_key_vault_path,
+                         engine_mount_point=vault_kv_mount,
+                         url=vault_addr,
+                         auth_method=vault_auth_method,
+                         role_id=vault_role_id | default(omit),
+                         secret_id=vault_secret_id | default(omit)).secret }}
+          run_once: true
+          no_log: true
+
+        - name: Ensure the local key directory exists
+          ansible.builtin.file:
+            path: "{{ k3s_ssh_key_local_path | dirname }}"
+            state: directory
+            mode: "0700"
+          delegate_to: localhost
+          become: false
+          run_once: true
+
+        # trim + an explicit trailing newline: OpenSSH rejects a key file
+        # whose final line isn't terminated, and `vault kv put
+        # PRIVATE_KEY=@file` is as likely to have stored one with trailing
+        # whitespace as not. 0600 for the same reason — ssh refuses a key file
+        # that's group- or world-readable. Both fail at connection time rather
+        # than here, which is a much less obvious error to read.
+        - name: Write the private key to the controller
+          ansible.builtin.copy:
+            dest: "{{ k3s_ssh_key_local_path }}"
+            content: "{{ k3s_ssh_key_secret.PRIVATE_KEY | trim }}\n"
+            mode: "0600"
+          delegate_to: localhost
+          become: false
+          run_once: true
+          no_log: true

build/config/ansible/playbooks/proxmox.yml

@@ -0,0 +1,29 @@
+---
+# Installs apps natively into Proxmox LXCs. The LXCs themselves are created
+# by Terraform (build/config/terraform) — run that first; this playbook
+# assumes the guests already exist and are reachable.
+#
+# Hosts come from the community.proxmox inventory plugin
+# (inventory/proxmox.yml), which queries the Proxmox API rather than listing
+# guests by hand. `apps` is declared per guest in host_vars/<guest>.yml,
+# mirroring how `stacks` works for Unraid hosts.
+- name: Install apps into Proxmox LXCs
+  hosts: proxmox_guests
+  gather_facts: true
+  vars:
+    repo_root: "{{ playbook_dir }}/../../../.."
+    # Comma-separated app names to restrict this run to. Shares the
+    # `only_stacks` name with playbooks/unraid.yml so a single `-e` narrows
+    # both platforms in one invocation of deploy.yml.
+    only_stacks: ""
+
+  tasks:
+    - name: Install each declared app
+      ansible.builtin.include_role:
+        name: lxc_app
+      loop: >-
+        {{ apps if only_stacks == ''
+           else apps | selectattr('name', 'in', only_stacks.split(',')) | list }}
+      loop_control:
+        loop_var: app
+        label: "{{ app.name }}"

build/config/ansible/playbooks/pve_host.yml

@@ -0,0 +1,14 @@
+---
+# Configures the Proxmox node itself — as distinct from the guests on it,
+# which playbooks/proxmox.yml handles. Right now that means backups: the NAS
+# storage archives are written to, and the vzdump schedule that fills it.
+#
+# Targets the static `proxmox_nodes` group in inventory/hosts.yml rather than
+# the API-backed dynamic inventory, which enumerates guests. If no node is
+# defined there yet, this play matches nothing and is skipped.
+- name: Configure Proxmox nodes
+  hosts: proxmox_nodes
+  gather_facts: true
+
+  roles:
+    - pve_backup

build/config/ansible/playbooks/unraid.yml

@@ -0,0 +1,24 @@
+---
+- name: Converge Unraid Docker Compose stacks
+  hosts: unraid_servers
+  gather_facts: true
+  vars:
+    repo_root: "{{ playbook_dir }}/../../../.."
+    # Comma-separated stack names to restrict this run to, e.g.
+    # `-e only_stacks=forgejo` or `-e only_stacks=forgejo,postgres`.
+    # Empty (the default) converges every stack declared for the host.
+    only_stacks: ""
+
+  tasks:
+    # "Converge", not "deploy": an entry carrying `state: absent` is torn down
+    # rather than brought up. Removals have to stay in the list to be acted
+    # on — see the header comment in roles/compose_stack/tasks/main.yml.
+    - name: Converge each declared stack
+      ansible.builtin.include_role:
+        name: compose_stack
+      loop: >-
+        {{ stacks if only_stacks == ''
+           else stacks | selectattr('name', 'in', only_stacks.split(',')) | list }}
+      loop_control:
+        loop_var: stack
+        label: "{{ stack.name }}"

build/config/ansible/requirements.yml

@@ -0,0 +1,15 @@
+---
+collections:
+  - name: community.docker
+    version: ">=3.10.0"
+  - name: community.hashi_vault
+    version: ">=6.2.0"
+  - name: community.postgresql
+    version: ">=3.0.0"
+  - name: ansible.posix
+    version: ">=1.5.0"
+  # Proxmox dynamic inventory (inventory/proxmox.yml). If your Ansible
+  # predates the split of Proxmox content out of community.general, drop
+  # this and use community.general instead — see the note in
+  # inventory/proxmox.yml.
+  - name: community.proxmox

build/config/ansible/roles/compose_stack/defaults/main.yml

@@ -0,0 +1,9 @@
+---
+compose_projects_root: /boot/config/plugins/compose.manager/projects
+vault_kv_mount: kv
+
+# Where a stack's committed icon.png is copied to on the Unraid host, for the
+# `net.unraid.docker.icon` label to point at. Under appdata rather than
+# /boot — it's regenerated content, not configuration worth putting on the
+# flash drive's write budget.
+unraid_icons_root: /mnt/user/appdata/icons

build/config/ansible/roles/compose_stack/tasks/deploy.yml

@@ -0,0 +1,183 @@
+---
+# Deploys one stack to the current Unraid host: sync the compose file, render
+# its .env from Vault, provision its database if it declares one, then
+# `docker compose up -d`. Compose files stay static/generic — only the
+# rendered .env differs per run, which is what keeps re-runs idempotent.
+#
+# Included from main.yml when the stack's `state:` is `present`; the facts it
+# relies on (stack_local_dir, stack_remote_dir) are set there.
+
+- name: Load portable stack variables
+  ansible.builtin.include_vars:
+    file: "{{ stack_local_dir }}/common/vars.yml"
+    name: stack_common_vars
+
+- name: Load Unraid-specific stack variables
+  ansible.builtin.include_vars:
+    file: "{{ stack_local_dir }}/ansible/unraid/vars.yml"
+    name: stack_platform_vars
+
+- name: Ensure remote project directory exists
+  ansible.builtin.file:
+    path: "{{ stack_remote_dir }}"
+    state: directory
+    mode: "0750"
+
+- name: Copy docker-compose.yml
+  ansible.builtin.copy:
+    src: "{{ stack_local_dir }}/ansible/unraid/docker-compose.yml"
+    dest: "{{ stack_remote_dir }}/docker-compose.yml"
+    mode: "0640"
+
+# --- Unraid Docker page presentation ---------------------------------------
+#
+# The committed icon.png has two destinations, because Unraid draws the stack
+# row and the containers under it from entirely different sources:
+#
+#   stack row  — Compose Manager serves <project dir>/icon.png directly
+#                (icon.php, which also accepts .jpg/.gif/.svg or a bare
+#                `icon`). No label, no template, no URL: a file or nothing.
+#                Present only in the maintained fork, Compose Manager Plus;
+#                the original plugin has no icon support at all, and there
+#                the copy is inert rather than harmful.
+#   containers — the Docker page reads a container's icon, WebUI link and
+#                console shell from the dockerMan template that created it.
+#                A Compose stack has no template, so 6.10+ falls back to the
+#                `net.unraid.docker.*` labels the compose files set, and
+#                STACK_ICON below is what the icon label resolves to.
+#
+# Hence the appdata copy as well: the label is a path the webgui resolves at
+# page-render time, and pointing it into /boot to reuse the project-dir copy
+# would put the flash drive in the path of every Docker page load.
+#
+# Only the second of those is per-container, so a stack running more than one
+# service can also commit icon-<service>.png files. Those get the appdata copy
+# only — there is exactly one stack row and it already has its icon. See
+# main.yml, which is where the names are resolved.
+- name: Sync stack icons to the Unraid host
+  when: stack_icon_dest | length > 0 or stack_service_icon_names | length > 0
+  block:
+    - name: Ensure the icon directory exists
+      ansible.builtin.file:
+        path: "{{ unraid_icons_root }}"
+        state: directory
+        mode: "0755"
+
+    - name: Copy stack icon for the container labels
+      ansible.builtin.copy:
+        src: "{{ stack_local_dir }}/ansible/unraid/icon.png"
+        dest: "{{ stack_icon_dest }}"
+        mode: "0644"
+      when: stack_icon_dest | length > 0
+
+    # Into the project folder on the flash drive, next to the compose file.
+    # `copy` is checksum-based, so a redeploy that hasn't changed the icon
+    # writes nothing — this costs one flash write per icon, not one per run.
+    - name: Copy stack icon for the Compose Manager stack row
+      ansible.builtin.copy:
+        src: "{{ stack_local_dir }}/ansible/unraid/icon.png"
+        dest: "{{ stack_remote_dir }}/icon.png"
+        mode: "0644"
+      when: stack_icon_dest | length > 0
+
+    # Appdata only, and no flash-drive copy: these label individual containers,
+    # which is the one thing the Compose Manager stack row isn't.
+    - name: Copy per-service icons for the container labels
+      ansible.builtin.copy:
+        src: "{{ stack_local_dir }}/ansible/unraid/icon-{{ item }}.png"
+        dest: "{{ unraid_icons_root }}/{{ stack.name }}-{{ item }}.png"
+        mode: "0644"
+      loop: "{{ stack_service_icon_names }}"
+
+- name: Look up stack secrets from Vault
+  ansible.builtin.set_fact:
+    vault_secrets: >-
+      {{ lookup('community.hashi_vault.vault_kv2_get',
+                 stack.vault_path,
+                 engine_mount_point=vault_kv_mount,
+                 url=vault_addr,
+                 auth_method=vault_auth_method,
+                 role_id=vault_role_id | default(omit),
+                 secret_id=vault_secret_id | default(omit)).secret }}
+  no_log: true
+
+- name: Render .env file
+  ansible.builtin.template:
+    src: env.j2
+    dest: "{{ stack_remote_dir }}/.env"
+    mode: "0600"
+  vars:
+    # The role-computed icon paths first, then portable values, then
+    # Unraid-specific overrides on top; Vault secrets win over all three (see
+    # env.j2). STACK_ICON and the per-service STACK_ICON_<SERVICE> keys sit at
+    # the bottom so either vars.yml layer can replace one with a hosted URL,
+    # and STACK_ICON is always defined so Compose never warns about an unset
+    # variable in the label block.
+    #
+    # STACK_ICON_<SERVICE> only exists when the matching icon-<service>.png is
+    # committed. A compose file that interpolates one without the file present
+    # gets Compose's empty-value warning and the placeholder icon — set it in
+    # vars.yml instead if the icon is meant to be a URL.
+    env_defaults: >-
+      {{ {'STACK_ICON': stack_icon_dest}
+         | combine(stack_service_icons)
+         | combine(stack_common_vars.env_defaults | default({}))
+         | combine(stack_platform_vars.env_defaults | default({})) }}
+  no_log: true
+
+- name: Provision application database
+  when: stack.db is defined
+  no_log: true
+  block:
+    - name: Look up Postgres superuser credentials from Vault
+      ansible.builtin.set_fact:
+        pg_admin_secrets: >-
+          {{ lookup('community.hashi_vault.vault_kv2_get',
+                     stack.db.admin_vault_path,
+                     engine_mount_point=vault_kv_mount,
+                     url=vault_addr,
+                     auth_method=vault_auth_method,
+                     role_id=vault_role_id | default(omit),
+                     secret_id=vault_secret_id | default(omit)).secret }}
+
+    - name: Ensure application database role exists
+      community.postgresql.postgresql_user:
+        name: "{{ stack.db.user }}"
+        password: "{{ vault_secrets[stack.db.password_vault_key] }}"
+        login_host: "{{ stack.db.provision_host }}"
+        login_port: "{{ stack.db.provision_port }}"
+        login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}"
+        login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}"
+        state: present
+      delegate_to: localhost
+      become: false
+
+    # Role first, then the database with `owner:` — see the same pair of
+    # tasks in roles/lxc_app for the full reasoning. The short version is
+    # that `priv: ALL` on postgresql_user was removed from
+    # community.postgresql in 4.0.0, and ownership is what the replacement
+    # should have been anyway.
+    #
+    # This instance is pinned to Postgres 13, so unlike the Proxmox one it
+    # was never *also* broken by the Postgres 15 `public` schema change —
+    # the removal of `priv` is the only thing that forces the edit here.
+    # Kept identical to the other two roles regardless: the point of the
+    # three app roles sharing this block is that an app moving between
+    # platforms gets the same database either way.
+    - name: Ensure application database exists
+      community.postgresql.postgresql_db:
+        name: "{{ stack.db.name }}"
+        owner: "{{ stack.db.user }}"
+        login_host: "{{ stack.db.provision_host }}"
+        login_port: "{{ stack.db.provision_port }}"
+        login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}"
+        login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}"
+        state: present
+      delegate_to: localhost
+      become: false
+
+- name: Deploy stack with Docker Compose
+  community.docker.docker_compose_v2:
+    project_src: "{{ stack_remote_dir }}"
+    state: present
+    pull: policy

build/config/ansible/roles/compose_stack/tasks/main.yml

@@ -0,0 +1,75 @@
+---
+# Converges one stack (one loop iteration of `stack` from playbooks/unraid.yml)
+# to the state its host_vars entry asks for.
+#
+# `state: present` (the default) deploys; `state: absent` tears down. Removal
+# has to be asked for explicitly because Ansible keeps no record of what it
+# deployed last run — deleting a stack from a host's `stacks:` list only stops
+# the loop visiting it, leaving the containers running and unmanaged on the
+# host forever. So the entry stays put as a tombstone, with `state: absent`,
+# until the teardown has actually been applied.
+#
+# Unraid-only by design. Proxmox guests install apps natively rather than as
+# Compose stacks — see the `lxc_app` role.
+
+- name: Set stack facts
+  ansible.builtin.set_fact:
+    stack_local_dir: "{{ repo_root }}/src/{{ stack.src }}"
+    stack_remote_dir: "{{ compose_projects_root }}/{{ stack.name }}"
+    stack_state: "{{ stack.state | default('present') }}"
+
+- name: Validate requested stack state
+  ansible.builtin.assert:
+    that: stack_state in ['present', 'absent']
+    fail_msg: >-
+      Stack '{{ stack.name }}' has state '{{ stack_state }}'; expected
+      'present' or 'absent'.
+    quiet: true
+
+# Which icons this stack ships, worked out from the repo checkout on the
+# controller rather than from anything on the host — `is exists` and the
+# `fileglob` lookup both evaluate locally. Computed here rather than in
+# deploy.yml because both branches need it: deploy copies these files out,
+# remove deletes them again.
+#
+# Two naming conventions, because Unraid has two icon consumers (see
+# deploy.yml for the full explanation):
+#
+#   icon.png            the stack — Compose Manager's project row, plus the
+#                       `net.unraid.docker.icon` label of whichever service
+#                       interpolates ${STACK_ICON}.
+#   icon-<service>.png  one container — becomes ${STACK_ICON_<SERVICE>}, so a
+#                       multi-container stack can label each of its services
+#                       with its own image instead of sharing the stack's.
+#
+# `<service>` is the compose service name; the variable is it uppercased with
+# `-` folded to `_` (icon-shelfarr-libation.png → STACK_ICON_SHELFARR_LIBATION).
+- name: Set stack icon facts
+  ansible.builtin.set_fact:
+    # Empty when the stack ships no icon.png — an empty label is what the
+    # webgui already assumes, so it falls back to the placeholder as before.
+    stack_icon_dest: >-
+      {{ ((stack_local_dir ~ '/ansible/unraid/icon.png') is exists)
+         | ternary(unraid_icons_root ~ '/' ~ stack.name ~ '.png', '') }}
+    stack_service_icon_names: "{{ _service_icon_names }}"
+    stack_service_icons: >-
+      {{ dict(_service_icon_names
+              | map('upper') | map('replace', '-', '_')
+              | map('regex_replace', '^(.+)$', 'STACK_ICON_\1')
+              | zip(_service_icon_names
+                    | map('regex_replace', '^(.+)$',
+                          unraid_icons_root ~ '/' ~ stack.name ~ '-\1.png'))) }}
+  vars:
+    _service_icon_names: >-
+      {{ query('fileglob', stack_local_dir ~ '/ansible/unraid/icon-*.png')
+         | map('basename')
+         | map('regex_replace', '^icon-(.+)\.png$', '\1')
+         | map('lower') | list }}
+
+- name: Deploy stack
+  ansible.builtin.include_tasks: deploy.yml
+  when: stack_state == 'present'
+
+- name: Remove stack
+  ansible.builtin.include_tasks: remove.yml
+  when: stack_state == 'absent'

build/config/ansible/roles/compose_stack/tasks/remove.yml

@@ -0,0 +1,106 @@
+---
+# Tears one stack down off the current Unraid host: `docker compose down`,
+# then delete the project folder, then optionally drop its database.
+#
+# Included from main.yml when the stack's `state:` is `absent`.
+#
+# Ordering matters and is not obvious: `docker compose down` reads the compose
+# file to know what it is removing, so the project folder has to survive until
+# after that step. Delete the folder (or the stack's `src/` tree) first and
+# there is nothing left to tell Docker what belonged to the project — the
+# containers and networks have to be cleaned up by hand instead.
+#
+# What `down` takes with it, and what it deliberately doesn't:
+#   - removed: the containers, and any network the project itself created
+#   - kept:    bind mounts. Appdata under /mnt/user/appdata/<app> survives, so
+#              a removal is reversible by flipping `state:` back to `present`.
+#   - kept:    external networks (caddy-net, unraid_shared) — they belong to
+#              another stack or to Unraid, not to this project.
+#   - opt-in:  named volumes (`remove_volumes: true`) and images
+#              (`remove_images: local` or `all`).
+#
+# Not touched at all: the stack's `homelab/<app>` path in Vault. That has its
+# own lifecycle and no reason to be destroyed by a redeployable teardown.
+
+- name: Check whether the stack's project directory is still present
+  ansible.builtin.stat:
+    path: "{{ stack_remote_dir }}/docker-compose.yml"
+  register: stack_compose_file
+
+- name: Tear down stack with Docker Compose
+  community.docker.docker_compose_v2:
+    project_src: "{{ stack_remote_dir }}"
+    state: absent
+    # Named volumes and images are destructive beyond "stop running this
+    # here", so they stay opt-in per stack rather than being implied by
+    # `state: absent`.
+    remove_volumes: "{{ stack.remove_volumes | default(false) }}"
+    remove_images: "{{ stack.remove_images | default(omit) }}"
+    remove_orphans: true
+  # Absent already — a re-run after a successful removal, or a stack that was
+  # never deployed to this host. Both are the desired end state, not an error.
+  when: stack_compose_file.stat.exists
+
+- name: Remove stack project directory
+  ansible.builtin.file:
+    path: "{{ stack_remote_dir }}"
+    state: absent
+
+# Unlike appdata, these are not state — they're copies of PNGs committed in
+# the repo, put there purely so the Docker page had something to render.
+# Nothing is lost by deleting them and a redeploy puts them back, so they
+# aren't opt-in the way volumes and images are.
+#
+# Listed from the repo checkout (main.yml) rather than globbed on the host:
+# `{{ stack.name }}-*.png` would also match the icons of any stack whose name
+# starts with this one's.
+- name: Remove stack icons
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: absent
+  loop: "{{ [unraid_icons_root ~ '/' ~ stack.name ~ '.png']
+            + stack_service_icons.values() | list }}"
+
+- name: Drop application database
+  when:
+    - stack.db is defined
+    - stack.remove_database | default(false)
+  no_log: true
+  block:
+    # Only the Postgres superuser credentials are needed here. The stack's own
+    # Vault secrets are not read, so a teardown still works after its
+    # `homelab/<app>` path has been deleted.
+    - name: Look up Postgres superuser credentials from Vault
+      ansible.builtin.set_fact:
+        pg_admin_secrets: >-
+          {{ lookup('community.hashi_vault.vault_kv2_get',
+                     stack.db.admin_vault_path,
+                     engine_mount_point=vault_kv_mount,
+                     url=vault_addr,
+                     auth_method=vault_auth_method,
+                     role_id=vault_role_id | default(omit),
+                     secret_id=vault_secret_id | default(omit)).secret }}
+
+    # Database before role: Postgres refuses to drop a role that still owns
+    # objects, and the role owns this database.
+    - name: Drop application database
+      community.postgresql.postgresql_db:
+        name: "{{ stack.db.name }}"
+        login_host: "{{ stack.db.provision_host }}"
+        login_port: "{{ stack.db.provision_port }}"
+        login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}"
+        login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}"
+        state: absent
+      delegate_to: localhost
+      become: false
+
+    - name: Drop application database role
+      community.postgresql.postgresql_user:
+        name: "{{ stack.db.user }}"
+        login_host: "{{ stack.db.provision_host }}"
+        login_port: "{{ stack.db.provision_port }}"
+        login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}"
+        login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}"
+        state: absent
+      delegate_to: localhost
+      become: false

build/config/ansible/roles/compose_stack/templates/env.j2

@@ -0,0 +1,7 @@
+{# Non-secret defaults first, then Vault secrets — secrets win on key clashes. #}
+{% for key, value in (env_defaults | default({})).items() %}
+{{ key }}="{{ value }}"
+{% endfor %}
+{% for key, value in (vault_secrets | default({})).items() %}
+{{ key }}="{{ value }}"
+{% endfor %}

build/config/ansible/roles/k3s_app/defaults/main.yml

@@ -0,0 +1,34 @@
+---
+# Ansible-side knobs for roles/k3s_app — the k3s equivalent of
+# compose_stack (Unraid) and lxc_app (Proxmox): the generic role that
+# deploys one *application* onto the homelab-utils cluster, as opposed to
+# the k3s_* roles that deploy one *cluster service*.
+#
+# The distinction is worth stating, because the file layout looks similar.
+# roles/k3s_metallb, k3s_monitoring, k3s_postgres and k3s_cert_manager each
+# hard-code a single thing the cluster itself needs, with their config in
+# defaults/ and group_vars. This role hard-codes nothing: it's driven by the
+# `k3s_apps:` list in inventory/group_vars/k3s_cluster.yml and reads each
+# app's config out of src/<app>/, the same src/ tree Unraid and Proxmox
+# already deploy from. An app that moves between platforms gets a new
+# platform vars file, not a rewrite — see CLAUDE.md → "How config is
+# layered".
+#
+# Everything reaches the cluster the same way every other k3s role does:
+# rendering a file into k3s's auto-deploying manifests directory and letting
+# k3s's own controllers reconcile it. No helm binary, no kubeconfig, no
+# kubernetes.core collection on the controller.
+
+# Where k3s watches for manifests to apply. Anything written here is applied
+# by k3s's deploy controller, and — importantly for remove.yml — anything
+# *deleted* from here has its resources torn down again, because the
+# controller tracks what each file created via an Addon CR.
+k3s_manifests_dir: /var/lib/rancher/k3s/server/manifests
+
+vault_kv_mount: kv
+
+# Which directory under src/<app>/ansible/ holds the Kubernetes deployment.
+# "kubernetes" rather than "k3s": what's in there is plain Kubernetes plus
+# k3s's HelmChart CR, and an app moved to a different distribution would
+# keep most of it.
+k3s_app_platform_dir: kubernetes

build/config/ansible/roles/k3s_app/tasks/deploy.yml

@@ -0,0 +1,170 @@
+---
+# Deploys one app to the k3s cluster: layer its config, provision its
+# database if it declares one, render its secrets into a Secret, then render
+# whatever manifests the app ships into k3s's auto-deploying directory.
+#
+# The shape deliberately mirrors compose_stack/deploy.yml, because the
+# separation it enforces is the same one: config that's committed, secrets
+# that never are. On Unraid the split is "static docker-compose.yml
+# referencing ${VAR}" + "rendered .env"; here it's "manifests carrying only
+# non-secret config" + "a rendered Secret the manifests reference by name".
+# In both cases the committed half is safe to read and the generated half
+# never lands in git.
+#
+# Included from main.yml when the app's `state:` is `present`; the facts it
+# relies on (app_local_dir) are set there.
+
+- name: Load portable app variables
+  ansible.builtin.include_vars:
+    file: "{{ app_local_dir }}/common/vars.yml"
+    name: app_common_vars
+
+- name: Load Kubernetes-specific app variables
+  ansible.builtin.include_vars:
+    file: "{{ app_local_dir }}/ansible/{{ k3s_app_platform_dir }}/vars.yml"
+    name: app_platform_vars
+
+# Portable values first, platform overrides on top — same precedence as
+# compose_stack's env_defaults merge, and the same reason: an app's ports and
+# database name don't change with the platform, its storage class and
+# ingress class do. Vault secrets are *not* merged in here; they go to the
+# Secret in a separate task below, so a manifest template can never
+# accidentally interpolate one into a world-readable file.
+- name: Merge app configuration
+  ansible.builtin.set_fact:
+    app_config: >-
+      {{ (app_common_vars.env_defaults | default({}))
+         | combine(app_platform_vars.env_defaults | default({})) }}
+
+- name: Look up app secrets from Vault
+  ansible.builtin.set_fact:
+    app_vault_secrets: >-
+      {{ lookup('community.hashi_vault.vault_kv2_get',
+                 app.vault_path,
+                 engine_mount_point=vault_kv_mount,
+                 url=vault_addr,
+                 auth_method=vault_auth_method,
+                 role_id=vault_role_id | default(omit),
+                 secret_id=vault_secret_id | default(omit)).secret }}
+  no_log: true
+
+# --- Database ---------------------------------------------------------------
+#
+# Identical to compose_stack's block, pointed at a different Postgres. That
+# it *can* be identical is the whole point of roles/k3s_postgres publishing
+# the CNPG primary on a MetalLB LoadBalancer: `community.postgresql` runs on
+# the controller (delegate_to: localhost) and needs a real host:port, which
+# an in-cluster ClusterIP Service isn't. See CLAUDE.md → "Key decisions" —
+# this is the deploy path that entry says is missing.
+#
+# provision_host is the LoadBalancer address, not the -rw ClusterIP; the app
+# itself still talks to the ClusterIP by DNS (see the app's vars.yml), so
+# the LAN-facing address is only ever used by the controller at deploy time.
+- name: Provision application database
+  when: app.db is defined
+  no_log: true
+  block:
+    - name: Look up Postgres superuser credentials from Vault
+      ansible.builtin.set_fact:
+        pg_admin_secrets: >-
+          {{ lookup('community.hashi_vault.vault_kv2_get',
+                     app.db.admin_vault_path,
+                     engine_mount_point=vault_kv_mount,
+                     url=vault_addr,
+                     auth_method=vault_auth_method,
+                     role_id=vault_role_id | default(omit),
+                     secret_id=vault_secret_id | default(omit)).secret }}
+
+    - name: Ensure application database role exists
+      community.postgresql.postgresql_user:
+        name: "{{ app.db.user }}"
+        password: "{{ app_vault_secrets[app.db.password_vault_key] }}"
+        login_host: "{{ app.db.provision_host }}"
+        login_port: "{{ app.db.provision_port }}"
+        login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}"
+        login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}"
+        state: present
+      delegate_to: localhost
+      become: false
+
+    # After the role, not before, and with owner: — unlike compose_stack,
+    # which creates the database first and then grants on it. The difference
+    # is that this database may be restored into from a pg_dump taken
+    # elsewhere (see docs/authentik-migration.md): a dump recreates objects
+    # with their original ownership, which only resolves if the owning role
+    # already exists and owns the database. Creating it owner-less and
+    # granting after works for an empty database and quietly leaves a
+    # restored one owned by postgres.
+    - name: Ensure application database exists
+      community.postgresql.postgresql_db:
+        name: "{{ app.db.name }}"
+        owner: "{{ app.db.user }}"
+        login_host: "{{ app.db.provision_host }}"
+        login_port: "{{ app.db.provision_port }}"
+        login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}"
+        login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}"
+        state: present
+      delegate_to: localhost
+      become: false
+
+# --- Manifests --------------------------------------------------------------
+
+# Namespace + Secret in one file, so the namespace an app's manifests target
+# is guaranteed to be created by something even if the app ships only a
+# HelmChart CR (whose createNamespace: true fires too late for a Secret the
+# chart's pods mount). Ordering between files doesn't otherwise matter here:
+# k3s's deploy controller retries a manifest whose namespace or CRDs don't
+# exist yet rather than failing once and giving up — the same property
+# roles/k3s_postgres and roles/k3s_metallb already rely on.
+#
+# 0600 and no_log because this one carries every value from the app's Vault
+# path. Note this is on-disk protection on the node only: the Secret's
+# contents are then base64 in etcd like any Kubernetes Secret, which is the
+# same trust boundary the rest of this cluster already assumes.
+- name: Render the app namespace and secrets manifest
+  ansible.builtin.template:
+    src: app-secrets.yaml.j2
+    dest: "{{ k3s_manifests_dir }}/{{ app.name }}-secrets.yaml"
+    owner: root
+    group: root
+    mode: "0600"
+  become: true
+  no_log: true
+
+# Whatever the app ships — a HelmChart CR, an Ingress, a PVC, a Certificate.
+# Enumerated from the repo checkout on the controller (fileglob evaluates
+# locally), so adding a manifest to an app means dropping a .yaml.j2 next to
+# the others, with no role change.
+#
+# Prefixed with the app name on the node, because every app's manifests share
+# one flat directory there. Name the files for their content and not for the
+# app (helmchart.yaml.j2, not authentik.helmchart.yaml.j2) — the prefix is
+# added here, and an app-named file stutters into authentik-authentik.yaml.
+- name: Find the app's Kubernetes manifest templates
+  ansible.builtin.set_fact:
+    app_manifest_templates: >-
+      {{ query('fileglob',
+               app_local_dir ~ '/ansible/' ~ k3s_app_platform_dir ~ '/*.yaml.j2')
+         | sort }}
+
+- name: Fail fast if the app ships no manifests
+  ansible.builtin.assert:
+    that: app_manifest_templates | length > 0
+    fail_msg: >-
+      App '{{ app.name }}' has no *.yaml.j2 under
+      src/{{ app.src }}/ansible/{{ k3s_app_platform_dir }}/ — nothing to
+      deploy. An app on this platform needs at least one manifest (normally
+      a HelmChart CR).
+    quiet: true
+
+- name: Render the app's Kubernetes manifests
+  ansible.builtin.template:
+    src: "{{ item }}"
+    dest: >-
+      {{ k3s_manifests_dir }}/{{ app.name }}-{{
+         item | basename | regex_replace('\.j2$', '') }}
+    owner: root
+    group: root
+    mode: "0644"
+  loop: "{{ app_manifest_templates }}"
+  become: true

build/config/ansible/roles/k3s_app/tasks/main.yml

@@ -0,0 +1,37 @@
+---
+# Converges one app (one loop iteration of `app` from playbooks/k3s.yml) to
+# the state its k3s_apps: entry asks for.
+#
+# `state: present` (the default) deploys; `state: absent` tears down. Same
+# explicit-removal rule as compose_stack, and for the same reason: Ansible
+# keeps no record of what it deployed last run, so deleting an app from
+# `k3s_apps:` only stops the loop visiting it — the workload keeps running,
+# unmanaged, until something says `absent`. The entry stays as a tombstone.
+#
+# Unlike compose_stack, removal here does tear the workload down completely
+# on the first pass, because k3s's deploy controller owns the resources a
+# manifest created and garbage-collects them when the file goes away. What
+# it does *not* touch, by the same "a default teardown should be
+# reversible" reasoning compose_stack applies: the app's database, its
+# PersistentVolumeClaims, and its Vault path.
+
+- name: Set app facts
+  ansible.builtin.set_fact:
+    app_local_dir: "{{ repo_root }}/src/{{ app.src }}"
+    app_state: "{{ app.state | default('present') }}"
+
+- name: Validate requested app state
+  ansible.builtin.assert:
+    that: app_state in ['present', 'absent']
+    fail_msg: >-
+      App '{{ app.name }}' has state '{{ app_state }}'; expected 'present'
+      or 'absent'.
+    quiet: true
+
+- name: Deploy app
+  ansible.builtin.include_tasks: deploy.yml
+  when: app_state == 'present'
+
+- name: Remove app
+  ansible.builtin.include_tasks: remove.yml
+  when: app_state == 'absent'

build/config/ansible/roles/k3s_app/tasks/remove.yml

@@ -0,0 +1,60 @@
+---
+# Tears one app off the cluster: delete the manifests it was deployed from
+# and let k3s's deploy controller garbage-collect what they created.
+#
+# This is the one place k3s is *less* work than Compose. `compose_stack`
+# has to run `docker compose down` and therefore needs the compose file to
+# still be on disk to know what it's tearing down (which is why removing an
+# app there is a two-pass job — see CLAUDE.md → "Removing an app"). Here the
+# deploy controller already tracks which resources each manifest file
+# created, via the Addon CR it writes alongside them, so deleting the file
+# is the teardown. src/<app>/ can be deleted in the same commit.
+#
+# Deliberately *not* removed, same reasoning as compose_stack's opt-in
+# flags — a default teardown should be reversible:
+#
+#   - the app's database and role on the shared Postgres
+#   - its PersistentVolumeClaims (the app's manifests own those; if the
+#     chart's PVCs carry a Helm ownership annotation they go with the
+#     HelmChart CR, so check `kubectl -n <ns> get pvc` after)
+#   - its Vault path
+#   - the namespace, which the secrets manifest below creates but the
+#     controller will only remove if nothing else landed in it
+#
+# There's no `remove_database`/`remove_volumes` equivalent yet. Add one the
+# day it's actually wanted rather than guessing at the shape now.
+
+- name: Find the app's Kubernetes manifest templates
+  ansible.builtin.set_fact:
+    app_manifest_templates: >-
+      {{ query('fileglob',
+               app_local_dir ~ '/ansible/' ~ k3s_app_platform_dir ~ '/*.yaml.j2')
+         | sort }}
+
+# The secrets manifest first, then the app's own. Order is cosmetic — the
+# controller reconciles each file's removal independently — but removing the
+# workload's namespace/Secret last would leave pods briefly running without
+# the credentials they were started with, and losing them noisily in a log
+# is worse than losing them quietly.
+- name: Remove the app's Kubernetes manifests
+  ansible.builtin.file:
+    path: >-
+      {{ k3s_manifests_dir }}/{{ app.name }}-{{
+         item | basename | regex_replace('\.j2$', '') }}
+    state: absent
+  loop: "{{ app_manifest_templates }}"
+  become: true
+
+- name: Remove the app namespace and secrets manifest
+  ansible.builtin.file:
+    path: "{{ k3s_manifests_dir }}/{{ app.name }}-secrets.yaml"
+    state: absent
+  become: true
+
+- name: Report what removal left behind
+  ansible.builtin.debug:
+    msg: >-
+      App '{{ app.name }}' manifests removed; k3s will garbage-collect the
+      resources they created. Its database, PVCs and Vault path were left
+      alone on purpose — remove those by hand if the teardown is meant to be
+      permanent.

build/config/ansible/roles/k3s_app/templates/app-secrets.yaml.j2

@@ -0,0 +1,35 @@
+{#
+  Managed by Ansible (roles/k3s_app) — do not edit on the node.
+
+  The k3s counterpart of compose_stack's rendered .env: every key from the
+  app's Vault path, verbatim, in one Secret the app's manifests reference by
+  name. Keys are not renamed or filtered on the way through, which is what
+  keeps this role generic — an app decides what its environment looks like by
+  choosing its Vault keys, exactly as it does on Unraid, and the role stays
+  ignorant of any particular app's variables.
+
+  The intended consumer is an envFrom/secretRef in the app's own manifest,
+  so the values become environment variables without ever being named in a
+  committed file. See src/authentik/ansible/kubernetes/ for the worked
+  example.
+
+  Namespace lives here rather than in the app's manifests so it's guaranteed
+  to exist before anything mounts this Secret — a HelmChart CR's
+  createNamespace: true happens when the chart installs, which is after the
+  helm-controller job needs somewhere to put it.
+-#}
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ app_config.K8S_NAMESPACE }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ app.name }}-secrets
+  namespace: {{ app_config.K8S_NAMESPACE }}
+type: Opaque
+stringData:
+{% for key, value in (app_vault_secrets | default({})) | dictsort %}
+  {{ key }}: {{ value | string | to_json }}
+{% endfor %}

build/config/ansible/roles/k3s_cert_manager/defaults/main.yml

@@ -0,0 +1,73 @@
+---
+# Ansible-side knobs for roles/k3s_cert_manager — cert-manager, which issues
+# and renews the TLS certificates Ingresses on this cluster serve. Installed
+# via k3s's own bundled helm-controller, same mechanism as roles/k3s_metallb,
+# k3s_monitoring and k3s_postgres (see tasks/main.yml) — no helm binary or
+# extra Ansible collection needed on the controller.
+#
+# This is the fourth cluster service, and the first one that exists purely
+# for apps rather than for the cluster: nothing here needs a certificate,
+# roles/k3s_app's tenants do. It's still a cluster service rather than an
+# app, because a certificate issuer is shared infrastructure in the same way
+# the shared Postgres is — one ClusterIssuer, every app's Ingress annotates
+# itself against it.
+
+k3s_cert_manager_namespace: cert-manager
+
+k3s_cert_manager_chart_repo: https://charts.jetstack.io
+
+# Pinned, same reasoning as every other chart version here — a rebuild
+# months from now should reproduce today's install, not whatever's newest.
+# Bump deliberately; check the current release first at
+# https://github.com/cert-manager/cert-manager/releases.
+k3s_cert_manager_chart_version: "v1.21.1"
+
+# The name every app's Ingress annotates itself with
+# (cert-manager.io/cluster-issuer: <this>). A ClusterIssuer rather than a
+# per-namespace Issuer, so an app doesn't need its own copy of the ACME
+# account and DNS credentials in its own namespace.
+k3s_cert_manager_issuer_name: letsencrypt
+
+# Let's Encrypt's production directory. Swap for the staging URL
+# (https://acme-staging-v02.api.letsencrypt.org/directory) while working out
+# a solver configuration: production has a hard rate limit of 5 failed
+# validations per account/hostname/hour, and burning it means waiting rather
+# than retrying. Changing this changes which ACME account the issuer uses, so
+# existing certificates are re-issued rather than renewed.
+k3s_cert_manager_acme_server: https://acme-v02.api.letsencrypt.org/directory
+
+# The address Let's Encrypt sends expiry warnings to. No default — an ACME
+# account is registered against it, so a wrong value is worth failing on
+# rather than guessing. Set in inventory/group_vars/k3s_cluster.yml.
+k3s_cert_manager_acme_email: ""
+
+# Where the ACME account's private key is kept. cert-manager generates it on
+# first registration; it is *not* the certificate key, and losing it means
+# re-registering rather than losing certificates.
+k3s_cert_manager_acme_key_secret: letsencrypt-account-key
+
+# --- Solver -----------------------------------------------------------------
+#
+# How cert-manager proves control of the domain. DNS-01 by default, not
+# HTTP-01, because HTTP-01 requires Let's Encrypt to reach this cluster from
+# the internet on port 80 — true only if the LAN is port-forwarded, and this
+# cluster deliberately isn't. DNS-01 needs no inbound path at all, and is the
+# only option that can issue wildcards.
+#
+# The provider block is left open rather than hard-coding Cloudflare, since
+# it's the one part of this role that depends on who runs the DNS. Set
+# k3s_cert_manager_solver in inventory/group_vars/k3s_cluster.yml to whatever
+# your provider's stanza looks like in cert-manager's docs
+# (https://cert-manager.io/docs/configuration/acme/dns01/); it's rendered
+# into the ClusterIssuer's solvers list as-is. Any secret it references
+# should name the Secret this role renders from Vault (below), so no token is
+# ever committed. tasks/main.yml fails fast while this is empty.
+k3s_cert_manager_solver: {}
+
+# The Vault path holding the DNS provider's API credentials. Every key at
+# this path becomes a key in a Secret named after it in the cert-manager
+# namespace, the same "Vault keys pass through verbatim" rule roles/k3s_app
+# uses — so the solver above references whichever key name you stored.
+# Leave empty for a solver that needs no credentials (e.g. HTTP-01).
+k3s_cert_manager_vault_path: homelab/k3s-cert-manager
+k3s_cert_manager_credentials_secret: cert-manager-dns-credentials

build/config/ansible/roles/k3s_cert_manager/tasks/main.yml

@@ -0,0 +1,75 @@
+---
+# Drops a HelmChart CR for cert-manager, plus a ClusterIssuer and the DNS
+# credentials it solves with, into k3s's auto-deploying manifests directory;
+# k3s's bundled helm-controller and deploy controller reconcile them — same
+# mechanism and same two-manifest shape as roles/k3s_metallb (chart CR +
+# plain config manifest referencing CRDs the chart hasn't installed yet).
+#
+# The ClusterIssuer is exactly that case: cert-manager.io/v1 doesn't exist
+# until the chart has installed, so this manifest is unappliable at the
+# moment it's written. That's fine and deliberate — k3s's deploy controller
+# retries a manifest referencing not-yet-existing CRDs until they show up,
+# rather than failing once and giving up.
+
+- name: Fail fast if cert-manager is not configured
+  ansible.builtin.assert:
+    that:
+      - k3s_cert_manager_acme_email | length > 0
+      - k3s_cert_manager_solver | length > 0
+    fail_msg: >-
+      k3s_cert_manager_acme_email and/or k3s_cert_manager_solver are unset —
+      set both in inventory/group_vars/k3s_cluster.yml before running
+      playbooks/k3s.yml. An ACME account is registered against the email, and
+      without a solver the ClusterIssuer would be created but never able to
+      prove domain control, leaving every Certificate pending indefinitely
+      rather than failing loudly.
+    quiet: true
+  run_once: true
+
+- name: Deploy the cert-manager HelmChart manifest
+  ansible.builtin.template:
+    src: cert-manager.helmchart.yaml.j2
+    dest: /var/lib/rancher/k3s/server/manifests/cert-manager.yaml
+    owner: root
+    group: root
+    mode: "0644"
+  become: true
+
+# Before the ClusterIssuer, because that's what references it. Ordering
+# between files isn't actually enforced (see the header) — this is for the
+# reader, not the controller.
+- name: Look up the DNS provider credentials from Vault
+  when: k3s_cert_manager_vault_path | length > 0
+  ansible.builtin.set_fact:
+    cert_manager_vault_secrets: >-
+      {{ lookup('community.hashi_vault.vault_kv2_get',
+                 k3s_cert_manager_vault_path,
+                 engine_mount_point=vault_kv_mount,
+                 url=vault_addr,
+                 auth_method=vault_auth_method,
+                 role_id=vault_role_id | default(omit),
+                 secret_id=vault_secret_id | default(omit)).secret }}
+  no_log: true
+
+# Separate file and mode from everything else here, same split
+# roles/k3s_postgres makes: this is the only manifest carrying a credential,
+# so it's the only one that isn't safe at 0644.
+- name: Deploy the DNS provider credentials Secret
+  when: k3s_cert_manager_vault_path | length > 0
+  ansible.builtin.template:
+    src: dns-credentials.yaml.j2
+    dest: /var/lib/rancher/k3s/server/manifests/cert-manager-dns-credentials.yaml
+    owner: root
+    group: root
+    mode: "0600"
+  become: true
+  no_log: true
+
+- name: Deploy the ClusterIssuer manifest
+  ansible.builtin.template:
+    src: cluster-issuer.yaml.j2
+    dest: /var/lib/rancher/k3s/server/manifests/cert-manager-cluster-issuer.yaml
+    owner: root
+    group: root
+    mode: "0644"
+  become: true

build/config/ansible/roles/k3s_cert_manager/templates/cert-manager.helmchart.yaml.j2

@@ -0,0 +1,59 @@
+{#
+  Managed by Ansible (roles/k3s_cert_manager) — do not edit on the node.
+
+  A HelmChart CR, same pattern as roles/k3s_metallb and roles/k3s_monitoring:
+  k3s's bundled helm-controller reconciles it, the same mechanism it uses to
+  install its own Traefik.
+-#}
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: cert-manager
+  namespace: kube-system
+spec:
+  chart: cert-manager
+  repo: {{ k3s_cert_manager_chart_repo }}
+  version: "{{ k3s_cert_manager_chart_version }}"
+  targetNamespace: {{ k3s_cert_manager_namespace }}
+  createNamespace: true
+  valuesContent: |-
+    # The chart installs its own CRDs. Without this they'd have to be applied
+    # separately before the chart, which the helm-controller gives no ordering
+    # hook for — and the ClusterIssuer this role also renders would then never
+    # become appliable.
+    crds:
+      enabled: true
+
+    # Sized for a Raspberry Pi 4, same reasoning as k3s_monitoring_values and
+    # k3s_postgres_resources. cert-manager is idle almost all of the time —
+    # it wakes to renew a certificate every 60 days — so these are set for
+    # "doesn't get OOM-killed during a renewal", not for throughput.
+    resources:
+      requests:
+        cpu: 10m
+        memory: 64Mi
+      limits:
+        memory: 128Mi
+    webhook:
+      resources:
+        requests:
+          cpu: 10m
+          memory: 64Mi
+        limits:
+          memory: 128Mi
+    cainjector:
+      resources:
+        requests:
+          cpu: 10m
+          memory: 96Mi
+        limits:
+          memory: 192Mi
+
+    # DNS-01 self-check queries the authoritative nameservers directly rather
+    # than going through the cluster's resolver. Without this, CoreDNS
+    # forwards to whatever the LAN's DNS is, and a split-horizon setup —
+    # where the internal view of the zone doesn't carry the _acme-challenge
+    # TXT record the public view does — makes cert-manager wait for a record
+    # it will never see, until the order times out.
+    dns01RecursiveNameservers: "1.1.1.1:53,9.9.9.9:53"
+    dns01RecursiveNameserversOnly: true

build/config/ansible/roles/k3s_cert_manager/templates/cluster-issuer.yaml.j2

@@ -0,0 +1,30 @@
+{#
+  Managed by Ansible (roles/k3s_cert_manager) — do not edit on the node.
+
+  Plain manifest, not a HelmChart — same pattern as
+  roles/k3s_metallb/templates/metallb-config.yaml.j2 and
+  roles/k3s_postgres/templates/postgres-cluster.yaml.j2: this references a
+  CRD (cert-manager.io) that only exists once the chart alongside it has
+  installed, and k3s's deploy controller retries rather than giving up.
+
+  ClusterIssuer rather than Issuer so it's usable from every namespace —
+  an app's Ingress just annotates itself with
+  `cert-manager.io/cluster-issuer: {{ k3s_cert_manager_issuer_name }}` and
+  needs no ACME account or DNS credentials of its own.
+-#}
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: {{ k3s_cert_manager_issuer_name }}
+spec:
+  acme:
+    server: {{ k3s_cert_manager_acme_server }}
+    email: {{ k3s_cert_manager_acme_email }}
+    # Where cert-manager keeps the ACME *account* key it generates on first
+    # registration — not any certificate's key. In the cert-manager namespace
+    # because a ClusterIssuer's secrets always resolve there, regardless of
+    # which namespace the Certificate using it lives in.
+    privateKeySecretRef:
+      name: {{ k3s_cert_manager_acme_key_secret }}
+    solvers:
+{{ [k3s_cert_manager_solver] | to_nice_yaml(indent=2) | indent(6, first=true) }}

build/config/ansible/roles/k3s_cert_manager/templates/dns-credentials.yaml.j2

@@ -0,0 +1,23 @@
+{#
+  Managed by Ansible (roles/k3s_cert_manager) — do not edit on the node.
+
+  The DNS provider API credentials the ACME DNS-01 solver authenticates with,
+  passed through from Vault verbatim — same rule roles/k3s_app's Secret
+  follows, so the key names in k3s_cert_manager_solver are whatever you chose
+  when populating the Vault path, and this role stays ignorant of which
+  provider is in use.
+
+  In the cert-manager namespace because a ClusterIssuer resolves every Secret
+  it references there, never in the namespace of the Certificate being
+  issued.
+-#}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ k3s_cert_manager_credentials_secret }}
+  namespace: {{ k3s_cert_manager_namespace }}
+type: Opaque
+stringData:
+{% for key, value in (cert_manager_vault_secrets | default({})) | dictsort %}
+  {{ key }}: {{ value | string | to_json }}
+{% endfor %}

build/config/ansible/roles/k3s_maintenance/tasks/main.yml

@@ -0,0 +1,100 @@
+---
+# Rolling "reboot this node if unattended-upgrades left one pending" for one
+# k3s node — one iteration of the serial: 1 loop in
+# playbooks/k3s_maintenance.yml. roles/unattended_upgrades installs updates on
+# its own schedule with Automatic-Reboot disabled, so a kernel/library update
+# can sit applied-but-inactive on a node indefinitely; this is what actually
+# reboots it — cordoned and drained first, so workloads move off before the
+# node disappears rather than during.
+#
+# kubectl commands are delegated to the control-plane node and run as
+# `k3s kubectl`, k3s's own bundled client — no separate kubectl install or
+# local kubeconfig needed. Works the same whether the node having its turn
+# right now *is* the control plane: delegating to itself, over SSH, before it
+# reboots itself.
+
+- name: Sanity-check there is exactly one control-plane node
+  ansible.builtin.assert:
+    that: groups['k3s_control_plane'] | length == 1
+    fail_msg: >-
+      k3s_maintenance delegates kubectl to groups['k3s_control_plane'][0] —
+      it assumes a single control-plane node. Update this role before adding
+      a second one for HA.
+    quiet: true
+
+- name: Check whether a reboot is required
+  ansible.builtin.stat:
+    path: /var/run/reboot-required
+  register: k3s_reboot_required
+
+- name: Node is up to date — nothing to do
+  ansible.builtin.debug:
+    msg: "{{ inventory_hostname }}: no reboot required, skipping."
+  when: not k3s_reboot_required.stat.exists
+
+- name: Reboot this node if unattended-upgrades left one pending
+  when: k3s_reboot_required.stat.exists
+  block:
+    # --force: this is a homelab, not a cluster with a policy against bare
+    # pods — better to evict them than have a stray one block every rebuild.
+    # --delete-emptydir-data: emptyDir contents are expected to be
+    # disposable; anything that isn't shouldn't be using emptyDir.
+    - name: Cordon and drain the node
+      ansible.builtin.command:
+        argv:
+          - k3s
+          - kubectl
+          - drain
+          - "{{ inventory_hostname }}"
+          - --ignore-daemonsets
+          - --delete-emptydir-data
+          - --force
+          - --timeout=120s
+      delegate_to: "{{ groups['k3s_control_plane'][0] }}"
+      become: false
+      changed_when: true
+
+    - name: Reboot the node
+      ansible.builtin.reboot:
+        reboot_timeout: 300
+
+    # Polls rather than trusting the reboot handshake alone — the node can be
+    # reachable over SSH before k3s (and, if this is the control-plane node
+    # itself, the API server it just took down with it) has finished coming
+    # back up.
+    - name: Wait for the node to report Ready again
+      ansible.builtin.command:
+        argv:
+          - k3s
+          - kubectl
+          - wait
+          - --for=condition=Ready
+          - "node/{{ inventory_hostname }}"
+          - --timeout=20s
+      delegate_to: "{{ groups['k3s_control_plane'][0] }}"
+      become: false
+      register: k3s_node_ready
+      changed_when: false
+      failed_when: false
+      until: k3s_node_ready.rc == 0
+      retries: 12
+      delay: 15
+
+    - name: Fail if the node never came back Ready
+      ansible.builtin.fail:
+        msg: >-
+          {{ inventory_hostname }} rebooted but never reported Ready again —
+          it's left cordoned; check it by hand before re-running this against
+          the rest of the cluster.
+      when: k3s_node_ready.rc != 0
+
+    - name: Uncordon the node
+      ansible.builtin.command:
+        argv: [k3s, kubectl, uncordon, "{{ inventory_hostname }}"]
+      delegate_to: "{{ groups['k3s_control_plane'][0] }}"
+      become: false
+      changed_when: true
+
+    - name: Node rebooted and rejoined the cluster
+      ansible.builtin.debug:
+        msg: "{{ inventory_hostname }}: rebooted, drained and uncordoned cleanly."

build/config/ansible/roles/k3s_metallb/defaults/main.yml

@@ -0,0 +1,31 @@
+---
+# Ansible-side knobs for roles/k3s_metallb — MetalLB, the LoadBalancer
+# implementation for the homelab-utils cluster. Installed via k3s's own
+# bundled helm-controller, same mechanism as roles/k3s_monitoring (see
+# tasks/main.yml) — no helm binary or extra Ansible collection needed on the
+# controller.
+#
+# MetalLB replaces k3s's bundled ServiceLB (Klipper), it doesn't sit
+# alongside it — both would otherwise try to satisfy the same LoadBalancer
+# Services and fight over IPs. That's why `k3s_extra_args` in
+# inventory/group_vars/k3s_cluster.yml carries `--disable=servicelb`.
+
+k3s_metallb_namespace: metallb-system
+
+k3s_metallb_chart_repo: https://metallb.github.io/metallb
+
+# Pinned, same reasoning as k3s_monitoring_chart_version — a rebuild months
+# from now should reproduce today's stack, not whatever's newest at the
+# time. Bump deliberately; check the current release first at
+# https://github.com/metallb/metallb/releases.
+k3s_metallb_chart_version: "0.14.9"
+
+# The pool of IPs MetalLB hands out to LoadBalancer-type Services, as a
+# "first-last" range or CIDR (e.g. "192.168.50.240-192.168.50.250"). Must be
+# addresses on the cluster's LAN that nothing else — DHCP, static
+# assignments, the hosts in inventory/hosts.yml — will ever claim: MetalLB
+# doesn't coordinate with your router, it just hands out whatever's in this
+# range. No default on purpose; set it in
+# inventory/group_vars/k3s_cluster.yml before the first run. tasks/main.yml
+# fails fast if it's still empty.
+k3s_metallb_address_range: ""

build/config/ansible/roles/k3s_metallb/tasks/main.yml

@@ -0,0 +1,39 @@
+---
+# Drops a HelmChart CR for MetalLB, plus its IPAddressPool/L2Advertisement
+# config, into k3s's auto-deploying manifests directory; k3s's bundled
+# helm-controller and deploy controller reconcile them — same mechanism as
+# roles/k3s_monitoring (see its tasks/main.yml for why this needs no helm
+# binary or kubeconfig on the controller).
+#
+# The config manifest references CRDs that only exist once the HelmChart
+# above has actually installed the chart, so it's templated in the same
+# pass rather than gated behind a "wait for CRDs" step: k3s's deploy
+# controller retries a manifest referencing not-yet-existing CRDs until they
+# show up, instead of failing once and giving up.
+
+- name: Fail fast if no MetalLB address range is configured
+  ansible.builtin.assert:
+    that:
+      - k3s_metallb_address_range | length > 0
+    fail_msg: >-
+      k3s_metallb_address_range is empty — set it in
+      inventory/group_vars/k3s_cluster.yml before running playbooks/k3s.yml.
+  run_once: true
+
+- name: Deploy the MetalLB HelmChart manifest
+  ansible.builtin.template:
+    src: metallb.helmchart.yaml.j2
+    dest: /var/lib/rancher/k3s/server/manifests/metallb.yaml
+    owner: root
+    group: root
+    mode: "0644"
+  become: true
+
+- name: Deploy the MetalLB address pool config
+  ansible.builtin.template:
+    src: metallb-config.yaml.j2
+    dest: /var/lib/rancher/k3s/server/manifests/metallb-config.yaml
+    owner: root
+    group: root
+    mode: "0644"
+  become: true

build/config/ansible/roles/k3s_metallb/templates/metallb-config.yaml.j2

@@ -0,0 +1,26 @@
+# Managed by Ansible (roles/k3s_metallb) — do not edit on the node.
+#
+# Plain manifests, not a HelmChart — these are MetalLB's own CRs, not chart
+# values. k3s's deploy controller applies anything dropped in this
+# directory, not only HelmChart CRs, and retries until the CRDs the
+# HelmChart alongside this file installs actually exist.
+#
+# L2 (ARP/NDP) mode, not BGP — the simplest fit for a flat home LAN with no
+# BGP-speaking router. One pool, one advertisement, both named "default".
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  name: default
+  namespace: {{ k3s_metallb_namespace }}
+spec:
+  addresses:
+    - {{ k3s_metallb_address_range }}
+---
+apiVersion: metallb.io/v1beta1
+kind: L2Advertisement
+metadata:
+  name: default
+  namespace: {{ k3s_metallb_namespace }}
+spec:
+  ipAddressPools:
+    - default

build/config/ansible/roles/k3s_metallb/templates/metallb.helmchart.yaml.j2

@@ -0,0 +1,18 @@
+# Managed by Ansible (roles/k3s_metallb) — do not edit on the node.
+#
+# A HelmChart CR, same pattern as
+# roles/k3s_monitoring/templates/kube-prometheus-stack.helmchart.yaml.j2 —
+# k3s's bundled helm-controller reconciles it, the same mechanism it uses to
+# install its own bundled Traefik and ServiceLB. No helm binary, kubeconfig,
+# or extra Ansible collection needed on the controller.
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: metallb
+  namespace: kube-system
+spec:
+  chart: metallb
+  repo: {{ k3s_metallb_chart_repo }}
+  version: "{{ k3s_metallb_chart_version }}"
+  targetNamespace: {{ k3s_metallb_namespace }}
+  createNamespace: true

build/config/ansible/roles/k3s_monitoring/defaults/main.yml

@@ -0,0 +1,86 @@
+---
+# Ansible-side knobs for roles/k3s_monitoring — the kube-prometheus-stack
+# Helm chart, deployed via k3s's own bundled helm-controller rather than a
+# helm binary or the kubernetes.core collection (see tasks/main.yml). Lean by
+# design: OpenLens's Metrics feature reads straight off the Prometheus API
+# through the k8s API server proxy, so there's no need for Grafana or
+# Alertmanager in-cluster — both are disabled below, mainly to save RAM on
+# the Pis.
+
+k3s_monitoring_namespace: monitoring
+
+k3s_monitoring_chart_repo: https://prometheus-community.github.io/helm-charts
+
+# Pinned, same reasoning as k3s_version in inventory/group_vars/k3s_cluster.yml
+# — a rebuild months from now should reproduce today's stack, not whatever's
+# newest at the time. Bump deliberately; check the current release first at
+# https://github.com/prometheus-community/helm-charts/releases (tags look
+# like kube-prometheus-stack-X.Y.Z).
+k3s_monitoring_chart_version: "88.3.0"
+
+# Handed to the chart verbatim as valuesContent (see templates/). Notes on
+# the less obvious choices:
+#  - grafana/alertmanager: off. OpenLens supplies its own dashboards and
+#    this cluster doesn't page anyone, so both would just be extra Pi RAM.
+#  - kubeControllerManager/kubeScheduler/kubeProxy/kubeEtcd: off. k3s bundles
+#    the control plane inside one static binary instead of exposing these as
+#    separate systemd units on their usual ports, so kube-prometheus-stack's
+#    default scrape targets for them sit permanently "down" — a known false
+#    alarm on k3s, not a sign anything's actually broken. kubelet (and the
+#    cAdvisor/node metrics it serves) is the one k3s does expose normally,
+#    and it's also the one OpenLens's node/pod metrics actually need.
+#  - prometheus retention/storage: short retention, no PVC template (so it
+#    runs on emptyDir). This stack exists to answer "what are the nodes
+#    doing right now", not to keep months of history, so losing the TSDB on
+#    a pod restart is an acceptable trade for not standing up persistent
+#    storage across 4 SD cards.
+#  - every component's resources: sized for a Raspberry Pi 4, not a
+#    datacenter node.
+k3s_monitoring_values:
+  grafana:
+    enabled: false
+  alertmanager:
+    enabled: false
+  kubeApiServer:
+    enabled: true
+  kubeControllerManager:
+    enabled: false
+  kubeScheduler:
+    enabled: false
+  kubeProxy:
+    enabled: false
+  kubeEtcd:
+    enabled: false
+  kubelet:
+    enabled: true
+  prometheusOperator:
+    resources:
+      requests:
+        cpu: 50m
+        memory: 64Mi
+      limits:
+        memory: 128Mi
+  prometheus:
+    prometheusSpec:
+      retention: 3d
+      scrapeInterval: 30s
+      resources:
+        requests:
+          cpu: 100m
+          memory: 256Mi
+        limits:
+          memory: 512Mi
+  kube-state-metrics:
+    resources:
+      requests:
+        cpu: 20m
+        memory: 32Mi
+      limits:
+        memory: 64Mi
+  prometheus-node-exporter:
+    resources:
+      requests:
+        cpu: 10m
+        memory: 16Mi
+      limits:
+        memory: 32Mi

build/config/ansible/roles/k3s_monitoring/tasks/main.yml

@@ -0,0 +1,16 @@
+---
+# Drops a HelmChart CR into k3s's auto-deploying manifests directory on the
+# control-plane node; k3s's bundled helm-controller reconciles it, the same
+# mechanism it uses to install its own bundled Traefik and ServiceLB. See
+# templates/kube-prometheus-stack.helmchart.yaml.j2 for why this needs no
+# helm binary or extra Ansible collection, and README.md "Metrics
+# (Prometheus)" for how to point OpenLens at the result.
+
+- name: Deploy the kube-prometheus-stack HelmChart manifest
+  ansible.builtin.template:
+    src: kube-prometheus-stack.helmchart.yaml.j2
+    dest: /var/lib/rancher/k3s/server/manifests/kube-prometheus-stack.yaml
+    owner: root
+    group: root
+    mode: "0644"
+  become: true

build/config/ansible/roles/k3s_monitoring/templates/kube-prometheus-stack.helmchart.yaml.j2

@@ -0,0 +1,24 @@
+---
+# Managed by Ansible (roles/k3s_monitoring) — do not edit on the node.
+#
+# A HelmChart CR, not a `helm install`: k3s ships its own helm-controller
+# that watches /var/lib/rancher/k3s/server/manifests/ (this file's
+# destination) the same way it watches for any other auto-deploying
+# manifest, and reconciles HelmChart resources found there — the same
+# mechanism k3s uses to install its own bundled Traefik and ServiceLB. That
+# means no helm binary, kubeconfig, or extra Ansible collection is needed on
+# the controller; re-running the playbook just re-templates this file and
+# the controller reconciles the diff.
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: kube-prometheus-stack
+  namespace: kube-system
+spec:
+  chart: kube-prometheus-stack
+  repo: {{ k3s_monitoring_chart_repo }}
+  version: "{{ k3s_monitoring_chart_version }}"
+  targetNamespace: {{ k3s_monitoring_namespace }}
+  createNamespace: true
+  valuesContent: |
+{{ k3s_monitoring_values | to_nice_yaml(indent=2) | indent(4, first=true) }}

build/config/ansible/roles/k3s_node/defaults/main.yml

@@ -0,0 +1,18 @@
+---
+# Ubuntu Server's boot partition. Same path on Raspberry Pi OS, so this
+# doesn't need to change if the Pis are ever reimaged to that instead.
+k3s_boot_cmdline_path: /boot/firmware/cmdline.txt
+
+# k3s (like any Kubernetes distro) needs the memory cgroup controller, which
+# isn't always on by default on a Pi kernel. Ubuntu's Pi images usually ship
+# these already — this task is a safety net for that assumption, not the
+# primary mechanism, and a no-op on most runs. See tasks/prep.yml.
+k3s_cgroup_params:
+  - cgroup_memory=1
+  - cgroup_enable=memory
+
+# Safety-net defaults for inventory/group_vars/k3s_cluster.yml's extra-args
+# vars (see tasks/server.yml, tasks/agent.yml) — empty so the role doesn't
+# break if a var is left undefined there.
+k3s_extra_args: []
+k3s_server_extra_args: []

build/config/ansible/roles/k3s_node/tasks/agent.yml

@@ -0,0 +1,100 @@
+---
+# Joins this node as a k3s agent (worker). Only ever runs after server.yml has
+# succeeded somewhere — playbooks/k3s.yml targets k3s_control_plane before
+# k3s_workers — so K3S_URL below always points at an already-live API server.
+#
+# --node-name pins the k8s node object to the Ansible inventory_hostname —
+# see the matching comment in server.yml, same reason.
+
+- name: Bootstrap the k3s agent
+  no_log: true # K3S_TOKEN passes through this block
+  block:
+    - name: Look up the k3s cluster secrets
+      ansible.builtin.set_fact:
+        k3s_secrets: >-
+          {{ lookup('community.hashi_vault.vault_kv2_get',
+                     k3s_vault_path,
+                     engine_mount_point=vault_kv_mount,
+                     url=vault_addr,
+                     auth_method=vault_auth_method,
+                     role_id=vault_role_id | default(omit),
+                     secret_id=vault_secret_id | default(omit)).secret }}
+
+    - name: Compute the desired k3s agent exec line
+      ansible.builtin.set_fact:
+        k3s_agent_exec: "agent --node-name {{ inventory_hostname }} {{ k3s_extra_args | join(' ') }}"
+
+    # Compared against what k3s was last installed with (see the copy task
+    # below) so a change to k3s_extra_args gets applied on the next run
+    # instead of silently sitting unused: the version check below has no way
+    # to notice an exec-line-only change. Missing file (first install) counts
+    # as changed.
+    - name: Read the exec line k3s was last installed with
+      ansible.builtin.slurp:
+        src: /etc/rancher/k3s/.ansible_install_exec
+      register: k3s_installed_exec_raw
+      failed_when: false
+      check_mode: false
+
+    - name: Determine whether the exec line has changed
+      ansible.builtin.set_fact:
+        k3s_exec_changed: >-
+          {{ k3s_installed_exec_raw.content is not defined
+             or (k3s_installed_exec_raw.content | b64decode) != k3s_agent_exec }}
+
+    - name: Check the installed k3s version
+      ansible.builtin.command: k3s --version
+      register: k3s_installed_version
+      changed_when: false
+      failed_when: false
+      check_mode: false
+
+    # The installer is safe to re-run — it's a no-op if the requested version
+    # and exec line are already active — but skipping it when neither changed
+    # avoids restarting the agent (and briefly dropping the node's kubelet)
+    # on every playbook run. Restarting the k3s-agent process itself (as
+    # opposed to rebooting the node, which roles/k3s_maintenance handles
+    # separately) doesn't touch already-running pods — containerd keeps them
+    # up underneath it. playbooks/k3s.yml still runs this play with
+    # serial: 1, so at most one node's kubelet is ever bouncing at a time.
+    - name: Install/upgrade k3s agent
+      ansible.builtin.shell: curl -sfL https://get.k3s.io | sh -
+      environment:
+        INSTALL_K3S_VERSION: "{{ k3s_version }}"
+        INSTALL_K3S_EXEC: "{{ k3s_agent_exec }}"
+        # The one control-plane node, addressed by its inventory IP rather
+        # than delegating a lookup to it — simple and correct as long as this
+        # cluster stays single-server. Revisit if it ever gets HA control
+        # plane nodes.
+        K3S_URL: "https://{{ hostvars[groups['k3s_control_plane'][0]].ansible_host }}:6443"
+        K3S_TOKEN: "{{ k3s_secrets.K3S_TOKEN }}"
+      when: >-
+        k3s_installed_version.rc != 0
+        or k3s_version not in k3s_installed_version.stdout
+        or k3s_exec_changed
+
+- name: Ensure the k3s-agent service is enabled and running
+  ansible.builtin.systemd_service:
+    name: k3s-agent
+    enabled: true
+    state: started
+
+# k3s creates /etc/rancher/k3s itself once it has something to put there
+# (a config.yaml, the generated kubeconfig on the server) — an agent with no
+# extra config doesn't necessarily end up with anything else prompting that,
+# so this marker can't assume the directory already exists.
+- name: Ensure /etc/rancher/k3s exists
+  ansible.builtin.file:
+    path: /etc/rancher/k3s
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Record the exec line k3s was installed with
+  ansible.builtin.copy:
+    dest: /etc/rancher/k3s/.ansible_install_exec
+    content: "{{ k3s_agent_exec }}"
+    owner: root
+    group: root
+    mode: "0600"

build/config/ansible/roles/k3s_node/tasks/main.yml

@@ -0,0 +1,24 @@
+---
+# Converges one node of the cluster: the same prep either way, then branch on
+# which k3s role it plays. k3s_node_role is set per group, not per host — see
+# inventory/group_vars/k3s_control_plane.yml (server) and k3s_workers.yml
+# (agent) — so adding a fifth Pi later is a hosts.yml edit, not a role change.
+
+- name: Validate k3s_node_role
+  ansible.builtin.assert:
+    that: k3s_node_role in ['server', 'agent']
+    fail_msg: >-
+      k3s_node_role is '{{ k3s_node_role }}'; expected 'server' or 'agent'.
+      Check inventory/group_vars/k3s_control_plane.yml and k3s_workers.yml.
+    quiet: true
+
+- name: Prepare the node for k3s
+  ansible.builtin.include_tasks: prep.yml
+
+- name: Install the k3s server (control plane)
+  ansible.builtin.include_tasks: server.yml
+  when: k3s_node_role == 'server'
+
+- name: Install the k3s agent (worker)
+  ansible.builtin.include_tasks: agent.yml
+  when: k3s_node_role == 'agent'

build/config/ansible/roles/k3s_node/tasks/prep.yml

@@ -0,0 +1,80 @@
+---
+# OS-level prerequisites, identical for a server and an agent node. Runs
+# before either install so a from-scratch Pi (fresh Ubuntu Server image, SSH
+# + the `ansible` user already set up) can go straight to a working cluster
+# in one playbook run — the "easy to rebuild" part of the design.
+
+- name: Read the current kernel boot parameters
+  ansible.builtin.command: cat {{ k3s_boot_cmdline_path }}
+  register: k3s_cmdline_current
+  changed_when: false
+  check_mode: false
+
+- name: Work out which cgroup parameters are missing
+  ansible.builtin.set_fact:
+    k3s_cmdline_missing: >-
+      {{ k3s_cgroup_params | reject('in', k3s_cmdline_current.stdout) | list }}
+
+# cmdline.txt is one line, space-separated — rewritten whole rather than
+# appended in place, since there's no line-based anchor to insert after.
+# Existing file mode is left alone (no `mode:` here) rather than guessed at.
+- name: Add missing cgroup parameters to the boot command line
+  ansible.builtin.copy:
+    dest: "{{ k3s_boot_cmdline_path }}"
+    content: >-
+      {{ (k3s_cmdline_current.stdout.split() + k3s_cmdline_missing) | join(' ') }}
+    owner: root
+    group: root
+  register: k3s_cmdline_updated
+  when: k3s_cmdline_missing | length > 0
+
+- name: Reboot to apply updated boot parameters
+  ansible.builtin.reboot:
+    reboot_timeout: 300
+  when: k3s_cmdline_updated is changed
+
+- name: Check active swap devices
+  ansible.builtin.command: swapon --summary
+  register: k3s_swap_active
+  changed_when: false
+  check_mode: false
+
+- name: Turn off active swap
+  ansible.builtin.command: swapoff -a
+  when: k3s_swap_active.stdout | trim | length > 0
+
+- name: Comment out swap entries in fstab
+  ansible.builtin.replace:
+    path: /etc/fstab
+    regexp: '^([^#\n]*\sswap\s.*)$'
+    replace: '# \1'
+
+# Best-effort: Ubuntu's zram-backed swap ships under different unit names
+# across releases, and most won't be present at all. A missing unit is not a
+# failure here — only an already-active one that we failed to disable would
+# leave swap coming back on the next boot, and swapoff -a above already
+# handles the running instance for this boot.
+#
+# `failed_when: false` rather than `ignore_errors: true`: systemd_service
+# raises "Could not find the requested service" as a hard module failure
+# when the unit is absent, and that's reported here as a genuine task
+# failure regardless of ignore_errors — failed_when overrides the result
+# directly instead of trying to catch it after the fact.
+- name: Disable Ubuntu's zram-backed swap, if present
+  ansible.builtin.systemd_service:
+    name: "{{ item }}"
+    enabled: false
+    state: stopped
+  loop:
+    - zram-config.service
+    - systemd-zram-setup@zram0.service
+  register: k3s_zram_disable
+  failed_when: false
+  changed_when: k3s_zram_disable is changed
+
+- name: Ensure curl is installed
+  ansible.builtin.apt:
+    name: curl
+    state: present
+    update_cache: true
+    cache_valid_time: 3600

build/config/ansible/roles/k3s_node/tasks/server.yml

@@ -0,0 +1,103 @@
+---
+# Bootstraps this node as the (single) k3s control plane. playbooks/k3s.yml
+# runs the k3s_control_plane play before k3s_workers, so by the time
+# agent.yml runs anywhere, K3S_URL below already points at something live.
+#
+# The token comes from Vault as a fixed, pre-shared value rather than letting
+# k3s generate one on first install — see docs/vault-secrets.md
+# (homelab/k3s-homelab-utils). That's what makes a full rebuild (wipe both SD
+# cards, reinstall) reproduce the same cluster identity instead of needing the
+# new token hunted down and re-distributed by hand.
+#
+# --node-name pins the k8s node object to the Ansible inventory_hostname
+# rather than whatever the OS hostname happens to be — roles/k3s_maintenance
+# addresses nodes by inventory_hostname when draining/uncordoning, and that
+# only works if the two names are guaranteed to match.
+
+- name: Bootstrap the k3s server
+  no_log: true # K3S_TOKEN passes through this block
+  block:
+    - name: Look up the k3s cluster secrets
+      ansible.builtin.set_fact:
+        k3s_secrets: >-
+          {{ lookup('community.hashi_vault.vault_kv2_get',
+                     k3s_vault_path,
+                     engine_mount_point=vault_kv_mount,
+                     url=vault_addr,
+                     auth_method=vault_auth_method,
+                     role_id=vault_role_id | default(omit),
+                     secret_id=vault_secret_id | default(omit)).secret }}
+
+    - name: Compute the desired k3s server exec line
+      ansible.builtin.set_fact:
+        k3s_server_exec: >-
+          server --node-name {{ inventory_hostname }} --tls-san {{ k3s_api_tls_san }}
+          --write-kubeconfig-mode 644 {{ (k3s_extra_args + k3s_server_extra_args) | join(' ') }}
+
+    # Compared against what k3s was last installed with (see the copy task
+    # below) so a change to k3s_extra_args/k3s_server_extra_args — or
+    # k3s_api_tls_san — gets applied on the next run instead of silently
+    # sitting unused: the version check below has no way to notice an
+    # exec-line-only change. Missing file (first install) counts as changed.
+    - name: Read the exec line k3s was last installed with
+      ansible.builtin.slurp:
+        src: /etc/rancher/k3s/.ansible_install_exec
+      register: k3s_installed_exec_raw
+      failed_when: false
+      check_mode: false
+
+    - name: Determine whether the exec line has changed
+      ansible.builtin.set_fact:
+        k3s_exec_changed: >-
+          {{ k3s_installed_exec_raw.content is not defined
+             or (k3s_installed_exec_raw.content | b64decode) != k3s_server_exec }}
+
+    - name: Check the installed k3s version
+      ansible.builtin.command: k3s --version
+      register: k3s_installed_version
+      changed_when: false
+      failed_when: false
+      check_mode: false
+
+    # The installer is safe to re-run — it's a no-op if the requested version
+    # and exec line are already active — but skipping it when neither changed
+    # avoids restarting the service (and briefly dropping the API) on every
+    # playbook run. Restarting the k3s process itself (as opposed to
+    # rebooting the node, which roles/k3s_maintenance handles separately) is
+    # a brief control-plane/kubelet blip, not a pod outage — containerd keeps
+    # every already-running pod up underneath it.
+    - name: Install/upgrade k3s server
+      ansible.builtin.shell: curl -sfL https://get.k3s.io | sh -
+      environment:
+        INSTALL_K3S_VERSION: "{{ k3s_version }}"
+        INSTALL_K3S_EXEC: "{{ k3s_server_exec }}"
+        K3S_TOKEN: "{{ k3s_secrets.K3S_TOKEN }}"
+      when: >-
+        k3s_installed_version.rc != 0
+        or k3s_version not in k3s_installed_version.stdout
+        or k3s_exec_changed
+
+- name: Ensure the k3s service is enabled and running
+  ansible.builtin.systemd_service:
+    name: k3s
+    enabled: true
+    state: started
+
+# k3s creates /etc/rancher/k3s itself once it has something to put there
+# (the generated kubeconfig, on the server) — defensive rather than relied
+# on, so this marker doesn't assume the directory already exists.
+- name: Ensure /etc/rancher/k3s exists
+  ansible.builtin.file:
+    path: /etc/rancher/k3s
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Record the exec line k3s was installed with
+  ansible.builtin.copy:
+    dest: /etc/rancher/k3s/.ansible_install_exec
+    content: "{{ k3s_server_exec }}"
+    owner: root
+    group: root
+    mode: "0600"

build/config/ansible/roles/k3s_postgres/defaults/main.yml

@@ -0,0 +1,71 @@
+---
+# Ansible-side knobs for roles/k3s_postgres — CloudNativePG (CNPG), the
+# Postgres operator for the homelab-utils cluster's shared database.
+# Installed via k3s's own bundled helm-controller, same mechanism as
+# roles/k3s_metallb and roles/k3s_monitoring (see tasks/main.yml) — no helm
+# binary or extra Ansible collection needed on the controller.
+#
+# Chosen over Bitnami's postgresql-ha (repmgr+pgpool, more moving parts per
+# pod, and Bitnami's free chart/image catalog was restructured into a
+# "legacy" repo in 2025 — a risky long-term dependency) and the Zalando
+# operator (mature but Patroni-based, heavier, and its own config
+# conventions fit less naturally into this repo's lean HelmChart-CR-only
+# pattern). CNPG installs as one operator chart plus one plain Cluster CR —
+# the same "HelmChart + plain config manifest" shape already used for
+# MetalLB (chart + IPAddressPool).
+#
+# The app-portable half of this config (instance count, storage size/class)
+# lives with the app it belongs to instead: see
+# src/shared/postgres/ansible/kubernetes/vars.yml, loaded into `pg_config`
+# by tasks/main.yml — same common/vars.yml + platform-vars.yml split
+# compose_stack and lxc_app use for every other app.
+
+k3s_postgres_operator_namespace: cnpg-system
+
+k3s_postgres_chart_repo: https://cloudnative-pg.github.io/charts
+
+# Pinned, same reasoning as k3s_monitoring_chart_version/k3s_metallb_chart_version
+# — a rebuild months from now should reproduce today's operator, not
+# whatever's newest at the time. Bump deliberately; check the current
+# release first at https://github.com/cloudnative-pg/charts/releases.
+k3s_postgres_chart_version: "0.29.0"
+
+# Where the actual Postgres Cluster (not the operator) lives.
+k3s_postgres_namespace: shared-postgres
+
+# Postgres major/minor pinned via the operand image, independent of the
+# chart/operator version above. This is a separate physical instance from
+# the Docker-based shared/postgres (pinned to 13 there — see
+# src/shared/postgres/common/vars.yml), so there's no need to match; pin to
+# a current stable major instead. Multi-arch (amd64/arm64) upstream, same
+# as every other image this cluster runs.
+k3s_postgres_image: ghcr.io/cloudnative-pg/postgresql:18.4
+
+# Same Vault path the Unraid/Proxmox shared-postgres instances already use
+# (see docs/vault-secrets.md) — a separate physical instance, but one
+# superuser identity for the "shared postgres" concept everywhere it runs.
+k3s_postgres_vault_path: homelab/shared/postgres
+
+# The LAN address the primary is published on, via the LoadBalancer Service
+# declared in templates/postgres-cluster.yaml.j2. Must be inside
+# k3s_metallb_address_range — MetalLB only assigns from its own pools, and a
+# request for anything outside them leaves the Service pending forever rather
+# than failing loudly.
+#
+# Empty here on purpose, same reasoning as k3s_metallb_address_range's own
+# default: the address is a property of the LAN, not of this role, so it's set
+# in inventory/group_vars/k3s_cluster.yml next to the pool it has to fall
+# inside. Unlike the pool, leaving it empty is survivable — the Service is
+# still created and MetalLB assigns the next free address; you just don't know
+# which one until you look, which is fine for in-cluster clients and no good
+# for anything that has to write the address down.
+k3s_postgres_loadbalancer_ip: ""
+
+# Resource requests/limits per instance — sized for a Raspberry Pi 4, not a
+# datacenter node, same reasoning as k3s_monitoring_values.
+k3s_postgres_resources:
+  requests:
+    cpu: 100m
+    memory: 256Mi
+  limits:
+    memory: 512Mi

build/config/ansible/roles/k3s_postgres/tasks/main.yml

@@ -0,0 +1,91 @@
+---
+# Deploys the shared Postgres cluster onto k3s: install CloudNativePG (the
+# operator) via a HelmChart CR, then define the actual cluster as a plain
+# Cluster CR — same two-manifest pattern as roles/k3s_metallb (chart CR +
+# plain config manifest), for the same reason: k3s's deploy controller
+# retries a manifest referencing CRDs the HelmChart above hasn't installed
+# yet instead of failing outright, so the Cluster CR doesn't need to wait on
+# the operator chart finishing first.
+#
+# Config is layered the same way compose_stack/lxc_app layer an app's
+# common/vars.yml + platform vars.yml — src/shared/postgres/ is the same
+# shared service already used on Unraid/Proxmox, just deployed a third way
+# here. Secrets come from the same Vault path those platforms already use
+# (homelab/shared/postgres) — this is a separate physical instance, but
+# reuses the path rather than inventing a k3s-specific one, same "one path
+# per app-concept" convention as every other stack.
+
+- name: Set shared-postgres source facts
+  ansible.builtin.set_fact:
+    pg_local_dir: "{{ repo_root }}/src/shared/postgres"
+
+- name: Load portable shared-postgres variables
+  ansible.builtin.include_vars:
+    file: "{{ pg_local_dir }}/common/vars.yml"
+    name: pg_common_vars
+
+- name: Load Kubernetes-specific shared-postgres variables
+  ansible.builtin.include_vars:
+    file: "{{ pg_local_dir }}/ansible/kubernetes/vars.yml"
+    name: pg_platform_vars
+
+- name: Merge shared-postgres configuration
+  ansible.builtin.set_fact:
+    pg_config: >-
+      {{ (pg_common_vars.env_defaults | default({}))
+         | combine(pg_platform_vars.env_defaults | default({})) }}
+
+- name: Look up shared-postgres superuser credentials from Vault
+  ansible.builtin.set_fact:
+    pg_vault_secrets: >-
+      {{ lookup('community.hashi_vault.vault_kv2_get',
+                 k3s_postgres_vault_path,
+                 engine_mount_point=vault_kv_mount,
+                 url=vault_addr,
+                 auth_method=vault_auth_method,
+                 role_id=vault_role_id | default(omit),
+                 secret_id=vault_secret_id | default(omit)).secret }}
+  no_log: true
+
+- name: Deploy the CloudNativePG operator HelmChart manifest
+  ansible.builtin.template:
+    src: cnpg-operator.helmchart.yaml.j2
+    dest: /var/lib/rancher/k3s/server/manifests/cnpg-operator.yaml
+    owner: root
+    group: root
+    mode: "0644"
+  become: true
+
+# Namespace + Cluster CR, no secrets — safe to render even though the
+# Cluster CR references the Secret below, for the same "later file, retried
+# reconcile" reasoning as the Secret referencing this file's Namespace.
+- name: Deploy the shared-postgres Cluster manifest
+  ansible.builtin.template:
+    src: postgres-cluster.yaml.j2
+    dest: /var/lib/rancher/k3s/server/manifests/shared-postgres-cluster.yaml
+    owner: root
+    group: root
+    mode: "0644"
+  become: true
+
+# Separate file (and mode) from the Cluster manifest above purely because
+# this one carries the Vault-sourced password — everything else here is safe
+# at 0644, this one isn't.
+#
+# The dest filename deliberately doesn't track the Secret's own name, which
+# gained a `-vault` suffix (see the template's header for why). k3s's deploy
+# controller records which resources each manifest *file* created and prunes
+# the ones that disappear from it, so renaming the Secret inside this same
+# file makes the old `shared-postgres-superuser` object get garbage-collected
+# on the next run. Renaming the file too would orphan it instead: Ansible
+# doesn't remove files it no longer writes, so the old manifest would sit
+# there keeping the stale Secret alive.
+- name: Deploy the shared-postgres superuser Secret
+  ansible.builtin.template:
+    src: postgres-superuser-secret.yaml.j2
+    dest: /var/lib/rancher/k3s/server/manifests/shared-postgres-superuser-secret.yaml
+    owner: root
+    group: root
+    mode: "0600"
+  become: true
+  no_log: true

build/config/ansible/roles/k3s_postgres/templates/cnpg-operator.helmchart.yaml.j2

@@ -0,0 +1,21 @@
+# Managed by Ansible (roles/k3s_postgres) — do not edit on the node.
+#
+# A HelmChart CR, same pattern as
+# roles/k3s_metallb/templates/metallb.helmchart.yaml.j2 and
+# roles/k3s_monitoring — k3s's bundled helm-controller reconciles it, the
+# same mechanism it uses to install its own bundled Traefik and ServiceLB.
+# No helm binary, kubeconfig, or extra Ansible collection needed on the
+# controller. This installs the CloudNativePG operator and its CRDs only —
+# the actual Postgres cluster is templates/postgres-cluster.yaml.j2, a plain
+# manifest applied once the CRD it references exists (see that file).
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: cloudnative-pg
+  namespace: kube-system
+spec:
+  chart: cloudnative-pg
+  repo: {{ k3s_postgres_chart_repo }}
+  version: "{{ k3s_postgres_chart_version }}"
+  targetNamespace: {{ k3s_postgres_operator_namespace }}
+  createNamespace: true

build/config/ansible/roles/k3s_postgres/templates/postgres-cluster.yaml.j2

@@ -0,0 +1,104 @@
+# Managed by Ansible (roles/k3s_postgres) — do not edit on the node.
+#
+# Plain manifests, not a HelmChart — same pattern as
+# roles/k3s_metallb/templates/metallb-config.yaml.j2: the Cluster CR below
+# references a CRD (postgresql.cnpg.io) that only exists once the
+# cloudnative-pg HelmChart (cnpg-operator.yaml, same manifests directory) has
+# actually installed the operator, so it's applied here rather than gated
+# behind a "wait for CRDs" step — k3s's deploy controller retries a manifest
+# referencing not-yet-existing CRDs until they show up, instead of failing
+# once and giving up. templates/postgres-superuser-secret.yaml.j2 relies on
+# the Namespace created here the same way, for the same reason.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ k3s_postgres_namespace }}
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: shared-postgres
+  namespace: {{ k3s_postgres_namespace }}
+spec:
+  # 1 primary + 1 replica (src/shared/postgres/ansible/kubernetes/vars.yml).
+  # CNPG's own default pod anti-affinity (preferred, topology key hostname)
+  # spreads them across distinct nodes; the nodeAffinity below narrows which
+  # nodes those can be.
+  instances: {{ pg_config.POSTGRES_INSTANCES }}
+  imageName: {{ k3s_postgres_image }}
+
+  # Password-based superuser login, from the Secret Ansible renders
+  # alongside this file. Off by default in CNPG — without this, the operator
+  # ignores the secret's content and sets the postgres user's password to
+  # NULL, disabling remote login as it.
+  #
+  # The name must stay clear of `<cluster>-superuser` (i.e.
+  # `shared-postgres-superuser`), which is what CNPG calls the secret it
+  # generates for itself when this stanza is absent. Claiming that name for
+  # our own object stops password changes reaching the database — see
+  # postgres-superuser-secret.yaml.j2 for the full symptom.
+  enableSuperuserAccess: true
+  superuserSecret:
+    name: shared-postgres-superuser-vault
+
+  # A LoadBalancer Service for the primary, on top of the ClusterIP -rw/-ro/-r
+  # Services CNPG creates for every Cluster. Declared here as a CNPG *managed
+  # service* rather than as a Service manifest of our own: selectorType: rw
+  # means the operator maintains the selector, so the LB follows a failover to
+  # the other instance the same way the built-in -rw Service does. A
+  # hand-written Service would need its selector re-pointed by hand after
+  # every promotion.
+  #
+  # This is what makes the cluster reachable from the Ansible controller at a
+  # fixed host:port, which is the thing per-app database provisioning needs —
+  # the community.postgresql tasks compose_stack/lxc_app use can't talk to a
+  # ClusterIP, and `kubectl port-forward` isn't a stable address. It does put
+  # the superuser on the LAN: acceptable here for the same reason the Unraid
+  # instance's published port is, and the LAN is the boundary either way.
+  managed:
+    services:
+      additional:
+        - selectorType: rw
+          serviceTemplate:
+            metadata:
+              name: shared-postgres-lb
+{% if k3s_postgres_loadbalancer_ip %}
+              # Pinned rather than left to MetalLB's next-free pick, because
+              # this address ends up in config elsewhere (inventory, an app's
+              # vars.yml) and shouldn't move when the Service is recreated.
+              annotations:
+                metallb.universe.tf/loadBalancerIPs: "{{ k3s_postgres_loadbalancer_ip }}"
+{% endif %}
+            spec:
+              type: LoadBalancer
+              # No ports: — the operator fills in Postgres's own (5432) from
+              # the same template it builds the -rw Service from; naming them
+              # here would only risk disagreeing with it.
+
+  # POSTGRES_PORT (common/vars.yml) is 5432, the same port CNPG always
+  # listens on inside the pod/Service — nothing to override here.
+  storage:
+    size: {{ pg_config.POSTGRES_STORAGE_SIZE }}
+    storageClass: {{ pg_config.POSTGRES_STORAGE_CLASS }}
+
+  resources:
+{{ k3s_postgres_resources | to_nice_yaml(indent=2) | indent(4, first=true) }}
+
+  # Workers only — k3s-ctrl-01 stays free of app pods, same boundary the
+  # cluster already keeps for monitoring/MetalLB.
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: DoesNotExist
+
+  # No bootstrap: stanza — CNPG's default initdb bootstrap creates a
+  # `postgres` superuser (above) plus a default `app` database owned by an
+  # auto-generated `app` role/secret. That default app database is unused
+  # today: no app is deployed onto k3s yet, and real per-app database
+  # provisioning here is a deferred design problem (see CLAUDE.md → "Key
+  # decisions") — the community.postgresql approach compose_stack/lxc_app
+  # use needs a controller-reachable host:port, which this cluster's
+  # in-cluster -rw Service isn't without further work.

build/config/ansible/roles/k3s_postgres/templates/postgres-superuser-secret.yaml.j2

@@ -0,0 +1,49 @@
+# Managed by Ansible (roles/k3s_postgres) — do not edit on the node.
+#
+# Superuser credentials for the shared-postgres Cluster
+# (postgres-cluster.yaml.j2), sourced from the same Vault path
+# (homelab/shared/postgres) the Unraid/Proxmox shared-postgres instances
+# already use — see docs/vault-secrets.md. CNPG requires
+# `type: kubernetes.io/basic-auth` with username/password keys, and
+# `enableSuperuserAccess: true` on the Cluster (set there) for this secret to
+# actually be usable for password login rather than just ignored.
+#
+# The `-vault` suffix is load-bearing, and removing it reintroduces a bug
+# that is very hard to read off the symptoms. CNPG generates its own
+# superuser secret named `<cluster>-superuser` when the Cluster doesn't name
+# one — for a Cluster called `shared-postgres` that is exactly
+# `shared-postgres-superuser`, the name this file used to claim. Writing our
+# own object at the operator's reserved name meant the operator treated it as
+# one it had already authored and reconciled, so a password change in Vault
+# propagated as far as this Secret and stopped there: the role in the
+# database kept whatever was set at bootstrap. Every check short of querying
+# pg_authid agrees the change landed, while `psql` from the LAN keeps
+# returning "password authentication failed for user postgres". Any name
+# outside the `<cluster>-*` pattern the operator owns avoids it.
+#
+# Namespace comes from postgres-cluster.yaml.j2, a separate file in the same
+# manifests directory — see that file's header for why applying this before
+# the Namespace exists isn't a problem.
+#
+# The username is layered the same way every other value in this repo is —
+# Vault on top, common/vars.yml underneath — rather than required from Vault.
+# It isn't a secret (only the password is), and the Vault path is allowed to
+# carry just POSTGRES_SUPERUSER_PASSWORD; setting POSTGRES_SUPERUSER there
+# still wins, which is what docs/vault-secrets.md means by "Vault is
+# authoritative since it's paired with the password".
+apiVersion: v1
+kind: Secret
+metadata:
+  name: shared-postgres-superuser-vault
+  namespace: {{ k3s_postgres_namespace }}
+type: kubernetes.io/basic-auth
+# to_json, not bare interpolation: it emits a double-quoted YAML scalar with
+# escaping handled, so a password containing `#`, `:`, a leading `*`/`&`, or
+# leading/trailing whitespace survives intact. Unquoted, YAML silently
+# reinterprets those (`#` starts a comment, an all-digit password becomes an
+# int) and the Secret ends up holding something other than what Vault has —
+# which fails authentication while every value still looks right in playbook
+# output. Alphanumeric-only passwords hide the problem rather than fix it.
+stringData:
+  username: {{ (pg_vault_secrets.POSTGRES_SUPERUSER | default(pg_config.POSTGRES_SUPERUSER)) | to_json }}
+  password: {{ pg_vault_secrets.POSTGRES_SUPERUSER_PASSWORD | to_json }}

build/config/ansible/roles/k3s_traefik/defaults/main.yml

@@ -0,0 +1,115 @@
+---
+# Ansible-side knobs for roles/k3s_traefik — the ingress controller k3s
+# bundles. The odd one out among the k3s_* cluster services: every other one
+# installs a chart this repo chose (roles/k3s_metallb, k3s_monitoring,
+# k3s_postgres, k3s_cert_manager), whereas Traefik is already installed by
+# k3s itself before Ansible ever connects. So this role installs nothing —
+# it only adjusts what's there, via a HelmChartConfig (see
+# templates/traefik.helmchartconfig.yaml.j2 for why that, and not an edit to
+# k3s's own traefik.yaml).
+#
+# Scope today is the dashboard, which a stock k3s serves 404s for. That's
+# not a broken install: Traefik builds the dashboard regardless
+# (`api.dashboard` defaults on), but the Traefik chart stopped creating the
+# router that reaches it in v28, so there is nothing routing /dashboard/ on
+# any entrypoint until something puts that router back.
+
+# Where k3s installs Traefik, and so where the HelmChartConfig has to live —
+# a HelmChartConfig only applies to the HelmChart of the same name in the
+# same namespace.
+k3s_traefik_namespace: kube-system
+
+# --- The dashboard, reachable by port-forward --------------------------------
+#
+# Re-enables the chart's own dashboard IngressRoute, which binds to Traefik's
+# internal `traefik` entrypoint (port 9000). That entrypoint is not published
+# on Traefik's Service, so this exposes nothing to the LAN; it makes exactly
+# one thing work:
+#
+#   kubectl -n kube-system port-forward deploy/traefik 9000:9000
+#   # then http://127.0.0.1:9000/dashboard/ — the trailing slash is required
+#
+# Left on even when the dashboard is also exposed on a hostname below, and
+# that's deliberate rather than redundant. Once the exposed route is behind
+# Authentik, the dashboard depends on Authentik being up, which depends on
+# CNPG being up, which depends on the cluster being healthy — precisely the
+# things you would open the dashboard to diagnose. Port-forward talks to the
+# pod and traverses none of it, so it stays as the break-glass path. Don't
+# "tidy this up" once the hostname works.
+k3s_traefik_dashboard_enabled: true
+
+# --- The dashboard, exposed on a hostname ------------------------------------
+#
+# Empty (the default) means the port-forward above is the only way in, and
+# tasks/main.yml renders no IngressRoute at all — it removes one it previously
+# rendered, so clearing this is a real teardown rather than a no-op.
+#
+# Set it to a hostname (e.g. traefik.turtlesystems.uk) to publish the
+# dashboard on the `websecure` entrypoint with a cert-manager certificate and
+# the forward-auth middleware below in front of it. Doing so requires
+# k3s_traefik_dashboard_auth_address to be set as well — the role refuses to
+# run otherwise, because the failure mode of getting this wrong is a
+# read-only view of every route, service and middleware in the cluster
+# published unauthenticated to the LAN.
+#
+# The hostname needs a DNS record pointing at Traefik's MetalLB address
+# (`kubectl -n kube-system get svc traefik`), same as any app's Ingress.
+k3s_traefik_dashboard_host: ""
+
+# The ClusterIssuer roles/k3s_cert_manager creates. Must match
+# k3s_cert_manager_issuer_name — stated rather than referenced because that
+# lives in the other role's defaults, which aren't in scope here. Same
+# convention as CERT_ISSUER in src/authentik/ansible/kubernetes/vars.yml.
+#
+# An explicit Certificate rather than an Ingress annotation, unlike every app
+# on this cluster: cert-manager watches Ingress resources, and the dashboard
+# can't be one. It's served by Traefik's internal `api@internal` service,
+# which has no Kubernetes Service for an Ingress to point at — hence an
+# IngressRoute, which cert-manager doesn't watch.
+k3s_traefik_dashboard_cert_issuer: letsencrypt
+k3s_traefik_dashboard_tls_secret: traefik-dashboard-tls
+
+# --- Authentication in front of the exposed dashboard ------------------------
+#
+# A Traefik forwardAuth middleware pointing at Authentik, which now runs on
+# this cluster (roles/k3s_app, src/authentik). No default, and required as
+# soon as k3s_traefik_dashboard_host is set: a default here would be a guess
+# at another app's service name, and a wrong guess that happens to be
+# unreachable is a 500, not an open dashboard — but one that happens to
+# resolve to the wrong thing is worse. Set it explicitly, next to the host,
+# in inventory/group_vars/k3s_cluster.yml.
+#
+# For Authentik's embedded outpost, in-cluster:
+#
+#   http://authentik-server.authentik.svc.cluster.local/outpost.goauthentik.io/auth/traefik
+#
+# The provider in Authentik must be a Proxy Provider in **forward auth
+# (domain level)** mode, not single-application. Single-application mode
+# additionally requires /outpost.goauthentik.io/ to be routed to Authentik on
+# *this* host, which from an IngressRoute in kube-system means referencing a
+# Service in the authentik namespace — a cross-namespace reference Traefik
+# rejects unless started with providers.kubernetescrd.allowCrossNamespace,
+# which it isn't. Domain-level mode handles the redirect on Authentik's own
+# hostname instead and needs no such route, so it's the mode that fits.
+k3s_traefik_dashboard_auth_address: ""
+
+k3s_traefik_dashboard_auth_middleware: dashboard-auth
+
+# What Authentik sets on the way back through, forwarded to the dashboard.
+# The dashboard itself reads none of them — it has no notion of a user — so
+# this is really about the headers existing for anything else that reuses
+# this middleware later, and about matching Authentik's documented list
+# rather than inventing a shorter one.
+k3s_traefik_dashboard_auth_response_headers:
+  - X-authentik-username
+  - X-authentik-groups
+  - X-authentik-entitlements
+  - X-authentik-email
+  - X-authentik-name
+  - X-authentik-uid
+  - X-authentik-jwt
+  - X-authentik-meta-jwks
+  - X-authentik-meta-outpost
+  - X-authentik-meta-provider
+  - X-authentik-meta-app
+  - X-authentik-meta-version

build/config/ansible/roles/k3s_traefik/tasks/main.yml

@@ -0,0 +1,59 @@
+---
+# Adjusts the Traefik k3s installs for itself, rather than installing
+# anything: a HelmChartConfig merged over k3s's own Traefik HelmChart, plus —
+# only when the dashboard is being published on a hostname — the
+# Certificate/Middleware/IngressRoute trio that puts it behind Authentik.
+# See defaults/main.yml and templates/traefik.helmchartconfig.yaml.j2.
+#
+# Same delivery mechanism as every other k3s_* role: template a file into
+# /var/lib/rancher/k3s/server/manifests/ and let k3s's bundled helm-controller
+# and deploy controller reconcile it. No helm binary, kubeconfig or extra
+# collection on the controller.
+
+- name: Fail fast if the dashboard would be exposed without authentication
+  when: k3s_traefik_dashboard_host | length > 0
+  ansible.builtin.assert:
+    that:
+      - k3s_traefik_dashboard_auth_address | length > 0
+    fail_msg: >-
+      k3s_traefik_dashboard_host is set but k3s_traefik_dashboard_auth_address
+      is empty — set both in inventory/group_vars/k3s_cluster.yml, or neither.
+      Publishing the dashboard without the forward-auth middleware would put a
+      read-only view of every router, service and middleware on this cluster
+      on the LAN unauthenticated, so this refuses rather than defaulting to an
+      address that might not be Authentik.
+    quiet: true
+  run_once: true
+
+- name: Deploy the Traefik HelmChartConfig
+  ansible.builtin.template:
+    src: traefik.helmchartconfig.yaml.j2
+    dest: /var/lib/rancher/k3s/server/manifests/traefik-config.yaml
+    owner: root
+    group: root
+    mode: "0644"
+  become: true
+
+- name: Deploy the exposed dashboard route
+  when: k3s_traefik_dashboard_host | length > 0
+  ansible.builtin.template:
+    src: dashboard-ingressroute.yaml.j2
+    dest: /var/lib/rancher/k3s/server/manifests/traefik-dashboard.yaml
+    owner: root
+    group: root
+    mode: "0644"
+  become: true
+
+# Clearing k3s_traefik_dashboard_host is a teardown, not just a stop-managing:
+# k3s's deploy controller garbage-collects the objects a manifest created when
+# the manifest is removed, so deleting this file withdraws the public route,
+# its middleware and its certificate. Same reasoning as roles/k3s_app's
+# removal path — with the difference that this one needs no `state: absent`
+# tombstone, because the hostname is a single value rather than a list entry
+# that could be silently dropped.
+- name: Withdraw the exposed dashboard route when no host is configured
+  when: k3s_traefik_dashboard_host | length == 0
+  ansible.builtin.file:
+    path: /var/lib/rancher/k3s/server/manifests/traefik-dashboard.yaml
+    state: absent
+  become: true

build/config/ansible/roles/k3s_traefik/templates/dashboard-ingressroute.yaml.j2

@@ -0,0 +1,83 @@
+{#
+  Managed by Ansible (roles/k3s_traefik) — do not edit on the node.
+
+  Plain manifests, not a HelmChart — same pattern as
+  roles/k3s_metallb/templates/metallb-config.yaml.j2 and
+  roles/k3s_cert_manager/templates/cluster-issuer.yaml.j2: CRs belonging to
+  charts installed elsewhere, dropped in the auto-deploying directory and
+  retried by k3s's deploy controller until the CRDs they need exist.
+
+  Rendered only when k3s_traefik_dashboard_host is set; tasks/main.yml
+  deletes this file when it isn't, and k3s's deploy controller
+  garbage-collects what the file created.
+
+  traefik.io/v1alpha1, not traefik.containo.us/v1alpha1 — the group changed
+  with Traefik v3, which is what current k3s bundles.
+-#}
+{# Explicit Certificate rather than the cert-manager.io/cluster-issuer
+   annotation every app's Ingress uses: cert-manager watches Ingress
+   resources, and this route can't be one (see below). DNS-01, so this
+   issues without the hostname resolving anywhere yet. -#}
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: traefik-dashboard
+  namespace: {{ k3s_traefik_namespace }}
+spec:
+  secretName: {{ k3s_traefik_dashboard_tls_secret }}
+  issuerRef:
+    name: {{ k3s_traefik_dashboard_cert_issuer }}
+    kind: ClusterIssuer
+  dnsNames:
+    - {{ k3s_traefik_dashboard_host }}
+---
+{# Authentik, in forward-auth (domain level) mode — see defaults/main.yml for
+   why domain level and not single-application. Fails closed: if Authentik is
+   down this returns an error rather than passing the request through, which
+   is the correct direction and the reason the port-forward path stays. -#}
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  name: {{ k3s_traefik_dashboard_auth_middleware }}
+  namespace: {{ k3s_traefik_namespace }}
+spec:
+  forwardAuth:
+    address: {{ k3s_traefik_dashboard_auth_address }}
+    # Traefik strips X-Forwarded-* from client requests by default; Authentik
+    # needs them to know which host and scheme the user actually asked for,
+    # and builds its redirect back out of them. Safe here because the only
+    # thing that can set them is Traefik itself — nothing reaches this
+    # middleware without passing through the entrypoint first.
+    trustForwardHeader: true
+    authResponseHeaders:
+{% for header in k3s_traefik_dashboard_auth_response_headers %}
+      - {{ header }}
+{% endfor %}
+---
+{# An IngressRoute rather than an Ingress, and not by preference: the
+   dashboard is served by api@internal, a Traefik-internal service with no
+   Kubernetes Service behind it, so there is nothing for an Ingress backend
+   to name. That single fact is why the Certificate above is explicit and why
+   the middleware is attached here rather than by annotation. -#}
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: traefik-dashboard
+  namespace: {{ k3s_traefik_namespace }}
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    # Both prefixes, because the dashboard is a static bundle under
+    # /dashboard/ that reads its data from /api — serving the first without
+    # the second gets you a page that loads and then stays empty.
+    - kind: Rule
+      match: Host(`{{ k3s_traefik_dashboard_host }}`) && (PathPrefix(`/dashboard`) || PathPrefix(`/api`))
+      middlewares:
+        - name: {{ k3s_traefik_dashboard_auth_middleware }}
+          namespace: {{ k3s_traefik_namespace }}
+      services:
+        - kind: TraefikService
+          name: api@internal
+  tls:
+    secretName: {{ k3s_traefik_dashboard_tls_secret }}

build/config/ansible/roles/k3s_traefik/templates/traefik.helmchartconfig.yaml.j2

@@ -0,0 +1,34 @@
+{#
+  Managed by Ansible (roles/k3s_traefik) — do not edit on the node.
+
+  A HelmChartConfig, not a HelmChart, because this is the one chart on the
+  cluster this repo doesn't install: k3s installs Traefik itself, rewriting
+  /var/lib/rancher/k3s/server/manifests/traefik.yaml on every server start
+  and overwriting whatever is in it. A HelmChartConfig carrying the same
+  name and namespace as that HelmChart is the supported way in — k3s's
+  helm-controller merges its valuesContent over the chart's values on the
+  next reconcile. Editing traefik.yaml directly works until the next
+  restart, and then silently doesn't.
+
+  Hence also the filename this renders to (traefik-config.yaml): anything
+  called traefik.yaml here belongs to k3s and would be reclaimed.
+-#}
+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+  name: traefik
+  namespace: {{ k3s_traefik_namespace }}
+spec:
+  valuesContent: |-
+    ingressRoute:
+      dashboard:
+        # Traefik builds the dashboard either way — `api.dashboard` is on by
+        # default. What the chart stopped shipping in v28 is the router that
+        # reaches it, which is why a stock k3s answers 404 to
+        # :9000/dashboard/ while :9000/ping happily returns 200. This puts
+        # that router back, on Traefik's internal `traefik` entrypoint
+        # (9000), which isn't published on the Service — so it is reachable
+        # by `kubectl port-forward` and nothing else. See defaults/main.yml
+        # for why that stays true even once the dashboard is also exposed on
+        # a hostname.
+        enabled: {{ k3s_traefik_dashboard_enabled | bool | lower }}

build/config/ansible/roles/lxc_app/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+vault_kv_mount: kv

build/config/ansible/roles/lxc_app/tasks/main.yml

@@ -0,0 +1,124 @@
+---
+# Installs one app (one loop iteration of `app` from playbooks/proxmox.yml)
+# natively into a Proxmox LXC: merge portable + Proxmox config, fetch secrets
+# from Vault, provision its database if it declares one, run the app's own
+# install steps, then manage its systemd unit.
+#
+# The Proxmox counterpart to `compose_stack`. Everything up to "Run
+# app-specific install steps" is deliberately the same shape as that role —
+# same vars merge, same Vault lookup, same DB provisioning — so config lives
+# in one place regardless of which platform an app lands on. What differs is
+# the deployment primitive: a systemd service built from packages/binaries
+# rather than `docker compose up -d`.
+#
+# UNPROVEN: `src/shared/postgres/` and `src/forgejo/` both ship an
+# `ansible/proxmox/install.yml` now, but neither has been run end to end
+# against a real LXC. The structure mirrors proven code; the details are not
+# yet exercised.
+
+- name: Set app facts
+  ansible.builtin.set_fact:
+    app_local_dir: "{{ repo_root }}/src/{{ app.src }}"
+
+- name: Load portable app variables
+  ansible.builtin.include_vars:
+    file: "{{ app_local_dir }}/common/vars.yml"
+    name: app_common_vars
+
+- name: Load Proxmox-specific app variables
+  ansible.builtin.include_vars:
+    file: "{{ app_local_dir }}/ansible/proxmox/vars.yml"
+    name: app_platform_vars
+
+- name: Merge app configuration
+  ansible.builtin.set_fact:
+    # Portable values first, Proxmox-specific overrides on top. Available to
+    # the app's install.yml, alongside `vault_secrets` below.
+    app_config: >-
+      {{ (app_common_vars.env_defaults | default({}))
+         | combine(app_platform_vars.env_defaults | default({})) }}
+
+- name: Look up app secrets from Vault
+  ansible.builtin.set_fact:
+    vault_secrets: >-
+      {{ lookup('community.hashi_vault.vault_kv2_get',
+                 app.vault_path,
+                 engine_mount_point=vault_kv_mount,
+                 url=vault_addr,
+                 auth_method=vault_auth_method,
+                 role_id=vault_role_id | default(omit),
+                 secret_id=vault_secret_id | default(omit)).secret }}
+  no_log: true
+
+- name: Provision application database
+  when: app.db is defined
+  no_log: true
+  block:
+    - name: Look up Postgres superuser credentials from Vault
+      ansible.builtin.set_fact:
+        pg_admin_secrets: >-
+          {{ lookup('community.hashi_vault.vault_kv2_get',
+                     app.db.admin_vault_path,
+                     engine_mount_point=vault_kv_mount,
+                     url=vault_addr,
+                     auth_method=vault_auth_method,
+                     role_id=vault_role_id | default(omit),
+                     secret_id=vault_secret_id | default(omit)).secret }}
+
+    - name: Ensure application database role exists
+      community.postgresql.postgresql_user:
+        name: "{{ app.db.user }}"
+        password: "{{ vault_secrets[app.db.password_vault_key] }}"
+        login_host: "{{ app.db.provision_host }}"
+        login_port: "{{ app.db.provision_port }}"
+        login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}"
+        login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}"
+        state: present
+      delegate_to: localhost
+      become: false
+
+    # Role first, then the database with `owner:` — the shape roles/k3s_app
+    # already uses, and not the `postgresql_user` + `priv: ALL` this role was
+    # written with. Two independent reasons it had to change:
+    #
+    #   1. `priv` was deprecated in community.postgresql 3.x and REMOVED in
+    #      4.0.0, so the old call is a hard "Unsupported parameters" failure
+    #      on any current collection. requirements.yml asks for >=3.0.0,
+    #      which installs 4.x.
+    #   2. On this instance it would have been wrong even if it still worked.
+    #      `priv: ALL` grants database-level privileges (CONNECT, CREATE,
+    #      TEMPORARY); since Postgres 15 the `public` schema no longer grants
+    #      CREATE to PUBLIC, so a role holding all of those still cannot
+    #      create a table. The Proxmox cluster is 17 (the Unraid one is 13,
+    #      which is why the same code never failed that way there). Making
+    #      the app role own the database covers it on both: on 15+ `public`
+    #      is owned by `pg_database_owner`, which resolves to whoever owns
+    #      the database.
+    #
+    # Ownership also makes a pg_dump restore land correctly, for the reason
+    # spelled out at the same tasks in roles/k3s_app.
+    - name: Ensure application database exists
+      community.postgresql.postgresql_db:
+        name: "{{ app.db.name }}"
+        owner: "{{ app.db.user }}"
+        login_host: "{{ app.db.provision_host }}"
+        login_port: "{{ app.db.provision_port }}"
+        login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}"
+        login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}"
+        state: present
+      delegate_to: localhost
+      become: false
+
+# Everything genuinely app-specific — fetching the binary or package, creating
+# the service user, laying out data directories, rendering the app's own
+# config file from `app_config` + `vault_secrets`, and installing a systemd
+# unit — lives with the app, not here. See src/<app>/ansible/proxmox/.
+- name: Run app-specific install steps
+  ansible.builtin.include_tasks: "{{ app_local_dir }}/ansible/proxmox/install.yml"
+
+- name: Enable and start the service
+  ansible.builtin.systemd_service:
+    name: "{{ app.service_name | default(app.name) }}"
+    enabled: true
+    state: started
+    daemon_reload: true

build/config/ansible/roles/pve_backup/defaults/main.yml

@@ -0,0 +1,61 @@
+---
+# Backup storage on the NAS, and the vzdump job that writes to it. Override
+# per-node in inventory/host_vars/<node>.yml.
+
+# Name the storage appears under in Proxmox. Also what `pve_backup_job`
+# targets.
+pve_backup_storage: nas-backup
+
+# `nfs` or `pbs`. NFS is the plain option and needs nothing on the NAS beyond
+# an export; PBS gets deduplication and incremental backups but needs a Proxmox
+# Backup Server to exist first. See tasks/main.yml for what each consumes.
+pve_backup_storage_type: nfs
+
+# NFS (pve_backup_storage_type: nfs)
+pve_backup_nfs_server: ""
+pve_backup_nfs_export: ""
+# 4.2 rather than 3: better locking and no separate portmapper.
+pve_backup_nfs_options: vers=4.2
+
+# PBS (pve_backup_storage_type: pbs)
+pve_backup_pbs_server: ""
+pve_backup_pbs_datastore: ""
+pve_backup_pbs_username: ""
+pve_backup_pbs_password: ""
+pve_backup_pbs_fingerprint: ""
+
+# How many archives the storage keeps. Proxmox prunes archives, not their
+# contents — anything inside a guest (such as Forgejo's nightly pg_dump output)
+# is pruned by that guest, on its own schedule.
+pve_backup_prune:
+  keep-daily: 7
+  keep-weekly: 4
+  keep-monthly: 6
+
+# systemd calendar event. Guests that dump a database into their own filesystem
+# first must finish before this — see src/forgejo/ansible/proxmox/README.md.
+pve_backup_schedule: "02:00"
+
+# Stable marker used to find this job again on later runs. The Proxmox API
+# assigns job IDs itself, so matching on the comment is what makes this
+# idempotent rather than creating a duplicate job every deploy.
+pve_backup_comment: managed-by-homelab-iac
+
+# Which guests to back up. `true` covers everything on the node, including
+# guests added later without an edit here; set it false and list VMIDs in
+# pve_backup_vmids to be selective.
+pve_backup_all: true
+pve_backup_vmids: []
+
+# snapshot: no downtime, and the guest filesystem is captured as if it had
+# crashed at that instant. Fine for Forgejo — git repositories tolerate it, and
+# the database is a separate consistent dump rather than live files.
+pve_backup_mode: snapshot
+pve_backup_compress: zstd
+pve_backup_enabled: true
+
+# Optional: address Proxmox mails backup reports to. Left empty means the node
+# default applies. PVE 8 moved most of this to its notification-target system;
+# `mailto` still works, but if you want anything richer, configure a
+# notification target in the UI rather than extending this role.
+pve_backup_mailto: ""

build/config/ansible/roles/pve_backup/tasks/main.yml

@@ -0,0 +1,161 @@
+---
+# Configures backups on a Proxmox node: a storage on the NAS to write archives
+# to, and a scheduled vzdump job that fills it.
+#
+# This is the off-box copy for every Proxmox guest. Guests deliberately keep
+# all their state on their own rootfs rather than bind-mounting anything in,
+# because vzdump excludes bind mounts — see src/forgejo/terraform/README.md.
+# Where an app's data lives outside the container (the shared Postgres), the
+# app dumps it into its own filesystem on a timer that runs before the window
+# below, so one archive is one restore point.
+#
+# Driven through `pvesm`/`pvesh` rather than modules: as of writing,
+# `community.proxmox` has no storage or backup-job module. Swap them in if that
+# changes — the shape here is create-if-absent, update-if-drifted, which is
+# what a module would do anyway.
+#
+# UNVERIFIED — this has not been run against a real node. `pvesm add` and
+# `pvesh create /cluster/backup` accept slightly different options across PVE
+# versions; check yours with `pvesm help add`, `pvesh help create
+# /cluster/backup`, and a `--check` run before trusting it.
+
+- name: Check that the storage backend is configured
+  ansible.builtin.assert:
+    that:
+      - pve_backup_storage_type in ['nfs', 'pbs']
+      - pve_backup_storage_type != 'nfs' or (pve_backup_nfs_server | length > 0
+          and pve_backup_nfs_export | length > 0)
+      - pve_backup_storage_type != 'pbs' or (pve_backup_pbs_server | length > 0
+          and pve_backup_pbs_datastore | length > 0)
+      - pve_backup_all or (pve_backup_vmids | length > 0)
+    fail_msg: >-
+      pve_backup is missing required settings for
+      pve_backup_storage_type={{ pve_backup_storage_type }}. Set them in
+      inventory/host_vars/{{ inventory_hostname }}.yml.
+    quiet: true
+
+- name: Build the prune-backups option string
+  ansible.builtin.set_fact:
+    pve_backup_prune_string: >-
+      {% for key, value in pve_backup_prune.items() %}{{ key }}={{ value }}{% if not loop.last %},{% endif %}{% endfor %}
+
+# --- Storage -------------------------------------------------------------
+
+- name: Read the configured storages
+  ansible.builtin.command:
+    argv: [pvesh, get, /storage, --output-format, json]
+  register: pve_storages
+  changed_when: false
+  check_mode: false
+
+- name: Add the backup storage (NFS)
+  ansible.builtin.command:
+    argv:
+      - pvesm
+      - add
+      - nfs
+      - "{{ pve_backup_storage }}"
+      - --server
+      - "{{ pve_backup_nfs_server }}"
+      - --export
+      - "{{ pve_backup_nfs_export }}"
+      - --options
+      - "{{ pve_backup_nfs_options }}"
+      - --content
+      - backup
+      - --prune-backups
+      - "{{ pve_backup_prune_string }}"
+  when:
+    - pve_backup_storage_type == 'nfs'
+    - pve_backup_storage not in (pve_storages.stdout | from_json | map(attribute='storage') | list)
+
+- name: Add the backup storage (PBS)
+  # The password goes on a command line, which is visible in the node's process
+  # list for the moment the command runs. `pvesm` has no stdin form, so the
+  # alternative is configuring PBS by hand — acceptable for a one-time create
+  # that only fires when the storage is absent.
+  ansible.builtin.command:
+    argv:
+      - pvesm
+      - add
+      - pbs
+      - "{{ pve_backup_storage }}"
+      - --server
+      - "{{ pve_backup_pbs_server }}"
+      - --datastore
+      - "{{ pve_backup_pbs_datastore }}"
+      - --username
+      - "{{ pve_backup_pbs_username }}"
+      - --password
+      - "{{ pve_backup_pbs_password }}"
+      - --fingerprint
+      - "{{ pve_backup_pbs_fingerprint }}"
+      - --content
+      - backup
+      - --prune-backups
+      - "{{ pve_backup_prune_string }}"
+  when:
+    - pve_backup_storage_type == 'pbs'
+    - pve_backup_storage not in (pve_storages.stdout | from_json | map(attribute='storage') | list)
+  no_log: true
+
+# --- Backup job ----------------------------------------------------------
+
+- name: Read the configured backup jobs
+  ansible.builtin.command:
+    argv: [pvesh, get, /cluster/backup, --output-format, json]
+  register: pve_backup_jobs
+  changed_when: false
+  check_mode: false
+
+# Proxmox assigns job IDs itself, so there is nothing stable to key on except
+# a comment we set. Without this the role would add a duplicate job on every
+# run.
+- name: Find a job this role already created
+  ansible.builtin.set_fact:
+    pve_backup_existing: >-
+      {{ (pve_backup_jobs.stdout | from_json)
+         | selectattr('comment', 'defined')
+         | selectattr('comment', 'equalto', pve_backup_comment)
+         | list | first | default({}) }}
+
+- name: Build the vzdump job arguments
+  ansible.builtin.set_fact:
+    pve_backup_job_args: >-
+      {{ ['--schedule', pve_backup_schedule,
+          '--storage', pve_backup_storage,
+          '--mode', pve_backup_mode,
+          '--compress', pve_backup_compress,
+          '--comment', pve_backup_comment,
+          '--enabled', ('1' if pve_backup_enabled else '0')]
+         + (['--all', '1'] if pve_backup_all
+            else ['--all', '0', '--vmid', pve_backup_vmids | join(',')])
+         + (['--mailto', pve_backup_mailto] if pve_backup_mailto | length > 0 else []) }}
+
+- name: Create the vzdump job
+  ansible.builtin.command:
+    argv: "{{ ['pvesh', 'create', '/cluster/backup'] + pve_backup_job_args }}"
+  when: pve_backup_existing | length == 0
+
+# Compared field by field rather than blindly re-applying, so the role reports
+# a change only when there is one. The VMID list is compared as the
+# comma-joined string Proxmox stores, which makes it order-sensitive — reorder
+# pve_backup_vmids and you get one no-op update.
+- name: Work out whether the existing job still matches
+  ansible.builtin.set_fact:
+    pve_backup_drifted: >-
+      {{ pve_backup_existing.schedule | default('') != pve_backup_schedule
+         or pve_backup_existing.storage | default('') != pve_backup_storage
+         or pve_backup_existing.mode | default('') != pve_backup_mode
+         or (pve_backup_existing.enabled | default(1) | int == 1) != pve_backup_enabled
+         or (pve_backup_existing.all | default(0) | int == 1) != pve_backup_all
+         or (not pve_backup_all
+             and pve_backup_existing.vmid | default('') != pve_backup_vmids | join(',')) }}
+  when: pve_backup_existing | length > 0
+
+- name: Update the vzdump job if its settings drifted
+  ansible.builtin.command:
+    argv: "{{ ['pvesh', 'set', '/cluster/backup/' ~ pve_backup_existing.id] + pve_backup_job_args }}"
+  when:
+    - pve_backup_existing | length > 0
+    - pve_backup_drifted | bool

build/config/ansible/roles/unattended_upgrades/defaults/main.yml

@@ -0,0 +1,19 @@
+---
+# Origins unattended-upgrades installs from — Ubuntu's own stock list
+# (security, plus the ESM ones, which are inert without Ubuntu Pro attached
+# rather than an error). Override per-host/group for less than this.
+unattended_upgrades_origins:
+  - "${distro_id}:${distro_codename}"
+  - "${distro_id}:${distro_codename}-security"
+  - "${distro_id}ESMApps:${distro_codename}-apps-security"
+  - "${distro_id}ESM:${distro_codename}-infra-security"
+
+# Never reboot automatically. playbooks/k3s_maintenance.yml (role
+# k3s_maintenance) does that instead — draining the node first, one at a
+# time — which is the entire reason this role exists rather than just
+# `apt install unattended-upgrades` with Ubuntu's own defaults (which reboot
+# at 02:00 unprompted, k3s node or not).
+unattended_upgrades_automatic_reboot: false
+
+unattended_upgrades_remove_unused_deps: true
+unattended_upgrades_remove_unused_kernel_packages: true

build/config/ansible/roles/unattended_upgrades/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: restart unattended-upgrades
+  ansible.builtin.systemd_service:
+    name: unattended-upgrades
+    state: restarted

build/config/ansible/roles/unattended_upgrades/tasks/main.yml

@@ -0,0 +1,43 @@
+---
+# Debian/Ubuntu-generic — not k3s-specific, kept independent of roles/k3s_node
+# so it can be pointed at any apt-based host later (a Proxmox guest, say)
+# without dragging k3s along. Currently only applied to k3s_cluster, from
+# playbooks/k3s.yml.
+#
+# Installs updates hands-off, but leaves rebooting to
+# playbooks/k3s_maintenance.yml — see unattended_upgrades_automatic_reboot in
+# defaults/main.yml for why.
+
+- name: Install unattended-upgrades
+  ansible.builtin.apt:
+    name:
+      - unattended-upgrades
+      - update-notifier-common
+    state: present
+    update_cache: true
+    cache_valid_time: 3600
+
+- name: Configure unattended-upgrades
+  ansible.builtin.template:
+    src: 50unattended-upgrades.j2
+    dest: /etc/apt/apt.conf.d/50unattended-upgrades
+    owner: root
+    group: root
+    mode: "0644"
+  notify: restart unattended-upgrades
+
+- name: Enable periodic apt updates and unattended-upgrades
+  ansible.builtin.copy:
+    dest: /etc/apt/apt.conf.d/20auto-upgrades
+    content: |
+      APT::Periodic::Update-Package-Lists "1";
+      APT::Periodic::Unattended-Upgrade "1";
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Ensure unattended-upgrades is enabled and running
+  ansible.builtin.systemd_service:
+    name: unattended-upgrades
+    enabled: true
+    state: started

build/config/ansible/roles/unattended_upgrades/templates/50unattended-upgrades.j2

@@ -0,0 +1,17 @@
+// Managed by Ansible (roles/unattended_upgrades) — edits here are overwritten
+// on the next run.
+
+Unattended-Upgrade::Allowed-Origins {
+{% for origin in unattended_upgrades_origins %}
+        "{{ origin }}";
+{% endfor %}
+};
+
+Unattended-Upgrade::Remove-Unused-Dependencies "{{ unattended_upgrades_remove_unused_deps | lower }}";
+Unattended-Upgrade::Remove-Unused-Kernel-Packages "{{ unattended_upgrades_remove_unused_kernel_packages | lower }}";
+
+// The whole point of this role: never reboot on its own. A reboot-required
+// package (kernel, libc, ...) is left applied-but-inactive until
+// playbooks/k3s_maintenance.yml (role k3s_maintenance) drains this node and
+// reboots it deliberately, one node at a time.
+Unattended-Upgrade::Automatic-Reboot "{{ unattended_upgrades_automatic_reboot | lower }}";

build/config/terraform/.terraform.lock.hcl

@@ -0,0 +1,23 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/bpg/proxmox" {
+  version = "0.111.1"
+  hashes = [
+    "h1:ML2D3UUZTM99yrll/EBXj7wBYMb8xmQgomqFNybEoxY=",
+    "zh:18fb7c31a08dde6bffa1a4d4a211e604d6d17eec7092fd59331b3db3c6f3742c",
+    "zh:1cd60761538289d4dd2a1086b3ae62a7b0bdd4b1a2f824e9a44e243413168dba",
+    "zh:2eb76f6fc8299b6820ff678c8252332cc3366e226b5ae2e61748fd2449c1ed92",
+    "zh:45e6f7ebd0bf48911d37060359a4f359b5743b3092e985295733990e406d0416",
+    "zh:4aa8ba912eae37975d2e983394d173e595ca34fc76b5bf220b37d0e99d76e98c",
+    "zh:58e0789923103a77d502a0a9fc3eb920625e8eb935ec2d4ac0d006aebd1d186c",
+    "zh:6df8aa85fb8865915537e946c19b02538ad188018a629759c213c6f03730f642",
+    "zh:6ed47bc00d0913a1d0880618fa1376115e9edab6b4a658c081061a7f0e4ca360",
+    "zh:c5b10ff4f33df7e4c29e8f1127d49845b561b37b57517e844fb0954d7923d65e",
+    "zh:d016510e14b738499f0db9d9b3aafe82fc6877fb4ab4e9f831fb68a8d70a1385",
+    "zh:d941f394069bbf24351b363da1c64383f487067aaee0a84f9b96476d4912e212",
+    "zh:ddf271dbc2632ae8ffa8de3972f243ee47d260cb2ac90aa784f2746d98e21a0f",
+    "zh:ed0caa3501c42f611b7e9622c9b1df69fd85dc25a3cd88d3076381829688cd62",
+    "zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597",
+  ]
+}

build/config/terraform/README.md

@@ -0,0 +1,66 @@
+# build/config/terraform
+
+Centralized Terraform config for Proxmox — provider, backend, and the module
+calls that say what infrastructure exists. The counterpart to
+`build/config/ansible`, which holds inventory and the shared roles.
+
+## Division of labour
+
+Terraform's job stops at the guest. It creates the LXC (template, cores,
+memory, disk, IP) and nothing more; Ansible then installs the app into it via
+the `lxc_app` role. That split is why Ansible is the common tool across both
+platforms and Terraform is Proxmox-only:
+
+| | Unraid | Proxmox |
+|---|---|---|
+| Provision the host | n/a — it already exists | Terraform (this directory) |
+| Install the app | Ansible + `compose_stack` (Docker Compose) | Ansible + `lxc_app` (native + systemd) |
+| App config | `src/<app>/common/vars.yml` + `src/<app>/ansible/unraid/vars.yml` | `src/<app>/common/vars.yml` + `src/<app>/ansible/proxmox/vars.yml` |
+
+Per-app LXC specs live in `src/<app>/terraform/` as modules, called from
+`main.tf` here.
+
+## State
+
+`backend "pg"` — specifically against the **CloudNativePG cluster on the k3s
+Pis**, reached on its MetalLB address (`k3s_postgres_loadbalancer_ip`,
+192.168.50.81). *Not* the shared Postgres this configuration itself
+provisions on Proxmox, which would be circular: Terraform would need the
+database to exist in order to create the container the database runs in.
+State goes somewhere Terraform has no hand in building, which breaks the
+cycle outright. Local state would make Terraform workstation-only; CI needs
+to see the same state.
+
+The connection string holds a password, so it is passed at init rather than
+committed:
+
+```sh
+terraform init \
+  -backend-config="conn_str=postgres://terraform:$PG_PASSWORD@192.168.50.81:5432/terraform_state"
+```
+
+The bootstrap dependency this creates is on **k3s, not Unraid**: the cluster
+and its CNPG instance must be up, with a `terraform_state` database and a
+`terraform` role created on it, before `terraform init` works. That is the
+one cross-platform dependency the rest of the repo avoids, and it is
+deliberate — the alternative is a local-state-then-`init -migrate-state`
+dance that has to be got right exactly once. Ordering is in
+`docs/postgres-proxmox.md`.
+
+## Credentials
+
+Proxmox API token and the Postgres connection string both come from Vault
+(`homelab/ci/proxmox`, `homelab/ci/terraform`), exported as environment
+variables before running — the same pattern `group_vars/all.yml` uses for
+`VAULT_ADDR` and friends. Nothing authenticating to anything is committed.
+
+```sh
+export TF_VAR_proxmox_endpoint='https://turtle-proxmox-01.home.turtlesystems.co.uk:8006/'
+export TF_VAR_proxmox_api_token='root@pam!terraform=<secret>'
+```
+
+## Status
+
+Not yet applied against anything. The provider version is intentionally
+unpinned until the first `terraform init` — pin what it resolves and commit
+`.terraform.lock.hcl` (which is not gitignored).

build/config/terraform/main.tf

@@ -0,0 +1,102 @@
+# Per-app LXC modules are called from here, so there is one place that knows
+# what infrastructure exists. Each module lives with its app in
+# src/<app>/terraform/ and outputs the guest's address, which the Ansible
+# side then picks up from the Proxmox API (inventory/proxmox.yml) rather than
+# by wiring outputs into inventory by hand.
+
+# The shared Postgres, and the first module here that is infrastructure other
+# modules depend on rather than an app in its own right. Declared first for
+# readability only — Terraform orders by dependency, not by position, and
+# nothing else in this file references it. The ordering that does matter is
+# on the Ansible side, where a guest's `db:` provisioning needs this cluster
+# already answering; see the runbook in docs/postgres-proxmox.md.
+module "postgres" {
+  source = "../../../src/shared/postgres/terraform"
+
+  node_name        = var.proxmox_node
+  template_file_id = var.lxc_template_file_id
+  ssh_public_keys  = var.ssh_public_keys
+
+  hostname   = "postgres"
+  vm_id      = 161
+  ip_address = "192.168.50.54/24"
+  gateway    = "192.168.50.254"
+
+  cores     = 2
+  memory    = 2048
+
+  # Sized for every database this host will ever hold, not for the OS. The
+  # one-volume rule below means growth happens here rather than by adding a
+  # second disk: `terraform apply` turns an increase into a `pct resize` of
+  # the rootfs, which on ZFS is a refquota change — online, no data move, no
+  # filesystem grow step. It is one-way, though; shrinking is a replacement
+  # and `prevent_destroy` blocks it, so overshoot rather than creep upward.
+  disk_size = 256
+
+  # The ZFS pool, so the guest's volume can be replicated at all, and the
+  # second node to replicate it to. Both are the whole point of this module
+  # differing from the one below — see src/shared/postgres/terraform/main.tf.
+  datastore_id            = "AppData"
+  replication_target_node = var.proxmox_replication_node
+}
+
+module "forgejo" {
+  source = "../../../src/forgejo/terraform"
+
+  # The second node, not `var.proxmox_node` — a deliberate split of the two
+  # guests across the cluster rather than an oversight. Written as a literal
+  # here for the same reason `hostname`/`vm_id`/`ip_address` are: it states
+  # where this one guest goes. `var.proxmox_replication_node` happens to hold
+  # the same string today, but it means "where Postgres replicates to", and
+  # borrowing it would tie Forgejo's placement to a decision about the
+  # database.
+  #
+  # Safe to change only while this container does not exist. Once it does, the
+  # provider treats `node_name` as a replacement — and `prevent_destroy` in
+  # the module turns that into a failed plan, which is the intended outcome:
+  # moving a live Forgejo between nodes is a Proxmox migration, not a
+  # `terraform apply`.
+  node_name        = "turtle-proxmox-02"
+  template_file_id = var.lxc_template_file_id
+  ssh_public_keys  = var.ssh_public_keys
+
+  hostname   = "forgejo"
+  vm_id      = 160
+  ip_address = "192.168.50.52/24"
+  gateway    = "192.168.50.254"
+
+  cores     = 2
+  memory    = 2048
+
+  # Overrides the module default only in the sense of restating it; both are
+  # 256 GiB. Growing this is one-way, same as the Postgres guest above:
+  # `terraform apply` turns an increase into an online `pct resize` of the
+  # rootfs, but shrinking is a replacement and `prevent_destroy` blocks it.
+  # Sized generously because *everything* Forgejo owns is on this volume —
+  # repos, LFS, attachments, indexers, and the local database dumps.
+  disk_size = 256
+
+  # The ZFS pool, which exists on both nodes, rather than the module's
+  # `local-lvm` default — this cluster has no storage by that name. No
+  # replication job to go with it, unlike the Postgres guest above: Forgejo's
+  # off-box copy is the vzdump archive `pve_backup` writes to the NAS, which
+  # is a complete filesystem restore point precisely because nothing here is
+  # bind-mounted.
+  datastore_id = "AppData"
+}
+
+# Consumed by the Proxmox nodes' host_vars, which can pin the vzdump job to
+# explicit VMIDs. Kept as an output rather than restated there so the two
+# can't drift.
+output "backup_vmids" {
+  description = "VMIDs of the guests this configuration creates, for the vzdump job."
+  value       = [module.postgres.vm_id, module.forgejo.vm_id]
+}
+
+# The address every app's `db.provision_host` and `DB_HOST` has to agree with.
+# An output rather than something to look up in the Proxmox UI, because it is
+# copied into several files by hand and this is the one authoritative copy.
+output "postgres_address" {
+  description = "LAN address of the shared Postgres LXC."
+  value       = module.postgres.ip_address
+}

build/config/terraform/providers.tf

@@ -0,0 +1,42 @@
+terraform {
+  required_version = ">= 1.6"
+
+  required_providers {
+    proxmox = {
+      source = "bpg/proxmox"
+      # Deliberately unpinned until the first real `terraform init` — pin to
+      # whatever it resolves, and commit .terraform.lock.hcl (which is not
+      # gitignored) so everyone and CI get the same provider.
+      # version = "~> 0.0"
+    }
+  }
+
+  # State in Postgres rather than a local file: CI and your workstation need
+  # to see the same state, and a local file makes Terraform workstation-only.
+  #
+  # Specifically the CloudNativePG cluster on k3s, reached at
+  # k3s_postgres_loadbalancer_ip — *not* the shared Postgres this
+  # configuration itself provisions on Proxmox. That would be circular:
+  # Terraform would need the database to exist in order to create the
+  # container the database runs in. Putting state on a cluster this
+  # configuration has no hand in building breaks the cycle outright, which is
+  # why it is worth the cross-platform dependency the rest of the repo
+  # otherwise avoids — `terraform apply` now needs the Pis up.
+  #
+  # Left empty on purpose — the connection string contains a password, so it
+  # is supplied at init time instead of being committed:
+  #
+  #   terraform init \
+  #     -backend-config="conn_str=postgres://terraform:$PG_PASSWORD@192.168.50.81:5432/terraform_state"
+  #
+  # Bootstrap ordering: the k3s cluster and its CNPG instance must exist (and
+  # a `terraform_state` database be created on it) before `terraform init`
+  # succeeds. See docs/postgres-proxmox.md.
+  backend "pg" {}
+}
+
+provider "proxmox" {
+  endpoint  = var.proxmox_endpoint
+  api_token = var.proxmox_api_token
+  insecure  = var.proxmox_insecure
+}

build/config/terraform/variables.tf

@@ -0,0 +1,78 @@
+variable "proxmox_endpoint" {
+  description = "Proxmox VE API endpoint, e.g. https://turtle-proxmox-01.home.turtlesystems.co.uk:8006/"
+  type        = string
+}
+
+variable "proxmox_api_token" {
+  description = <<-EOT
+    Proxmox API token in `USER@REALM!TOKENID=SECRET` form. Sourced from Vault
+    (homelab/ci/proxmox), never committed — pass via the TF_VAR_proxmox_api_token
+    environment variable.
+  EOT
+  type        = string
+  sensitive   = true
+}
+
+variable "proxmox_insecure" {
+  description = "Skip TLS verification — true while Proxmox has a self-signed cert."
+  type        = bool
+  default     = true
+}
+
+variable "proxmox_node" {
+  description = <<-EOT
+    Proxmox node guests are created on. Must be the node's **short** name as
+    the cluster knows it (`pvecm nodes`), never an FQDN, even though the API
+    endpoint is one.
+
+    pveproxy compares this against its own short hostname to decide whether a
+    request is local. An FQDN never matches, so it proxies the request onward
+    — to itself — over a connection it verifies against the cluster CA, which
+    a self-signed cert fails. The result is an HTTP 596 carrying an OpenSSL
+    `certificate verify failed`, which reads like a TLS misconfiguration on
+    this side and is not: `proxmox_insecure` governs the provider's own
+    connection and has no bearing on Proxmox's internal one.
+  EOT
+  type        = string
+  default     = "turtle-proxmox-02"
+}
+
+variable "proxmox_replication_node" {
+  description = <<-EOT
+    The other node in the cluster, which guests on a ZFS pool replicate to.
+    Only the shared Postgres module uses this today; it lives here rather than
+    in that module's own defaults because it names a node of this cluster, the
+    same as `proxmox_node` does.
+
+    Null disables replication. Set it explicitly even on a single-node
+    cluster — a null default is indistinguishable from an oversight, and the
+    failure mode is a database with no second copy that looks entirely
+    healthy.
+  EOT
+  type        = string
+  default     = "turtle-proxmox-01"
+}
+
+variable "lxc_template_file_id" {
+  description = <<-EOT
+    Container template every LXC module is built from, e.g.
+    `local:vztmpl/debian-12-standard_12.7-1_amd64.tar.zst`. Must already be
+    downloaded on the node — `pveam list local` shows what is there, `pveam
+    available`/`pveam download local <name>` fetches one. Not defaulted: the
+    exact filename moves with each point release, so a stale default would fail
+    at apply time rather than here.
+  EOT
+  type        = string
+}
+
+variable "ssh_public_keys" {
+  description = <<-EOT
+    Public keys installed for root in every container, so Ansible can reach
+    them. The matching private key is in Vault at `homelab/ci/ssh` — see "SSH
+    access" in the top-level README.md. Public keys aren't secret, but they are
+    site-specific, so pass rather than commit:
+
+      export TF_VAR_ssh_public_keys='["'"$(cat ~/.ssh/unraid_ansible.pub)"'"]'
+  EOT
+  type        = list(string)
+}

docs/authentik-migration.md

@@ -0,0 +1,420 @@
+# Migrating Authentik from Unraid to k3s
+
+A one-time, hand-run cutover: move the Authentik database off Unraid onto the
+cluster's CloudNativePG instance, bring Authentik up on k3s from
+`src/authentik/`, verify it, then change DNS.
+
+This is deliberately not automated. It runs once, most of its steps are
+irreversible in the wrong order, and the verification between them is a
+judgement call rather than a task result — the sort of thing a playbook makes
+harder rather than easier. The repo half *is* automated: everything from
+"deploy Authentik" onwards is `playbooks/k3s.yml`, and re-running it later is
+an ordinary converge.
+
+Read the whole thing before starting. The dump is taken with Authentik
+stopped, so there's a service outage from step 5 to step 11 — budget an hour
+and do it when nobody needs to log in.
+
+---
+
+## What moves, and what doesn't
+
+| | Where it lives now | How it gets across |
+|---|---|---|
+| Users, groups, applications, providers, flows, tokens, certificates | Postgres | `pg_dump` \| `psql` (step 7) |
+| Uploaded icons and flow backgrounds | `/media` on disk | `scp` + `kubectl cp` (step 9) |
+| Session cookies | signed by `AUTHENTIK_SECRET_KEY` | carry the key over (step 2) |
+| Cache / task queue | in-process | nothing — recent Authentik has no Redis, and the chart has no Redis dependency |
+
+Authentik stores its certificates (including the signing keys OIDC/SAML
+providers use) **in the database**, not in a `certs/` directory, so a
+successful database restore carries them. That is what lets integrations like
+Shelfarr's OIDC (`src/arr/`) keep working without being re-registered.
+
+The one piece with no automatic path is anything you mounted into the Unraid
+container by hand — custom templates, a `certs/` directory you populated
+yourself. Step 1 is where you find out whether you have any.
+
+---
+
+## 1. Gather the facts
+
+Everything below needs these. Run on the Unraid box (web terminal, or SSH).
+
+```sh
+# Which containers make up the current deployment, and what image/tag.
+docker ps -a --format '{{.Names}}\t{{.Image}}\t{{.Status}}' | grep -i authentik
+```
+
+Write down the **image tag** — that is the Authentik version, and step 3
+depends on it.
+
+```sh
+# Every mount, so you can see where /media lives and spot anything unexpected.
+docker inspect authentik-server \
+  --format '{{range .Mounts}}{{.Source}} -> {{.Destination}}{{println}}{{end}}'
+
+# The database connection it currently uses, and the secret key.
+docker inspect authentik-server \
+  --format '{{range .Config.Env}}{{println .}}{{end}}' | grep -E 'POSTGRES|SECRET_KEY'
+```
+
+That last command tells you the thing this document can't assume: **which
+Postgres the Unraid Authentik talks to**. Two possibilities, and they change
+step 7:
+
+- `AUTHENTIK_POSTGRESQL__HOST=shared-postgres` (or `192.168.50.1`) — it uses
+  the shared instance this repo already manages, `src/shared/postgres/`. The
+  superuser password is in Vault at `homelab/shared/postgres`.
+- anything else (`authentik-postgresql`, `postgres`, a container name from
+  Authentik's own compose bundle) — the Unraid UI deployment brought its own
+  Postgres container. You'll need *its* credentials, which are in that
+  container's own environment:
+
+  ```sh
+  docker inspect <that-container> \
+    --format '{{range .Config.Env}}{{println .}}{{end}}' | grep POSTGRES
+  ```
+
+  and it is probably not published on a host port, which step 7 handles.
+
+Also note `AUTHENTIK_SECRET_KEY` from the output above — step 2 needs it
+verbatim.
+
+---
+
+## 2. Populate Vault
+
+```sh
+vault kv put kv/homelab/authentik \
+  AUTHENTIK_SECRET_KEY='<the value from step 1>' \
+  AUTHENTIK_POSTGRESQL__PASSWORD='<a new strong password>'
+```
+
+Two different rules for those two values, and it's worth being clear about
+why:
+
+- **`AUTHENTIK_SECRET_KEY` must be the existing one.** It signs session
+  cookies. A fresh value doesn't break anything permanently, but it
+  invalidates every active session at the moment DNS flips — so instead of a
+  silent cutover, everyone gets logged out and the migration announces
+  itself. Copy it across.
+- **`AUTHENTIK_POSTGRESQL__PASSWORD` can be new.** You are creating a new
+  role on a different Postgres; nothing in the dump references it. Generate a
+  fresh one.
+
+If you're also using cert-manager's DNS-01 solver (step 4), populate its
+credentials now too:
+
+```sh
+vault kv put kv/homelab/k3s-cert-manager \
+  CLOUDFLARE_API_TOKEN='<token with Zone:DNS:Edit on the zone>'
+```
+
+---
+
+## 3. Confirm the repo's values
+
+Three values are guesses until you check them. All are flagged in-file.
+
+**`src/authentik/common/vars.yml`**
+
+- `AUTHENTIK_VERSION` — **set this to the image tag from step 1**, not to
+  whatever is newest. Authentik runs Django migrations on startup against
+  whatever schema it finds, and they are one-way: a newer Authentik pointed
+  at an older dump upgrades the schema silently, and if the cutover then has
+  to be rolled back, the Unraid instance can no longer read its own database.
+  Migrate first, upgrade second, in a separate commit.
+- `AUTHENTIK_HOST` / `AUTHENTIK_URL` — the hostname you'll be moving. Must
+  match what's there today, or every OIDC/SAML redirect URI registered in
+  Authentik breaks.
+
+**`build/config/ansible/inventory/group_vars/k3s_cluster.yml`**
+
+- `k3s_cert_manager_solver` — empty by default, and
+  `roles/k3s_cert_manager` refuses to run until it's set. The commented
+  Cloudflare example in that file is the common case; for anything else take
+  the stanza from
+  [cert-manager's DNS-01 docs](https://cert-manager.io/docs/configuration/acme/dns01/).
+
+While you're testing, point `k3s_cert_manager_acme_server` at Let's Encrypt
+staging. Production allows five failed validations per hostname per hour, and
+exhausting it means waiting rather than retrying.
+
+---
+
+## 4. Deploy cert-manager and confirm the issuer works
+
+```sh
+cd build/config/ansible
+ansible-playbook playbooks/k3s.yml --tags cert-manager
+```
+
+Then check it actually registered — a `ClusterIssuer` is created whether or
+not its solver is valid, so "the resource exists" proves nothing:
+
+```powershell
+$env:KUBECONFIG = "$PWD\local\k3s\homelab-utils.kubeconfig"
+kubectl get clusterissuer letsencrypt -o jsonpath='{.status.conditions[*].message}'
+```
+
+You want `The ACME account was registered with the ACME server`. Anything
+else, fix it here — not after Authentik is deployed and waiting on a
+certificate.
+
+Worth knowing: with DNS-01, the certificate can be issued **before** DNS
+points at the cluster. The challenge is a TXT record, not a request to your
+web server, so cert-manager never needs to be reachable. That's why TLS is
+sorted out in this step and not after the flip.
+
+---
+
+## 5. Freeze the Unraid instance
+
+Stop writes before dumping. Skipping this gives you a dump that's missing
+whatever happened during it — usually invisible, occasionally a user account
+that no longer exists.
+
+```sh
+docker stop authentik-server authentik-worker
+```
+
+Leave its Postgres running — step 7 reads from it. **The outage starts
+here.**
+
+---
+
+## 6. Create the role and database on CNPG
+
+The restore needs the `authentik` role to exist first so the dump's objects
+land with the right ownership.
+
+```powershell
+$env:KUBECONFIG = "$PWD\local\k3s\homelab-utils.kubeconfig"
+kubectl -n shared-postgres exec -i shared-postgres-1 -- psql -v ON_ERROR_STOP=1 <<'SQL'
+CREATE ROLE authentik LOGIN PASSWORD 'the-password-from-step-2';
+CREATE DATABASE authentik OWNER authentik;
+SQL
+```
+
+`kubectl exec` into a CNPG pod lands you as the `postgres` superuser over the
+local socket, so bare `psql` needs no credentials.
+
+You could instead let `roles/k3s_app` create these — its tasks are idempotent
+and will run in step 8 regardless — but doing it by hand here keeps the
+database creation and the restore adjacent, and avoids deploying Authentik
+against an empty database first (which would have it run initial migrations
+and build a schema the restore then has to fight).
+
+---
+
+## 7. Dump and restore
+
+Run the whole thing **inside the CNPG pod**, piping `pg_dump` straight into
+`psql`. Three reasons this is better than dumping to a file on Windows:
+
+- `pg_dump` must be at least the version of the server being dumped, and the
+  CNPG pod ships Postgres 18 client tools — dumping the Unraid Postgres 13
+  from there is the supported direction. Your workstation probably has no
+  `pg_dump` at all.
+- PowerShell re-encodes bytes passing through `>` and `|`, which silently
+  corrupts a dump file. Nothing on the Windows filesystem means nothing to
+  corrupt.
+- No intermediate file holding the entire user database in cleartext.
+
+### If Authentik used the shared Postgres (published on `192.168.50.1:5432`)
+
+```powershell
+$env:KUBECONFIG = "$PWD\local\k3s\homelab-utils.kubeconfig"
+$srcPw = Read-Host 'Unraid postgres superuser password'
+
+kubectl -n shared-postgres exec -i shared-postgres-1 -- `
+  env "PGPASSWORD=$srcPw" bash -c @'
+set -euo pipefail
+{ echo "SET ROLE authentik;"
+  pg_dump -h 192.168.50.1 -p 5432 -U postgres -d authentik \
+          --no-owner --no-privileges
+} | psql -v ON_ERROR_STOP=1 -d authentik
+'@
+```
+
+`SET ROLE authentik` before the dump stream, combined with `--no-owner`, is
+what gets ownership right: the session creates every object as `authentik`,
+so the application role owns its own schema. Without it, everything lands
+owned by `postgres` and Authentik can read but not migrate.
+
+`ON_ERROR_STOP=1` matters — without it `psql` reports success having skipped
+every statement that failed.
+
+### If it used its own Postgres container (not published)
+
+Publish it temporarily, then use the command above with the container's own
+credentials and port:
+
+```sh
+# on Unraid
+docker run -d --rm --name pgbridge -p 55432:5432 \
+  --network container:<authentik-postgres-container> alpine/socat \
+  TCP-LISTEN:5432,fork TCP:127.0.0.1:5432
+```
+
+then `-h 192.168.50.1 -p 55432 -U <its user>` in the `pg_dump` above, and
+`docker stop pgbridge` afterwards. Simpler alternative if you'd rather not:
+`docker exec <container> pg_dump ... > /mnt/user/appdata/authentik.sql` on
+Unraid, `scp` it to your workstation, and `kubectl exec -i ... -- psql -d
+authentik < authentik.sql` — but a plain-SQL file through PowerShell
+redirection is exactly the encoding hazard noted above, so run that last
+command from WSL.
+
+### Check the restore
+
+```powershell
+kubectl -n shared-postgres exec -i shared-postgres-1 -- `
+  psql -d authentik -c '\dt' -c 'SELECT count(*) FROM authentik_core_user;'
+```
+
+You should see a long table list and a user count matching what the old
+instance had. An empty table list means the restore did nothing.
+
+---
+
+## 8. Deploy Authentik on k3s
+
+```sh
+cd build/config/ansible
+ansible-playbook playbooks/k3s.yml --tags apps
+```
+
+A green run means the manifests landed and the database was provisioned — not
+that the workload came up. Watch it actually start:
+
+```powershell
+kubectl -n authentik get pods -w
+```
+
+The first start takes several minutes on a Pi: pulling the image, then the
+worker checking migrations. Because the deployed version matches the dumped
+version (step 3), it should find nothing to migrate.
+
+If pods stay `Pending`, check the PVC — `local-path` uses
+`WaitForFirstConsumer`, so a `Pending` PVC alongside a `Pending` pod is
+normal only until the scheduler picks a node:
+
+```powershell
+kubectl -n authentik describe pvc authentik-media
+kubectl -n authentik logs -l app.kubernetes.io/component=worker --tail=50
+```
+
+---
+
+## 9. Copy `/media` across
+
+Two hops rather than a pipe, again to keep binary data out of a PowerShell
+pipeline:
+
+```powershell
+# from the media path you noted in step 1
+scp -r root@192.168.50.1:/mnt/user/appdata/authentik/media ./local/authentik-media
+
+$pod = kubectl -n authentik get pod -l app.kubernetes.io/component=server `
+  -o jsonpath='{.items[0].metadata.name}'
+kubectl -n authentik cp ./local/authentik-media/. "${pod}:/media"
+```
+
+`local/` is already gitignored, so the staging copy won't be committed.
+
+Skip this entirely if you never uploaded custom icons or backgrounds —
+Authentik falls back to its built-in assets, and the directory is empty on a
+stock install.
+
+---
+
+## 10. Verify before touching DNS
+
+The hostname still resolves to Unraid, so test by overriding resolution
+rather than changing it. `--resolve` sends the request to Traefik with the
+right SNI and `Host` header:
+
+```powershell
+curl.exe -sv --resolve auth.turtlesystems.uk:443:192.168.50.80 `
+  https://auth.turtlesystems.uk/-/health/ready/
+```
+
+Check, in order:
+
+1. The TLS certificate is the Let's Encrypt one, not Traefik's self-signed
+   default (`kubectl -n authentik get certificate` → `READY  True`). If you
+   used the staging server in step 3, it will be untrusted — expected; switch
+   to production and re-run `--tags cert-manager` once satisfied.
+2. `/-/health/ready/` returns 204.
+3. Add a hosts-file entry pointing the name at `192.168.50.80` and log in
+   through a browser with a **real user account**. This is the check that
+   matters — it exercises the restored password hashes, the flows, and the
+   session cookie signed with the carried-over secret key.
+4. Open an existing application from the Authentik dashboard and confirm the
+   OIDC/SAML round-trip still works (Shelfarr is the one this repo knows
+   about — see `src/arr/`).
+
+Remove the hosts-file entry afterwards.
+
+---
+
+## 11. Flip DNS
+
+Point `auth.turtlesystems.uk` at `192.168.50.80` (Traefik's MetalLB address).
+
+Lower the record's TTL a few hours beforehand if you can — it shortens the
+window in which a rollback is still invisible to clients.
+
+---
+
+## Rollback
+
+Viable until users start making changes in the new instance; after that,
+rolling back loses whatever they did.
+
+1. Point DNS back at Unraid.
+2. `docker start authentik-server authentik-worker`.
+
+The old database was only ever read, never written, so it's exactly as it was
+at step 5. Nothing to restore.
+
+Do **not** run both instances at once against their separate databases — they
+diverge immediately, and there is no way to merge them afterwards. That is
+the same "never declare an app on two platforms" rule from CLAUDE.md, and it
+is why `src/authentik/ansible/unraid/` is a README rather than a compose file.
+
+---
+
+## After it has settled
+
+Once you're confident, in this order:
+
+1. **Delete the Unraid containers.** A UI action, not `state: absent` — this
+   repo never deployed them, so it has nothing to tear down. Keep the appdata
+   directory a while longer.
+2. **Drop the old database**, if it was on the shared Postgres:
+   ```sh
+   docker exec -it shared-postgres psql -U postgres -c 'DROP DATABASE authentik;'
+   ```
+   Take a copy first if you want one; nothing else does.
+3. **Upgrade Authentik**, now as an ordinary change: bump
+   `AUTHENTIK_VERSION` in `src/authentik/common/vars.yml`, read the release
+   notes for anything between your version and the target, and
+   `ansible-playbook playbooks/k3s.yml --tags apps -e only_apps=authentik`.
+   Do this as its own commit, so a bad upgrade is one `git revert` away from
+   the version you know worked.
+
+## Backups, afterwards
+
+Worth stating plainly: the cluster's CNPG has **no backups configured**
+(CLAUDE.md → "Key decisions" — HA replication only). Before this migration,
+Authentik's database sat on Unraid, inside whatever covers `/mnt/user/appdata`.
+After it, the only redundancy is a streaming replica on another Pi, which
+protects against a dead SD card and not against a bad migration, a dropped
+table, or a mistake in this document.
+
+The gap this leaves is real and this cutover widens it. CNPG's answer is a
+`ScheduledBackup` to object storage or an NFS volume on the NAS; that's not
+built here yet, and it's the obvious next piece of work after Authentik is
+settled.

docs/forgejo-proxmox.md

@@ -0,0 +1,287 @@
+# Bringing up Forgejo on Proxmox
+
+A run-once bootstrap, same shape as `postgres-proxmox.md` and deliberately
+downstream of it — Forgejo's database lives on the shared Postgres that
+runbook creates, so none of this works until that one is finished.
+
+The end state: an LXC on the `AppData` pool of **turtle-proxmox-02** (VMID
+160, 192.168.50.52), running Forgejo natively under systemd, with its database
+on the shared Postgres LXC at 192.168.50.54 and a nightly `pg_dump` landing on
+its own disk in time for the node's 02:00 vzdump.
+
+## Why turtle-proxmox-02
+
+Nothing forces it. Postgres is on `-01`, so putting Forgejo on `-02` splits
+the two guests across the cluster: a node going down takes one of them with
+it rather than both. Note what that does *not* buy — Forgejo with its database
+unreachable is not a working forge, so this is about not losing both
+filesystems at once, not about staying up.
+
+It is set as a literal `node_name` on the `forgejo` module in
+`build/config/terraform/main.tf`, not from `var.proxmox_node`. **Change it
+only before the container exists**: the provider treats `node_name` as a
+replacement, and `prevent_destroy` in `src/forgejo/terraform/main.tf` turns
+that into a failed plan. Moving a live Forgejo is a Proxmox migration
+(`pct migrate`) followed by editing the literal to match, not a
+`terraform apply`.
+
+## 1. Secrets
+
+`homelab/forgejo` needs five keys. Four of them may already exist from the
+Unraid stack; **`LFS_JWT_SECRET` almost certainly does not** — the Compose
+deployment let Forgejo generate it, and the native install renders `app.ini`
+in full with `INSTALL_LOCK = true`, so there is no first boot for Forgejo to
+invent one on.
+
+```sh
+vault kv get kv/homelab/forgejo
+```
+
+If `LFS_JWT_SECRET` is missing, generate one and patch it in without
+disturbing the others:
+
+```sh
+vault kv patch kv/homelab/forgejo \
+  LFS_JWT_SECRET="$(docker run --rm codeberg.org/forgejo/forgejo:10 forgejo generate secret)"
+```
+
+Full key table in `vault-secrets.md`. `install.yml` asserts on all five before
+it touches the container, so a missing one fails on the first task with a
+message naming it rather than three tasks later on an undefined variable.
+
+## 2. The container template
+
+The template must be present on **turtle-proxmox-02**, not just on `-01` where
+Postgres was built. If `StorageOne` is not shared across the cluster this is a
+separate download:
+
+```sh
+ssh root@turtle-proxmox-02 'pveam list StorageOne'
+# if the ubuntu-26.04 template is absent:
+ssh root@turtle-proxmox-02 'pveam update && pveam download StorageOne ubuntu-26.04-standard_26.04-1_amd64.tar.zst'
+```
+
+Whatever is there has to match `lxc_template_file_id` exactly — the filename
+moves with each point release, which is why that variable has no default.
+
+## 3. Terraform
+
+State is on the CNPG cluster, so the Pis have to be up (see
+`postgres-proxmox.md` § "Why this order").
+
+```sh
+cd build/config/terraform
+terraform init -backend-config=...        # as per postgres-proxmox.md
+terraform plan -out=deploy.plan -target=module.forgejo
+terraform apply deploy.plan
+```
+
+Read the plan before applying. It should be **one resource to add** and
+nothing to change or destroy; anything touching `module.postgres` — in
+particular a *replacement* — means stop and work out why, because that guest
+is live.
+
+Note `deploy.plan` embeds the state and every input variable, `sensitive` ones
+included, so it holds the Proxmox API token in the clear. `.gitignore` covers
+`*.plan`; delete it once applied rather than leaving it in the tree.
+
+Confirm the guest exists and Ansible can see it:
+
+```sh
+ssh root@turtle-proxmox-02 'pct list | grep 160'
+cd ../ansible && ansible-inventory --list --yaml proxmox_guests | grep -A2 forgejo
+```
+
+If it is absent from the inventory but present in `pct list`, the `terraform`
+tag is missing — that filter is what tells this repo's guests from hand-made
+ones, and a guest without it is silently never deployed to.
+
+## 4. Install Forgejo into it
+
+```sh
+cd build/config/ansible
+ansible-playbook playbooks/proxmox.yml -e only_stacks=forgejo
+```
+
+This provisions the `forgejo` database and role on 192.168.50.54, then
+installs the binary, `app.ini`, the systemd unit and the dump timer. The
+database step runs from **your controller**, not from inside the container
+(`delegate_to: localhost`), so 192.168.50.54:5432 has to be reachable from
+wherever you run this.
+
+Check:
+
+```sh
+ssh root@192.168.50.52 'systemctl is-active forgejo && systemctl list-timers forgejo-dbdump.timer'
+curl -sI http://192.168.50.52:3000/ | head -1
+```
+
+## 4b. The first admin user
+
+A fresh install has **no way in**: `INSTALL_LOCK = true` skips the setup
+wizard, which is where the first admin would normally be created, and
+`DISABLE_REGISTRATION = true` means you cannot self-register either (both in
+`src/forgejo/ansible/proxmox/templates/app.ini.j2`). Nothing in `install.yml`
+creates a user, so the account has to be made from the CLI inside the
+container:
+
+```sh
+ssh root@192.168.50.52
+
+sudo -u git forgejo admin user create   --admin   --username <you>   --email <you>@turtlesystems.co.uk   --password '<strong-password>'   --config /etc/forgejo/app.ini   --work-path /var/lib/forgejo
+```
+
+Three things that are easy to get wrong:
+
+- **`sudo -u git`, not root.** `RUN_USER` is `git`; running the CLI as root
+  leaves root-owned files under `/var/lib/forgejo` that Forgejo then cannot
+  write. `app.ini` is `0640 root:git`, so the `git` user can read it.
+- **`--config` is not optional.** Forgejo looks for `custom/conf/app.ini`
+  under the work path by default; this install puts it at
+  `/etc/forgejo/app.ini` (`FORGEJO_CONFIG_DIR` in `vars.yml`). Without the
+  flag the CLI reads a config that isn't there and never reaches the
+  database.
+- Recent versions default `--must-change-password` to true, so expect a
+  forced change on first web login. Pass `--must-change-password=false` to
+  skip it. Confirm the flags for the pinned `FORGEJO_RELEASE` with
+  `forgejo admin user create --help` rather than assuming.
+
+Then log in at `http://192.168.50.52:3000/user/login` — by IP, because
+`ROOT_URL` is `https://git.turtlesystems.uk` and nothing resolves there until
+the cutover in step 7. The login form works over the IP; some links and
+redirects Forgejo renders will point at the not-yet-live hostname.
+
+**Skip this step if you are restoring a database in step 6.** Users live in
+the database, so a dump brings its own admin back and an account created here
+is overwritten by the restore. Create one only if this is genuinely an empty
+forge — or after the restore, if the dump turns out to have no usable admin.
+
+## 5. Node backups
+
+VMID 160 is on `-02`, so it is `-02`'s vzdump job that covers it.
+`pve_backup_all: true` in `group_vars/proxmox_nodes.yml` means no edit is
+needed — but the job has to actually exist on that node:
+
+```sh
+ansible-playbook playbooks/pve_host.yml --limit turtle-proxmox-02
+ssh root@turtle-proxmox-02 'cat /etc/pve/jobs.cfg'
+```
+
+The 01:30 dump and the 02:00 vzdump are a pair. Move one, move the other —
+see `src/forgejo/ansible/proxmox/README.md`.
+
+## 6. Repository data
+
+**Ansible does not move any of this.** A fresh install is a working, empty
+forge; the repositories are a separate restore into
+`/var/lib/forgejo/data/forgejo-repositories`.
+
+The original instructions here were an `rsync` off nas2, which no longer
+exists as a machine to read from — whatever repository data survives has to
+come from wherever it went when that host was retired. After restoring:
+
+```sh
+ssh root@192.168.50.52 'chown -R git:git /var/lib/forgejo/data && systemctl restart forgejo'
+```
+
+Restore the database the same way, from whatever dump you have, before
+starting Forgejo against it — an empty database with populated repositories
+gives back every file and no issues, pull requests, users or permissions.
+
+## 7. Cutover
+
+Last, once the above is verified:
+
+- Point `git.turtlesystems.uk` at this container.
+- **SSH clone URLs move from port 2222 to 22.** The Unraid stack published
+  2222 to dodge the host's own sshd; this container has its own IP and uses
+  its own sshd, with Forgejo managing the `git` user's `authorized_keys`.
+  Every existing remote needs editing — there is no redirect for this.
+- The Forgejo Actions runner registration is tied to the instance. Re-register
+  it against the new host, or CI stops running (`.forgejo/workflows/`).
+
+## 8. Sign-in through Authentik (optional)
+
+Forgejo supports OIDC, and `app.ini` is already set up for it — but only the
+*policy* half. The provider itself cannot be configured from `app.ini`:
+Forgejo keeps authentication sources in its **database**. That makes this the
+one part of Forgejo's configuration Ansible does not own, and the reason it is
+a runbook step rather than a task in `install.yml`. It is run once and then
+carried forward by the nightly `pg_dump`, the same as every other row in that
+database.
+
+**Do this after step 7, not before.** The redirect URI has to match `ROOT_URL`
+(`https://git.turtlesystems.uk`), so DNS and TLS must already be live.
+Configuring it against `http://192.168.50.52:3000` means doing it twice.
+
+### On the Authentik side
+
+Create an OAuth2/OIDC **Provider** plus an **Application** for it, as for any
+other app on `auth.turtlesystems.uk`. The redirect URI is:
+
+```
+https://git.turtlesystems.uk/user/oauth2/authentik/callback
+```
+
+The last path segment is the *name of the auth source in Forgejo*, not a fixed
+string — it has to match the `--name` below. Store the generated client secret
+in Vault (`vault-secrets.md` → `OIDC_CLIENT_SECRET`).
+
+### On the Forgejo side
+
+```sh
+ssh root@192.168.50.52
+
+sudo -u git forgejo admin auth add-oauth   --name authentik   --provider openidConnect   --key <client-id>   --secret <client-secret>   --auto-discover-url https://auth.turtlesystems.uk/application/o/forgejo/.well-known/openid-configuration   --config /etc/forgejo/app.ini   --work-path /var/lib/forgejo
+```
+
+Same `sudo -u git` and `--config` requirements as step 4b, for the same
+reasons. Verify the flag names against the pinned `FORGEJO_RELEASE` with
+`forgejo admin auth add-oauth --help` before running — and list what exists
+afterwards with `forgejo admin auth list`, which is also how you find the `id`
+for `update-oauth` if the secret is ever rotated.
+
+No restart is needed: the source is a database row, not a file Forgejo reads
+at boot.
+
+### What the app.ini side already does
+
+Set in `src/forgejo/ansible/proxmox/templates/app.ini.j2`, so a redeploy keeps
+them — the comments there carry the detail:
+
+| Setting | Effect |
+|---|---|
+| `DISABLE_REGISTRATION = false` | It blocks OIDC auto-registration too, not just the local signup form. Left `true`, Authentik logins authenticate and are then refused an account. |
+| `ALLOW_ONLY_EXTERNAL_REGISTRATION = true` | Restores "no self-service signup" without blocking Authentik. |
+| `ENABLE_AUTO_REGISTRATION = true` | Creates the Forgejo account on first successful SSO login. |
+| `ACCOUNT_LINKING = auto` | Attaches the OIDC identity to an existing local account with the same email. |
+
+`ACCOUNT_LINKING` is the one that matters on the first login: if the
+break-glass admin from step 4b uses the same email address as your Authentik
+account, signing in through Authentik lands you *in that admin account*. With
+linking off you would arrive as a second, unprivileged user and have to
+promote it from the CLI.
+
+### Keep the local admin
+
+Do not delete the step 4b account or convert it to SSO-only. The login path is
+now Forgejo → Authentik → CloudNativePG → a healthy k3s cluster, and any of
+those failing takes SSO down with it — including the cases you would most want
+to log into the forge to investigate. This is the same reasoning
+`roles/k3s_traefik` uses for keeping the port-forward-only dashboard route
+alive alongside the Authentik-published one.
+
+### Git over HTTPS
+
+Users who arrive through Authentik have no Forgejo password, so HTTPS clones
+need a personal access token. SSH keys are unaffected — and since step 7 put
+git-over-SSH on port 22 with Forgejo managing the `git` user's
+`authorized_keys`, SSH is the smoother default to point people at.
+
+## Rolling back
+
+There is no `state: absent` for `lxc_app` — removal would need a per-app
+`uninstall.yml` and Forgejo has none. To back out: stop the service
+(`systemctl stop forgejo`), point DNS back at whatever was serving before, and
+leave the container in place. Destroying it needs the `prevent_destroy` block
+removed by hand first, which is deliberate.

docs/postgres-proxmox.md

@@ -0,0 +1,237 @@
+# Bringing up the shared Postgres on Proxmox
+
+A run-once bootstrap, in the order the dependencies actually demand. Each step
+exists because the one after it cannot start without it; if you already have a
+piece, skip it and check the assertion at the end of the section.
+
+The end state: a Debian LXC on the `AppData` ZFS pool of `turtle-proxmox-01`,
+running Postgres 17, replicated every five minutes to `turtle-proxmox-02`, and
+answering on 192.168.50.54:5432 for every app that declares a `db:`.
+
+## Why this order
+
+Two chains have to be satisfied before `terraform apply` will run at all:
+
+- **State.** Terraform stores state in Postgres. Storing it in the database
+  this configuration provisions would be circular, so it goes on the
+  CloudNativePG cluster on the k3s Pis instead — a cluster Terraform has no
+  hand in building. That cluster therefore has to be up first.
+- **Inventory.** `playbooks/proxmox.yml` targets `proxmox_guests`, a group
+  that only exists because of the API-backed dynamic inventory. Ansible cannot
+  install into a container Terraform has just made until it can see it.
+
+## 1. The k3s cluster and its CNPG instance
+
+Needed only for Terraform state. If the cluster is already up, confirm the
+LoadBalancer answers and move on.
+
+```sh
+cd build/config/ansible
+ansible-playbook playbooks/k3s.yml --tags services
+```
+
+Then, against `k3s_postgres_loadbalancer_ip` (192.168.50.81), create the state
+database and its role. This is the only hand-run SQL in the whole flow —
+everything else provisions itself:
+
+```sql
+CREATE ROLE terraform LOGIN PASSWORD '<generate-a-strong-password>';
+CREATE DATABASE terraform_state OWNER terraform;
+```
+
+Store the resulting connection string at `homelab/ci/terraform` in Vault (see
+`vault-secrets.md`), then check it:
+
+```sh
+psql "postgres://terraform:$PG_PASSWORD@192.168.50.81:5432/terraform_state" -c '\conninfo'
+```
+
+## 2. Secrets
+
+`homelab/shared/postgres` is the same path the Unraid and k3s instances
+already use — one superuser identity for the "shared postgres" concept
+wherever it runs. If it is populated, nothing to do:
+
+```sh
+vault kv get kv/homelab/shared/postgres
+```
+
+`homelab/ci/proxmox` needs a Proxmox API token, and `homelab/ci/ssh` the key
+pair whose public half Terraform installs into the container.
+
+## 3. The container template
+
+On the node, once:
+
+```sh
+pveam available | grep debian-12
+pveam download local debian-12-standard_12.7-1_amd64.tar.zst
+```
+
+## 4. Terraform
+
+```sh
+cd build/config/terraform
+terraform init -backend-config="conn_str=$PG_CONN_STR"
+```
+
+Pin the provider to whatever this resolves and commit the resulting
+`.terraform.lock.hcl`. Then:
+
+```sh
+export TF_VAR_proxmox_endpoint='https://turtle-proxmox-01.home.turtlesystems.co.uk:8006/'
+export TF_VAR_proxmox_api_token='...'          # homelab/ci/proxmox
+export TF_VAR_lxc_template_file_id='local:vztmpl/debian-12-standard_12.7-1_amd64.tar.zst'
+export TF_VAR_ssh_public_keys='["'"$(cat ~/.ssh/unraid_ansible.pub)"'"]'
+
+terraform apply -target=module.postgres
+```
+
+`-target` on purpose: the root module also defines the Forgejo LXC, and
+bringing that up is a separate decision with its own migration story
+(`inventory/host_vars/forgejo.yml`). Drop the flag once you want both.
+
+If the apply 403s on `changing feature flags (except nesting) is only allowed
+for root@pam`, the container already exists without nesting enabled and the
+`ansible@pam` token cannot add it. Set it as root on the node, then re-plan —
+it will read clean, and Postgres's systemd unit needs it:
+
+```sh
+ssh root@turtle-proxmox-01.home.turtlesystems.co.uk 'pct set 161 --features nesting=1'
+ssh root@turtle-proxmox-01.home.turtlesystems.co.uk 'pct reboot 161'
+```
+
+Check the replication job exists — this is the step most easily missed,
+because a container with no job looks identical in the storage view:
+
+```sh
+ssh root@turtle-proxmox-01 'pvesr status'
+```
+
+## 5. Install Postgres into it
+
+```sh
+cd build/config/ansible
+ansible-inventory -i inventory/proxmox.yml --graph      # the guest should appear
+ansible-playbook playbooks/proxmox.yml -e only_stacks=postgres
+```
+
+Then from the controller, confirming both that it listens on the LAN and that
+the Vault password took:
+
+```sh
+psql "postgres://postgres:$PGPASSWORD@192.168.50.54:5432/postgres" -c 'SELECT version();'
+```
+
+## 6. Node backups
+
+Replication covers a dead node, not a dropped table. `roles/pve_backup` writes
+vzdump archives to the NAS:
+
+```sh
+ansible-playbook playbooks/pve_host.yml
+```
+
+## Growing the disk
+
+Everything lives on the rootfs — that is what makes a replication snapshot
+atomic (`src/shared/postgres/terraform/README.md`), so there is no second
+volume to add when space runs short. Raise `disk_size` on the `postgres`
+module in `build/config/terraform/main.tf` and apply.
+
+Check the pool has the room first. Terraform will not: ZFS lets an apply
+overcommit the pool happily, and the failure surfaces later as a write inside
+the guest hitting ENOSPC.
+
+```sh
+ssh root@turtle-proxmox-01 'zpool list AppData; zfs list -o name,used,avail,refquota -r AppData'
+ssh root@turtle-proxmox-02 'zpool list AppData'   # the target needs the room too
+```
+
+Then:
+
+```sh
+cd build/config/terraform
+terraform plan     # expect an in-place update to disk.size, NOT a replacement
+terraform apply
+```
+
+A **replacement** in that plan means something other than the size changed —
+`prevent_destroy` will refuse it, which is the point. Never work around it
+here; replacing this container destroys every database on it.
+
+The resize itself is a `pct resize` of the rootfs, which on ZFS-backed storage
+is a refquota change rather than a partition operation. It applies to a
+running guest, ships no data, and needs nothing done inside the container
+afterwards — no `resize2fs`, no Postgres restart. Confirm from inside:
+
+```sh
+ssh root@192.168.50.54 'df -h /'
+```
+
+Two things the larger number does *not* change: replication traffic, which is
+a function of what actually gets written rather than of the quota, and vzdump
+archive size, which covers used data only. Both track the databases, not the
+headroom.
+
+Shrinking is not available — it is a replacement, and `prevent_destroy` blocks
+it — so overshoot rather than raising this every few months.
+
+## Failing over
+
+Replication makes the far copy a volume, not a running guest, so failover is
+deliberate. Planned, with both nodes up, is an ordinary migration — fast,
+because replication means only the delta has to ship:
+
+```sh
+ssh root@turtle-proxmox-01 'pct migrate 161 turtle-proxmox-02 --restart'
+```
+
+Unplanned, with the source node gone, means telling Proxmox to run the guest
+from the replicated volume, and it loses every transaction committed since the
+last successful send — up to `replication_schedule`, five minutes. Check what
+you are about to accept first:
+
+```sh
+ssh root@turtle-proxmox-02 'pvesr status'      # look at "Last Sync"
+```
+
+Automating this is what a HA resource would add, and it is deliberately not
+configured: automatic failover on a two-node cluster with no third vote is a
+good way to get both nodes deciding they are the survivor.
+
+## Afterwards: repointing clients
+
+The address is written down in several files that cannot discover it. The
+authoritative copy is `ip_address` on the `postgres` module in
+`build/config/terraform/main.tf`, published as the `postgres_address` output.
+
+| Where | What |
+|---|---|
+| `inventory/host_vars/<guest>.yml` | `db.provision_host` on each app that declares one |
+| `src/<app>/ansible/proxmox/vars.yml` | that app's own `DB_HOST` |
+| `src/<app>/ansible/unraid/vars.yml` | ditto, for anything still on Unraid — no `unraid_shared` network alias reaches this host |
+
+Apps on the k3s cluster keep using the in-cluster CloudNativePG instance and
+are unaffected.
+
+## Migrating an existing database onto it
+
+Per database, not `pg_dumpall` — the roles are recreated by the `db:` block on
+each app's own entry, so only the data needs moving:
+
+```sh
+pg_dump -h <old-host> -U postgres -Fc forgejo > forgejo.dump
+# create the role and database by deploying the app once with its `db:` block,
+# then:
+pg_restore -h 192.168.50.54 -U postgres -d forgejo --no-owner --role=forgejo forgejo.dump
+```
+
+`--no-owner --role=` rather than a straight restore: the dump carries
+ownership from the old cluster, and the role provisioning in `lxc_app` /
+`compose_stack` has already created the owner here with a Vault-managed
+password. Restoring ownership from the dump would fight it.
+
+Stop the application before dumping. A dump of a live database is consistent
+as of its start, so anything written during it is silently absent from the
+restore.

docs/vault-secrets.md

@@ -0,0 +1,551 @@
+# Vault secret layout
+
+KV v2 mount: `kv` (matches `vault_kv_mount` in
+`build/config/ansible/inventory/group_vars/all.yml`).
+One path per stack, fetched whole via `community.hashi_vault.vault_kv2_get`.
+
+Paths are prefixed `homelab/`, not `unraid/`: an app may run as a Compose
+stack on Unraid or natively in a Proxmox LXC, and its secrets are the same
+either way. If you populated the old `unraid/*` paths before this rename,
+move them and delete the originals:
+
+```sh
+for p in shared/postgres forgejo ci/ssh; do
+  vault kv get -format=json "kv/unraid/$p" \
+    | jq -r '.data.data | to_entries | map("\(.key)=\(.value)") | .[]' \
+    | xargs vault kv put "kv/homelab/$p"
+  vault kv metadata delete "kv/unraid/$p"
+done
+```
+
+## Prerequisites
+
+- [Vault CLI](https://developer.hashicorp.com/vault/docs/install) installed
+  on whatever machine you're running these commands from (your workstation
+  during bootstrap — not the Unraid box, and not the same thing as the
+  Ansible controller needing `hvac`, though in practice it's the same WSL
+  shell for both).
+- `VAULT_ADDR` set to your Vault instance:
+
+  ```sh
+  export VAULT_ADDR='https://vault.example.internal:8200'
+  ```
+
+  If Vault is using a self-signed cert, either trust it properly or, for a
+  homelab-only instance, set `export VAULT_SKIP_VERIFY=true` — don't do this
+  against anything reachable outside your LAN.
+
+## Login
+
+Log in interactively once per shell session (or whenever the token expires);
+the resulting token is cached to `~/.vault-token` and picked up automatically
+by later `vault` commands and by `VAULT_TOKEN` if you export it.
+
+Token auth (simplest, fine for a single-operator homelab):
+
+```sh
+vault login
+# prompts for the token
+```
+
+If you're using a different auth backend, log in with that method instead —
+adjust to whatever your Vault instance actually has enabled:
+
+```sh
+vault login -method=userpass username=<your-username>
+vault login -method=oidc                                 # opens a browser
+vault login -method=ldap username=<your-username>
+```
+
+Confirm the login worked and check the token's TTL:
+
+```sh
+vault token lookup
+```
+
+## One-time setup: enable the KV v2 mount
+
+Only needed once, the first time this Vault instance is used for this repo
+(skip if `kv/` already exists — check with `vault secrets list`):
+
+```sh
+vault secrets enable -path=kv -version=2 kv
+```
+
+## `homelab/shared/postgres`
+
+| Key | Purpose |
+|---|---|
+| `POSTGRES_SUPERUSER` | Superuser name (matches `vars.yml` default `postgres`, but Vault is authoritative since it's paired with the password below) |
+| `POSTGRES_SUPERUSER_PASSWORD` | Superuser password |
+
+```sh
+vault kv put kv/homelab/shared/postgres \
+  POSTGRES_SUPERUSER=postgres \
+  POSTGRES_SUPERUSER_PASSWORD='<generate-a-strong-password>'
+```
+
+Also seeds the k3s cluster's shared Postgres (`roles/k3s_postgres`,
+CloudNativePG) — a separate physical instance from the Unraid/Proxmox ones,
+but the same superuser identity, same path. See README.md → "Postgres
+(CloudNativePG)".
+
+## `homelab/forgejo`
+
+| Key | Purpose |
+|---|---|
+| `DB_PASSWORD` | Password for the `forgejo` Postgres role (Ansible creates the role with this password) |
+| `SECRET_KEY` | Forgejo `security.SECRET_KEY` |
+| `INTERNAL_TOKEN` | Forgejo `security.INTERNAL_TOKEN` |
+| `JWT_SECRET` | Forgejo `oauth2.JWT_SECRET` |
+| `LFS_JWT_SECRET` | Forgejo `server.LFS_JWT_SECRET` — **Proxmox only**, see below |
+| `OIDC_CLIENT_SECRET` | Authentik OAuth2 provider secret — **optional**, see below |
+
+Generate the Forgejo secrets once and store them, rather than letting Forgejo
+auto-generate on first boot — that keeps a from-scratch redeploy (fresh data
+volume) reproducible instead of silently rotating tokens:
+
+```sh
+docker run --rm codeberg.org/forgejo/forgejo:10 forgejo generate secret
+```
+
+Run that four times (once per secret) and write them all together with the
+DB password:
+
+```sh
+vault kv put kv/homelab/forgejo \
+  DB_PASSWORD='<generate-a-strong-password>' \
+  SECRET_KEY='<output-of-generate-secret>' \
+  INTERNAL_TOKEN='<output-of-generate-secret>' \
+  JWT_SECRET='<output-of-generate-secret>' \
+  LFS_JWT_SECRET='<output-of-generate-secret>'
+```
+
+`LFS_JWT_SECRET` is only consumed by the Proxmox native install, which renders
+`app.ini` in full and sets `INSTALL_LOCK` — nothing is left for Forgejo to
+generate on first boot, and a value it invented for itself would be overwritten
+on the next deploy anyway. The Unraid Compose stack doesn't reference it; the
+key being present just means one extra unused line in the rendered `.env`. If
+you only run the Unraid stack today, add it before the Proxmox cutover —
+`src/forgejo/ansible/proxmox/install.yml` asserts on it rather than failing
+three tasks later with an undefined-variable error:
+
+```sh
+vault kv patch kv/homelab/forgejo LFS_JWT_SECRET='<output-of-generate-secret>'
+```
+
+### `OIDC_CLIENT_SECRET`
+
+Only needed if Forgejo signs users in through Authentik (step 8 of
+`docs/forgejo-proxmox.md`). Unlike every other key on this path it is **not
+read by Ansible** — nothing templates it, and `install.yml` does not assert on
+it, so its absence is not an error. Forgejo keeps authentication sources in
+its database, not in `app.ini`, so the value is consumed once by a
+`forgejo admin auth add-oauth` run and lives in the database from then on.
+
+It is kept here anyway, rather than only in Authentik, for the same reason as
+the rest: a rebuild that restores from a `pg_dump` gets the auth source back
+with the dump, but a rebuild from *nothing* has to re-run that command, and
+this is where it looks for the value. Generated by Authentik when you create
+the provider, not by you:
+
+```sh
+vault kv patch kv/homelab/forgejo OIDC_CLIENT_SECRET='<from-the-authentik-provider>'
+```
+
+The matching client ID is not a secret and is not stored here — it is an
+argument to that one command, recorded in the runbook step alongside it.
+
+## `homelab/arr`
+
+| Key | Purpose |
+|---|---|
+| `SECRET_KEY_BASE` | Signs sessions and cookies |
+| `ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY` | Encrypts every credential Shelfarr stores — indexer API keys, download client passwords, OIDC secrets |
+| `ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY` | As above, for values Shelfarr needs to query on |
+| `ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT` | As above |
+| `SHELFARR_SETTING_OIDC_CLIENT_SECRET` | Client secret of the Authentik provider for Shelfarr |
+| `PROWLARR_API_KEY` | Prowlarr's API key, for anything integrating with it |
+
+One path for the whole stack, not one per container — `compose_stack` fetches
+`vault_path` once and renders every key into a single `.env` that both
+services read, so Prowlarr's key belongs here rather than under a path of its
+own.
+
+Shelfarr's entrypoint generates the first four on first run if they're unset, into
+`/rails/storage/.secret_key_base` and `/rails/storage/.encryption_keys` — i.e.
+into the appdata volume. Set them explicitly for the same reason Forgejo's
+three secrets are fixed above: a from-scratch redeploy on a fresh volume would
+otherwise generate new ones, and everything encrypted with the old values
+becomes unreadable.
+
+```sh
+vault kv put kv/homelab/arr \
+  SECRET_KEY_BASE="$(openssl rand -hex 64)" \
+  ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY="$(openssl rand -base64 32)" \
+  ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY="$(openssl rand -base64 32)" \
+  ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT="$(openssl rand -base64 32)"
+```
+
+Set them **before the first start**, and set the three encryption keys as a
+group — the entrypoint only falls back to its generated file when the primary
+key is absent, so a partial set silently mixes provided and generated values.
+Changing them after Shelfarr has stored anything means re-entering every
+credential through Admin → Settings.
+
+> **Do not add `RAILS_MASTER_KEY` here.** Despite the name it is not a
+> generatable secret: it decrypts the `config/credentials.yml.enc` compiled
+> into the upstream image, so only upstream's own key works. Supplying a value
+> of your own aborts startup with
+> `ActiveSupport::MessageEncryptor::InvalidMessage` during `db:prepare`.
+
+The OIDC client secret is the one key here that isn't generated — copy it off
+the Authentik provider:
+
+```sh
+vault kv patch kv/homelab/arr \
+  SHELFARR_SETTING_OIDC_CLIENT_SECRET='<authentik-client-secret>'
+```
+
+`SHELFARR_SETTING_` is how upstream maps environment variables onto the
+Admin → Settings store; the non-secret half of the OIDC config lives in
+`src/arr/common/vars.yml`. Both halves have to be named in
+`docker-compose.yml`'s `environment:` — `compose_stack` renders every Vault
+key and every `env_defaults` entry into `.env`, but Compose reads `.env` only
+to interpolate `${...}` in the compose file. A var that nothing references
+never reaches the container, and Shelfarr just carries on with OIDC off.
+
+Prowlarr's API key is generated the same way as Shelfarr's secrets, and for
+the same reason — left unset, Prowlarr writes one of its own into
+`/config/config.xml` on first start, where the only way to find it is the UI:
+
+```sh
+vault kv patch kv/homelab/arr PROWLARR_API_KEY="$(openssl rand -hex 16)"
+```
+
+Prowlarr binds config-file settings to environment as `PROWLARR__` plus the
+section and key, so this reaches it as `PROWLARR__AUTH__APIKEY`. Releases
+before the auth section was split read the same value as `PROWLARR__APIKEY`;
+an unrecognised variable is ignored rather than fatal, so the symptom of the
+wrong form is a key in the UI that doesn't match Vault, not a container that
+won't start.
+
+## `homelab/k3s-homelab-utils`
+
+| Key | Purpose |
+|---|---|
+| `K3S_TOKEN` | Cluster join token — shared by the server and every agent |
+
+Not app config — this is what makes `playbooks/k3s.yml` reproducible.
+`k3s server` normally invents a token on first install and writes it to
+`/var/lib/rancher/k3s/server/node-token`; every agent then has to be told that
+value. Pinning it in Vault instead means a full rebuild (wipe both SD cards,
+reinstall) reproduces the same cluster identity, because both the control
+plane and every worker fetch the *same* value independently rather than one
+generating it and handing it to the other.
+
+Generate it once, before the first run of `playbooks/k3s.yml`:
+
+```sh
+vault kv put kv/homelab/k3s-homelab-utils \
+  K3S_TOKEN="$(openssl rand -hex 32)"
+```
+
+Changing this after the cluster is up means re-running the playbook against
+every node — the server re-issues its cert bundle around the new token and
+every agent needs to reconnect with it, which is disruptive enough that
+there's no automatic rotation path here, only a manual one.
+
+## `homelab/authentik`
+
+| Key | Purpose |
+|---|---|
+| `AUTHENTIK_SECRET_KEY` | Signs session cookies |
+| `AUTHENTIK_POSTGRESQL__PASSWORD` | Password for the `authentik` role on the cluster's shared Postgres |
+
+The first app on the k3s cluster, deployed by `roles/k3s_app` from the
+`k3s_apps:` list in `inventory/group_vars/k3s_cluster.yml`. Both keys are
+named for the environment variables Authentik itself reads, because
+`roles/k3s_app` passes Vault keys through to the app's Kubernetes Secret
+verbatim — so one value at `AUTHENTIK_POSTGRESQL__PASSWORD` serves both the
+database provisioning step and the running app, with nothing restated.
+
+```sh
+vault kv put kv/homelab/authentik \
+  AUTHENTIK_SECRET_KEY='<carried over from the Unraid deployment>' \
+  AUTHENTIK_POSTGRESQL__PASSWORD="$(openssl rand -base64 32)"
+```
+
+`AUTHENTIK_SECRET_KEY` is the exception to "generate a fresh secret" — it
+must be copied from the existing Unraid deployment, not invented. It signs
+session cookies, so a new value logs every user out at the moment DNS moves.
+See `docs/authentik-migration.md`, which is the only reason this path has a
+"copy the old value" step at all; a from-scratch install would generate both.
+
+## `homelab/k3s-cert-manager`
+
+| Key | Purpose |
+|---|---|
+| `CLOUDFLARE_API_TOKEN` | DNS provider API token for cert-manager's ACME DNS-01 solver |
+
+Consumed by `roles/k3s_cert_manager`, which renders every key at this path
+into a Secret in the `cert-manager` namespace — so the key name above is a
+convention, not a requirement: store whatever your DNS provider needs and
+reference that name from `k3s_cert_manager_solver` in
+`inventory/group_vars/k3s_cluster.yml`.
+
+```sh
+vault kv put kv/homelab/k3s-cert-manager \
+  CLOUDFLARE_API_TOKEN='<token with Zone:DNS:Edit on the zone>'
+```
+
+DNS-01 rather than HTTP-01 because HTTP-01 needs Let's Encrypt to reach the
+cluster from the internet on port 80, and this LAN deliberately isn't
+reachable. Scope the token to the single zone it manages — cert-manager only
+ever needs to write `_acme-challenge` TXT records.
+
+## `homelab/ci/ssh`
+
+Not consumed by `compose_stack` — this is the SSH private key Ansible itself
+connects to the Unraid hosts with. It lives in Vault rather than as a
+Forgejo Actions secret so there's one place secrets come from, not two; see
+README.md "SSH access" for generating the key pair and installing the public
+half on each host.
+
+| Key | Purpose |
+|---|---|
+| `PRIVATE_KEY` | OpenSSH private key `ansible_user` (root) authenticates with |
+
+```sh
+# `@path` reads the value from a file, which handles the embedded newlines
+# in a private key correctly — don't try to inline it as a normal KEY=value.
+# Use $HOME, not `~`: this is a plain command argument, not a shell
+# assignment, so bash won't tilde-expand a `~` sitting after the `@` — it'd
+# get passed through literally and vault would look for a file named `~`.
+vault kv put kv/homelab/ci/ssh PRIVATE_KEY=@"$HOME/.ssh/unraid_ansible"
+```
+
+## `homelab/ci/ssh-k3s`
+
+The SSH private key Ansible connects to the 4 k3s Pis with. A separate key
+pair from `homelab/ci/ssh` above, deliberately — the two host groups don't
+share a trust boundary, so a leaked key for one shouldn't also be a working
+key for the other. Same shape as the Unraid key otherwise:
+
+| Key | Purpose |
+|---|---|
+| `PRIVATE_KEY` | OpenSSH private key the `ansible` user (per `inventory/hosts.yml` → `k3s_cluster`) authenticates with |
+
+```sh
+ssh-keygen -t ed25519 -f ~/.ssh/k3s_ansible -C "ansible@homelab-utils" -N ""
+```
+
+Create the `ansible` user on each of the 4 Pis with NOPASSWD sudo and install
+the public half as its `authorized_keys` — see README.md "SSH access". Then:
+
+```sh
+vault kv put kv/homelab/ci/ssh-k3s PRIVATE_KEY=@"$HOME/.ssh/k3s_ansible"
+```
+
+Unlike the Unraid key above, nothing needs loading into `ssh-agent` to use
+this: `playbooks/k3s_ssh_key.yml` reads this path and writes the key to
+`local/k3s/homelab-utils.key` on the controller, and both `playbooks/k3s.yml`
+and `playbooks/k3s_maintenance.yml` import it as their first play. A manual
+run needs only `VAULT_ADDR`/`VAULT_TOKEN` in the environment (the "Login"
+section above — no AppRole involvement yet, both k3s playbooks being
+manual-only for now). See README.md "SSH access" for the fallback flags if
+Vault is unreachable.
+
+## `homelab/ci/proxmox`
+
+Proxmox VE API token, used by two things: the `community.proxmox` dynamic
+inventory plugin (`build/config/ansible/inventory/proxmox.yml`), and the
+Terraform provider that creates LXCs. A token rather than a password — same
+reasoning as the SSH key, a password means an interactive prompt or a
+plaintext secret on disk.
+
+Create the user and its token on either node (ACLs are cluster-wide), then
+grant it a role. This is more involved than it looks, and the failure modes
+give errors that don't obviously point at permissions:
+
+```sh
+pveum user add ansible@pam
+pveum user token add ansible@pam terraform --privsep 0
+
+pveum role add TerraformProv -privs "Datastore.Allocate \
+  Datastore.AllocateSpace Datastore.AllocateTemplate Datastore.Audit \
+  Pool.Allocate SDN.Use Sys.Audit Sys.Console Sys.Modify \
+  VM.Allocate VM.Audit VM.Clone VM.Console VM.Migrate VM.PowerMgmt \
+  VM.Replicate \
+  VM.Config.CPU VM.Config.Disk VM.Config.HWType VM.Config.Memory \
+  VM.Config.Network VM.Config.Options"
+
+pveum acl modify / --users 'ansible@pam' --roles TerraformProv
+```
+
+Three things that otherwise cost an afternoon:
+
+- **`--privsep 0` is load-bearing.** A token created with privilege separation
+  on (the UI default) carries its *own* ACL and ignores what the user was
+  granted, so `pveum acl modify --users` looks like it did nothing. Either
+  turn it off as above, or grant the token as well with
+  `--tokens 'ansible@pam!terraform'`. With a purpose-made user rather than
+  `root@pam` there is nothing gained by maintaining both layers — least
+  privilege is already enforced on the user.
+- **Privilege names are version-sensitive.** `VM.Monitor` appears in most
+  guides for this and is rejected outright by current PVE (it is a QEMU
+  monitor privilege, meaningless for LXC). `SDN.Use` is required from PVE 8.2
+  on, where attaching a container to `vmbr0` is gated on
+  `/sdn/zones/localnetwork/vmbr0`. Check against your own node rather than
+  against any list, this one included: `pvesh get /access/roles/Administrator`
+  prints the complete valid set.
+- **`VM.Replicate` is there for the shared Postgres**, whose module creates a
+  `pvesr` job (`replication_target_node` — see
+  `src/shared/postgres/terraform/`). A module without replication won't need
+  it.
+
+Verify the grant landed before reaching for Terraform — this answers the
+token's effective permissions, not the user's:
+
+```sh
+pveum user permissions ansible@pam --token terraform
+```
+
+Every `terraform apply` failure here is an HTTP 403 naming the exact missing
+privilege, so the recovery loop is `pveum role modify TerraformProv --privs
+"<name>" --append` and re-run. Note `--append`: without it, `--privs` replaces
+the whole list rather than adding to it.
+
+Then:
+
+| Key | Purpose |
+|---|---|
+| `PROXMOX_URL` | API endpoint, e.g. `https://turtle-proxmox-01.home.turtlesystems.co.uk:8006/` |
+| `PROXMOX_USER` | Token owner, e.g. `ansible@pam` |
+| `PROXMOX_TOKEN_ID` | Token ID (the part after `!`) |
+| `PROXMOX_TOKEN_SECRET` | Token secret — shown once at creation |
+
+```sh
+vault kv put kv/homelab/ci/proxmox \
+  PROXMOX_URL='https://turtle-proxmox-01.home.turtlesystems.co.uk:8006/' \
+  PROXMOX_USER='ansible@pam' \
+  PROXMOX_TOKEN_ID='terraform' \
+  PROXMOX_TOKEN_SECRET='<shown-once-at-creation>'
+```
+
+Ansible's inventory plugin reads these from the environment, so export them
+before running anything that touches `proxmox_guests`. Terraform wants the
+token in one combined string instead:
+
+```sh
+export TF_VAR_proxmox_api_token="${PROXMOX_USER}!${PROXMOX_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
+```
+
+## `homelab/ci/terraform`
+
+Connection string for the Postgres database Terraform keeps its state in
+(`backend "pg"` — see `build/config/terraform/README.md`). Separate from the
+superuser credentials above: Terraform gets its own role, scoped to its own
+database.
+
+| Key | Purpose |
+|---|---|
+| `PG_CONN_STR` | `postgres://terraform:<password>@192.168.50.81:5432/terraform_state` |
+
+The host is the **CloudNativePG cluster on k3s**
+(`k3s_postgres_loadbalancer_ip`), not either shared Postgres this repo
+deploys — Terraform cannot keep its state in a database it provisions itself.
+Create the role and database once, against that cluster, then store the
+string:
+
+```sh
+vault kv put kv/homelab/ci/terraform \
+  PG_CONN_STR='postgres://terraform:<generate-a-strong-password>@192.168.50.81:5432/terraform_state'
+```
+
+## Verifying what was written
+
+```sh
+vault kv get kv/homelab/shared/postgres
+vault kv get kv/homelab/forgejo
+```
+
+`vault kv put` (used above) replaces the whole secret at that path — to add
+or change a single key without touching the others, use `vault kv patch`
+instead:
+
+```sh
+vault kv patch kv/homelab/forgejo DB_PASSWORD='<new-password>'
+```
+
+## Adding a new stack's secrets
+
+```sh
+vault kv put kv/homelab/<stack-name> KEY1=value1 KEY2=value2
+```
+
+Then reference the same path as `vault_path` in the stack's entry under
+`build/config/ansible/inventory/host_vars/<host>.yml`.
+
+## AppRole for CI (Forgejo Actions)
+
+The bootstrap commands above use your own human token. The Forgejo Actions
+runner (`.forgejo/workflows/deploy.yml`) authenticates as an AppRole instead,
+scoped to read-only on the `kv/homelab/*` paths. One-time setup:
+
+```sh
+# 1. Policy: read-only on every stack's secrets
+vault policy write unraid-deploy - <<'EOF'
+path "kv/data/homelab/*" {
+  capabilities = ["read"]
+}
+EOF
+
+# 2. Enable AppRole auth (skip if already enabled — check `vault auth list`)
+vault auth enable approle
+
+# 3. Create the role, bound to that policy
+vault write auth/approle/role/unraid-deploy \
+  token_policies="unraid-deploy" \
+  token_ttl=15m \
+  token_max_ttl=1h
+
+# 4. Fetch the role ID (stable, not secret on its own)
+vault read auth/approle/role/unraid-deploy/role-id
+
+# 5. Generate a secret ID (treat as a secret — shown once)
+vault write -f auth/approle/role/unraid-deploy/secret-id
+```
+
+Store the `role_id` and `secret_id` from steps 4–5 as `VAULT_ROLE_ID` and
+`VAULT_SECRET_ID` in the Forgejo repo's Actions secrets (alongside
+`VAULT_ADDR`) — that's what `deploy.yml` and
+`build/config/ansible/inventory/group_vars/all.yml`
+(`vault_auth_method: approle`) expect.
+
+These never live in a file in this repo, in either mode:
+
+- **CI:** Forgejo repo → Settings → Actions → Secrets. `deploy.yml` reads
+  them from there via `${{ secrets.VAULT_ROLE_ID }}` /
+  `${{ secrets.VAULT_SECRET_ID }}` and exports them as job env vars.
+- **Manual runs** (bootstrap, or any ad hoc `ansible-playbook` invocation):
+  export them as shell env vars first — `group_vars/all.yml` picks them up
+  via `lookup('env', ...)`:
+
+  ```sh
+  export VAULT_AUTH_METHOD=approle
+  export VAULT_ROLE_ID='<from step 4>'
+  export VAULT_SECRET_ID='<from step 5>'
+  ```
+
+  (Bootstrap can also just use your own human token instead — leave
+  `VAULT_AUTH_METHOD` unset, it defaults to `token`, and `vault login` from
+  the "Login" section above is enough.)
+
+Secret IDs can be regenerated (step 5) and old ones revoked without touching
+the role itself if one ever leaks:
+
+```sh
+vault write auth/approle/role/unraid-deploy/secret-id-accessor/destroy \
+  secret_id_accessor='<accessor-from-step-5-output>'
+```

src/arr/ansible/proxmox/README.md

@@ -0,0 +1,22 @@
+# ansible/proxmox/ — arr
+
+Not built. Shelfarr runs as a Compose stack on nas1 (`../unraid/`), and there
+is no reason to move it — this folder exists because every app keeps the same
+shape whether or not both platforms are in use.
+
+A native install would be a poor fit here in a way most apps aren't. Shelfarr
+is a Rails app shipping two containers' worth of runtime (Ruby, a bundled
+Solid Queue worker, and a .NET companion for the Libation sidecar), with no
+released binary or package — upstream distributes container images only. Where
+`forgejo` drops a single Go binary and a systemd unit into an LXC,
+reproducing this natively means building the Rails app from source on the
+guest.
+
+If it ever needs to move, the realistic option is Docker inside a privileged
+LXC rather than the `lxc_app` role, which would be a new deployment primitive
+for this repo — decide that deliberately rather than by extending the role.
+
+What wouldn't change: `../../common/vars.yml` already holds the portable
+config (version, published port), and the Vault path stays `homelab/arr`.
+Only paths and PUID/PGID are Unraid-specific, and those live in
+`../unraid/vars.yml`.

src/arr/ansible/unraid/.env.example

@@ -0,0 +1,101 @@
+# Reference only — real values are rendered by Ansible from vars.yml (non-
+# secret) and Vault path homelab/arr (secret). Do not fill this in and
+# deploy it directly.
+
+# Non-secret (see ../../common/vars.yml)
+SHELFARR_VERSION=2026.08.05.1
+SHELFARR_HTTP_PORT=5056
+
+# Non-secret (see vars.yml) — check the media paths against your own shares
+SHELFARR_DATA_PATH=/mnt/user/appdata/arr/data
+SHELFARR_AUDIOBOOKS_PATH=/mnt/user/Media/AudioBooks
+SHELFARR_EBOOKS_PATH=/mnt/user/Media/Books
+SHELFARR_DOWNLOADS_PATH=/mnt/user/downloads
+PUID=99
+PGID=100
+CHOWN_ON_START=auto
+
+# Docker network to join, and it must already exist on the target host —
+# Compose declares it `external`, so it will not create it. Defined in
+# Unraid's Docker settings, not by any stack in this repo.
+SHELFARR_NETWORK=caddy-net
+
+# Non-secret, supplied by the compose_stack role rather than either vars.yml —
+# the path it copies ansible/unraid/icon.png to on the host, for the
+# net.unraid.docker.icon label. Empty when the stack ships no icon.png, which
+# just leaves Unraid's question-mark placeholder. Override it in vars.yml to
+# point at a hosted PNG instead.
+STACK_ICON=/mnt/user/appdata/icons/arr.png
+
+# Secret — Vault key: SECRET_KEY_BASE
+# Signs sessions and cookies. Generate with:
+#   openssl rand -hex 64
+SECRET_KEY_BASE=
+
+# Secret — Vault keys: ACTIVE_RECORD_ENCRYPTION_*
+# Encrypt every credential Shelfarr stores (indexer API keys, download client
+# passwords). Generate each with:
+#   openssl rand -base64 32
+# Set these before the first start, and set all three together. Changing them
+# later makes everything already encrypted with the old values unreadable.
+ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY=
+ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY=
+ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT=
+
+# Non-secret (see ../../common/vars.yml) — OIDC against Authentik.
+# Upstream maps environment onto the settings store by SHELFARR_SETTING_ plus
+# the uppercased setting key. There is no redirect-URI variable: the callback
+# is fixed at <shelfarr-url>/auth/oidc/callback and is registered on the
+# Authentik provider.
+SHELFARR_SETTING_OIDC_ENABLED=true
+SHELFARR_SETTING_OIDC_PROVIDER_NAME=Authentik
+SHELFARR_SETTING_OIDC_ISSUER=https://auth.turtlesystems.uk/application/o/shelfarr/
+SHELFARR_SETTING_OIDC_CLIENT_ID=
+SHELFARR_SETTING_OIDC_SCOPES=openid profile email
+SHELFARR_SETTING_OIDC_AUTO_CREATE_USERS=true
+SHELFARR_SETTING_OIDC_DEFAULT_ROLE=user
+SHELFARR_SETTING_OIDC_LINK_EXISTING_USERS=true
+SHELFARR_SETTING_OIDC_AUTO_REDIRECT=false
+
+# Secret — Vault key: SHELFARR_SETTING_OIDC_CLIENT_SECRET
+# The client secret of the Authentik provider for Shelfarr.
+SHELFARR_SETTING_OIDC_CLIENT_SECRET=
+
+# Do NOT set RAILS_MASTER_KEY. It decrypts the credentials file built into the
+# image, so only upstream's key works — a self-generated one aborts startup
+# with ActiveSupport::MessageEncryptor::InvalidMessage.
+
+# Only if the Libation companion is enabled — see docker-compose.yml
+# LIBATION_CONFIG_PATH=/mnt/user/appdata/arr/libation/config
+# LIBATION_BOOKS_PATH=/mnt/user/appdata/arr/libation/books
+# LIBATION_CONTROL_PATH=/mnt/user/appdata/arr/libation/control
+
+# Only if running behind a reverse proxy at a sub-path, e.g. /arr
+# RAILS_RELATIVE_URL_ROOT=/
+
+# --- Prowlarr ---------------------------------------------------------------
+
+# Non-secret (see ../../common/vars.yml). LinuxServer tags are the upstream
+# version plus their build suffix (1.37.0.5076-ls117); pin to a full one read
+# off the registry rather than leaving this at latest.
+PROWLARR_VERSION=latest
+PROWLARR_HTTP_PORT=9696
+
+# Non-secret, supplied by the compose_stack role — the path it copies
+# icon-prowlarr.png to, for the prowlarr container's net.unraid.docker.icon
+# label. Per-service, so it's separate from STACK_ICON above, which is the
+# stack's own icon.png and labels shelfarr.
+STACK_ICON_PROWLARR=/mnt/user/appdata/icons/arr-prowlarr.png
+
+# Non-secret (see vars.yml)
+PROWLARR_CONFIG_PATH=/mnt/user/appdata/arr/prowlarr
+TZ=Europe/London
+
+# Secret — Vault key: PROWLARR_API_KEY
+# Bound to Prowlarr's config.xml via PROWLARR__AUTH__APIKEY, so the key is
+# known before first boot and anything integrating with Prowlarr can be
+# configured from the same Vault path. Generate with:
+#   openssl rand -hex 16
+# Leave it out and Prowlarr generates its own on first start — then the value
+# only exists in the UI and in /config/config.xml.
+PROWLARR_API_KEY=

src/arr/ansible/unraid/docker-compose.yml

@@ -0,0 +1,252 @@
+services:
+  shelfarr:
+    image: ghcr.io/pedro-revez-silva/shelfarr:${SHELFARR_VERSION}
+    container_name: arr
+    restart: unless-stopped
+    # Unraid's Docker page normally reads icon, WebUI link and console shell
+    # from the dockerMan template that created the container. A Compose stack
+    # has no template, so 6.10+ falls back to these labels; without them the
+    # container renders as a question mark with no WebUI or Console entry.
+    #
+    # STACK_ICON comes from the compose_stack role (a path under
+    # /mnt/user/appdata/icons, or a URL if vars.yml overrides it). `[IP]` is
+    # substituted by the webgui with the host's address; the port has to be
+    # the published one, hence SHELFARR_HTTP_PORT rather than the container's
+    # 80. If you drop the `ports:` mapping below and reach Shelfarr only
+    # through Caddy, replace the whole value with the proxied URL — the
+    # webgui has no way to know about the reverse proxy.
+    labels:
+      net.unraid.docker.icon: ${STACK_ICON}
+      net.unraid.docker.webui: "http://[IP]:${SHELFARR_HTTP_PORT}/"
+      net.unraid.docker.shell: bash
+    networks:
+      - proxy
+    environment:
+      # Runs the Solid Queue background worker inside the Puma process rather
+      # than as a second container. Upstream's own compose does this; Shelfarr
+      # is a single-user-scale app and a separate worker buys nothing here.
+      SOLID_QUEUE_IN_PUMA: "1"
+      PUID: ${PUID}
+      PGID: ${PGID}
+      CHOWN_ON_START: ${CHOWN_ON_START}
+      # Deliberately NOT RAILS_MASTER_KEY. That variable decrypts the
+      # config/credentials.yml.enc baked into the image at build time, so only
+      # upstream's own key works — setting it to a generated value aborts boot
+      # with ActiveSupport::MessageEncryptor::InvalidMessage.
+      #
+      # These four are the ones the entrypoint actually treats as overridable,
+      # and pinning them is what stops a fresh appdata volume from orphaning
+      # stored credentials (same reasoning as forgejo's SECRET_KEY). Left
+      # unset, the entrypoint generates them into /rails/storage/.secret_key_base
+      # and /rails/storage/.encryption_keys — fine until that directory is lost.
+      SECRET_KEY_BASE: ${SECRET_KEY_BASE}
+      # Encrypt every credential Shelfarr stores (indexer API keys, download
+      # client passwords). All three must be set together; the entrypoint only
+      # falls back to its generated file when the primary key is absent.
+      ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY: ${ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY}
+      ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY: ${ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY}
+      ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT: ${ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT}
+      # --- OIDC (Authentik) ------------------------------------------------
+      #
+      # Upstream maps environment onto the Admin → Settings store by
+      # `SHELFARR_SETTING_` + the uppercased setting key, so these ten are the
+      # whole of the SSO config; there is no redirect-URI variable, the
+      # callback path is fixed at /auth/oidc/callback and is registered on the
+      # Authentik side.
+      #
+      # Every one has to be named here. The .env `compose_stack` renders holds
+      # them already, but Compose reads .env only to interpolate `${...}` in
+      # this file — it does not inject it into the container. An unreferenced
+      # var reaches the host's Compose process and stops there, which is why
+      # OIDC stayed off with the values apparently set.
+      SHELFARR_SETTING_OIDC_ENABLED: ${SHELFARR_SETTING_OIDC_ENABLED}
+      SHELFARR_SETTING_OIDC_PROVIDER_NAME: ${SHELFARR_SETTING_OIDC_PROVIDER_NAME}
+      SHELFARR_SETTING_OIDC_ISSUER: ${SHELFARR_SETTING_OIDC_ISSUER}
+      SHELFARR_SETTING_OIDC_CLIENT_ID: ${SHELFARR_SETTING_OIDC_CLIENT_ID}
+      # Secret — Vault key of the same name, not either vars.yml.
+      SHELFARR_SETTING_OIDC_CLIENT_SECRET: ${SHELFARR_SETTING_OIDC_CLIENT_SECRET}
+      SHELFARR_SETTING_OIDC_SCOPES: ${SHELFARR_SETTING_OIDC_SCOPES}
+      SHELFARR_SETTING_OIDC_AUTO_CREATE_USERS: ${SHELFARR_SETTING_OIDC_AUTO_CREATE_USERS}
+      SHELFARR_SETTING_OIDC_DEFAULT_ROLE: ${SHELFARR_SETTING_OIDC_DEFAULT_ROLE}
+      SHELFARR_SETTING_OIDC_LINK_EXISTING_USERS: ${SHELFARR_SETTING_OIDC_LINK_EXISTING_USERS}
+      SHELFARR_SETTING_OIDC_AUTO_REDIRECT: ${SHELFARR_SETTING_OIDC_AUTO_REDIRECT}
+    volumes:
+      - ${SHELFARR_DATA_PATH}:/rails/storage
+      - ${SHELFARR_AUDIOBOOKS_PATH}:/audiobooks
+      - ${SHELFARR_EBOOKS_PATH}:/ebooks
+      - ${SHELFARR_DOWNLOADS_PATH}:/downloads
+    ports:
+      - "${SHELFARR_HTTP_PORT}:80"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:80/up"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+
+  # Indexer manager for the rest of the stack. Shelfarr doesn't talk to
+  # trackers itself — it asks Prowlarr, which owns the indexer definitions and
+  # their credentials. Both are on `proxy`, so Shelfarr reaches it as
+  # `http://prowlarr:9696` (Admin → Settings → Indexers) rather than via nas1's
+  # LAN IP and the published port.
+  prowlarr:
+    image: lscr.io/linuxserver/prowlarr:${PROWLARR_VERSION}
+    container_name: prowlarr
+    restart: unless-stopped
+    # Same reasoning as the labels on `shelfarr` above — a Compose stack has no
+    # dockerMan template, so without these the container is a question mark
+    # with no WebUI entry. STACK_ICON_PROWLARR rather than STACK_ICON: labels
+    # are per-service, and ${STACK_ICON} is the stack's own icon.png, which is
+    # Shelfarr's. This one comes from the committed icon-prowlarr.png in this
+    # directory — see README.md → "Icons on the Unraid Docker page".
+    labels:
+      net.unraid.docker.icon: ${STACK_ICON_PROWLARR}
+      net.unraid.docker.webui: "http://[IP]:${PROWLARR_HTTP_PORT}/"
+      # Alpine-based, so `sh` — same as forgejo, not `bash` like the
+      # Debian-based shelfarr above.
+      net.unraid.docker.shell: sh
+    networks:
+      - proxy
+    environment:
+      PUID: ${PUID}
+      PGID: ${PGID}
+      # LinuxServer's init sets the container clock from this. It matters more
+      # here than for Shelfarr: Prowlarr timestamps search history and applies
+      # per-indexer rate limits, and both read wrong at UTC.
+      TZ: ${TZ}
+      # Prowlarr binds its config file to environment via `PROWLARR__` plus the
+      # config section and key, double-underscore separated. Pinning the API
+      # key means Shelfarr's indexer config can be written from the same Vault
+      # path instead of being copied by hand out of the UI after first boot —
+      # the same argument as SECRET_KEY_BASE above, applied to a value that is
+      # otherwise generated into /config/config.xml on first start.
+      #
+      # Older releases read this as PROWLARR__APIKEY (no section). If the key
+      # in the UI doesn't match Vault after a deploy, that's which form this
+      # image wants; an unrecognised variable is ignored silently rather than
+      # failing the container, so the symptom is a generated key, not a crash.
+      PROWLARR__AUTH__APIKEY: ${PROWLARR_API_KEY}
+    volumes:
+      # config.xml, the indexer definitions, and Prowlarr's own SQLite
+      # database. This directory *is* the application state — same standing as
+      # SHELFARR_DATA_PATH, back it up the same way.
+      - ${PROWLARR_CONFIG_PATH}:/config
+    ports:
+      - "${PROWLARR_HTTP_PORT}:9696"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9696/ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+
+networks:
+  # `external: true` means "attach to a network that already exists; don't
+  # create it, don't delete it on `down`". Without it Compose would create a
+  # *new* network named `arr_<key>` — the deploy would still succeed and
+  # Caddy would silently fail to resolve `arr`. `name:` is what stops the
+  # project-name prefix being applied.
+  #
+  # The local alias is a fixed `proxy` so the compose file stays static; which
+  # real network it maps to is a per-host decision in vars.yml
+  # (SHELFARR_NETWORK). Same shape as forgejo's `shared` → `unraid_shared`.
+  #
+  # The alias MUST be a literal, not `${SHELFARR_NETWORK}:`. Compose
+  # interpolates values, never mapping keys, so a variable used as the key
+  # stays the literal string `${SHELFARR_NETWORK}` while the service's
+  # `networks:` entry (a list value) expands to `caddy-net` — and the two no
+  # longer match:
+  #     service "arr" refers to undefined network caddy-net
+  # `name:` below is a value, so it interpolates normally. That asymmetry is
+  # the whole reason for the alias indirection.
+  #
+  # caddy-net is defined in Unraid's Docker settings on nas1 rather than by a
+  # stack in this repo, so nothing here has to deploy before Shelfarr does.
+  # But it must exist on the target host — an absent external network fails
+  # the deploy outright, which is why `unraid_shared` (created by the shared
+  # Postgres stack, and that runs on nas2) is deliberately not used here.
+  # Shelfarr is SQLite-only and needs no database anyway.
+  proxy:
+    name: ${SHELFARR_NETWORK}
+    external: true
+
+# Now that Caddy and Shelfarr share a network, Caddy should proxy to
+# `arr:80` — the container name on a user-defined network — rather than
+# to nas1's LAN IP and the published port. The `ports:` mapping above is kept
+# only for direct access that bypasses the proxy; drop it if you don't want
+# 5056 reachable on the LAN.
+#
+# Reaching the download client and indexers stays a runtime concern, not a
+# compose one: they're configured in Admin → Settings. Anything not on
+# caddy-net is addressed via nas1's LAN IP and its published port
+# (e.g. http://192.168.50.1:8080), which works whichever network it's on.
+
+# --- Import layout ---------------------------------------------------------
+#
+# The three media binds above follow upstream's documented layout, which works
+# regardless of how your shares are arranged. The cost: /downloads and
+# /audiobooks are separate mounts inside the container, and rename() across
+# mounts returns EXDEV even when both sit on the same Unraid filesystem — so
+# every import is a full copy rather than an instant move.
+#
+# If downloads and the library live under one share, replace the three binds
+# with a single one and set the paths in Admin → Settings to match:
+#
+#     - ${SHELFARR_MEDIA_PATH}:/media       # e.g. /mnt/user/media
+#
+#   Admin → Settings:  /media/downloads, /media/audiobooks, /media/ebooks
+#
+# One mount means one filesystem, so imports become atomic moves and hardlinks
+# work. Worth doing if your download client is already writing under the same
+# share.
+
+# --- Libation companion (Audible backup, beta) -----------------------------
+#
+# Left out for now; purely additive. To enable it: uncomment the service and
+# the `volumes:` block below, add these three to the main service's
+# environment, and add the two mounts to its `volumes:`, then redeploy. The
+# appdata bind survives the container recreate, so nothing is lost.
+#
+#   environment:
+#     SHELFARR_LIBATION_URL: http://shelfarr-libation:8080
+#     SHELFARR_LIBATION_TOKEN_FILE: /run/shelfarr-libation/token
+#     SHELFARR_LIBATION_IMPORT_ROOT: /imports/libation
+#   volumes:
+#     - ${LIBATION_BOOKS_PATH}:/imports/libation:ro
+#     - ${LIBATION_CONTROL_PATH}:/run/shelfarr-libation:ro
+#
+# Note the two services must then share a user-defined network so the main
+# container can resolve `shelfarr-libation` by name — the default bridge has no
+# DNS between containers.
+#
+#   shelfarr-libation:
+#     image: ghcr.io/pedro-revez-silva/shelfarr-libation:${SHELFARR_VERSION}
+#     container_name: shelfarr-libation
+#     restart: unless-stopped
+#     expose:
+#       - "8080"
+#     environment:
+#       PUID: ${PUID}
+#       PGID: ${PGID}
+#       CHOWN_ON_START: ${CHOWN_ON_START}
+#       LIBATION_FILES_DIR: /config
+#       LIBATION_BOOKS_DIR: /data
+#       LIBATION_IN_PROGRESS_DIR: /config/in-progress
+#       COMPANION_STATE_DIR: /config/shelfarr-companion
+#       COMPANION_TOKEN_FILE: /control/token
+#       ASPNETCORE_URLS: http://0.0.0.0:8080
+#     volumes:
+#       - ${LIBATION_CONFIG_PATH}:/config
+#       - ${LIBATION_BOOKS_PATH}:/data
+#       - ${LIBATION_CONTROL_PATH}:/control
+#     healthcheck:
+#       test: ["CMD", "/companion/Shelfarr.Libation.Companion", "--healthcheck"]
+#       interval: 30s
+#       timeout: 10s
+#       retries: 3
+#       start_period: 40s
+#
+# Upstream uses named volumes for libation_config/books/control. Bind mounts
+# under /mnt/user/appdata/arr/ instead, to match every other stack here
+# and so the data is visible on the array rather than buried in Docker's
+# storage. Add the paths to ansible/unraid/vars.yml when enabling.

src/arr/ansible/unraid/vars.yml

@@ -0,0 +1,59 @@
+---
+# Unraid-specific overrides, merged over ../../common/vars.yml (these win).
+env_defaults:
+  # SQLite lives here — all four production databases (primary, cache, queue,
+  # cable) plus Active Storage. Shelfarr has no external database, so unlike
+  # forgejo this directory *is* the application state, and it is also where
+  # everything configured in Admin → Settings is kept. Back it up accordingly.
+  SHELFARR_DATA_PATH: /mnt/user/appdata/arr/data
+
+  # CHECK THESE AGAINST YOUR OWN SHARES before the first deploy — they are
+  # conventional Unraid layouts, not something read off nas1.
+  #
+  # `/mnt/user/...`, never `/mnt/cache/...` or `/mnt/disk1/...`: the latter
+  # bypass the user-share layer and strand files where the mover won't find
+  # them.
+  #
+  # See the note in docker-compose.yml about keeping downloads and the library
+  # on one mount — as three separate binds, imports are copies rather than
+  # instant moves.
+  SHELFARR_AUDIOBOOKS_PATH: /mnt/user/Media/Books/audio
+  SHELFARR_EBOOKS_PATH: /mnt/user/Media/Books/ebooks
+  SHELFARR_DOWNLOADS_PATH: /mnt/user/downloads
+
+  # 99:100 is nobody:users — what Unraid owns share content as. Upstream
+  # defaults to 1000:1000, which is a normal Linux desktop user and wrong
+  # here; files written as 1000 are invisible to everything else on the box.
+  PUID: "99"
+  PGID: "100"
+
+  # `auto` chowns only when it has to. `always` would fight Unraid's own
+  # permissions handling on every restart; `never` risks a container that
+  # can't write to its own appdata after a share rebuild.
+  CHOWN_ON_START: auto
+
+  # Prowlarr's config.xml, indexer definitions and SQLite database. Sibling of
+  # SHELFARR_DATA_PATH under the same stack directory — one appdata folder per
+  # stack, one subdirectory per container.
+  PROWLARR_CONFIG_PATH: /mnt/user/appdata/arr/prowlarr
+
+  # Container clock. Prowlarr timestamps its search history and enforces
+  # per-indexer rate limits against it; Shelfarr takes its time from the Rails
+  # default and doesn't read this.
+  TZ: Europe/London
+
+  # The stack's Docker network — both containers join it, which is how
+  # Shelfarr resolves `prowlarr` by name.
+  SHELFARR_NETWORK: caddy-net
+
+  # --- Unraid Docker page presentation (net.unraid.docker.* labels) --------
+  #
+  # STACK_ICON is normally left to the compose_stack role, which points it at
+  # /mnt/user/appdata/icons/<stack>.png when this directory contains an
+  # icon.png. Uncomment to use a hosted icon instead of committing one:
+  #
+  #   STACK_ICON: https://example.org/arr.png
+  #
+  # PNG only — Unraid renders its placeholder for SVG and nothing at all for
+  # WebP, and a URL it can't reach when the Docker page renders fails the same
+  # way, which is the argument for committing the file.

src/arr/common/vars.yml

@@ -0,0 +1,64 @@
+---
+# Portable config — values that hold regardless of where Shelfarr runs.
+# Platform-specific values (filesystem paths, the PUID/PGID the container
+# writes as) live in ../ansible/<platform>/vars.yml and are merged over these
+# at deploy time.
+env_defaults:
+  # Pinned rather than `latest`. Upstream tags releases YYYY.MM.DD.N and the
+  # image tag carries no leading `v`, unlike the git tag — `v2026.08.05.1` is
+  # the release, `2026.08.05.1` is the image. One tag pins both the app image
+  # and the (currently unused) Libation companion.
+  SHELFARR_VERSION: "2026.08.09.1"
+
+  # Published port. 5056 is upstream's convention; the container listens on 80
+  # internally, which is left at its default rather than being made
+  # configurable — there is nothing else in the container to collide with.
+  SHELFARR_HTTP_PORT: "5056"
+
+  # SECRET_KEY_BASE and the three ACTIVE_RECORD_ENCRYPTION_* values are
+  # secrets — sourced from Vault (homelab/arr), never set here. The
+  # encryption trio is not just a session key: it encrypts every credential
+  # Shelfarr stores (indexer API keys, download client passwords), so losing
+  # it means re-entering all of them.
+  #
+  # RAILS_MASTER_KEY is deliberately absent. It belongs to the
+  # config/credentials.yml.enc compiled into the image, so any value you
+  # generate yourself fails to decrypt it and the container won't boot. See
+  # the note in ../ansible/unraid/docker-compose.yml.
+
+  SHELFARR_SETTING_OIDC_ENABLED: "true"
+  SHELFARR_SETTING_OIDC_PROVIDER_NAME: Authentik
+  SHELFARR_SETTING_OIDC_ISSUER: https://auth.turtlesystems.uk/application/o/shelfarr/
+  SHELFARR_SETTING_OIDC_CLIENT_ID: jf1IKMxxYjgZxZikvONdFfOTauFrFiC4AYmJEbmN
+  SHELFARR_SETTING_OIDC_SCOPES: openid profile email
+  SHELFARR_SETTING_OIDC_AUTO_CREATE_USERS: "true"
+  SHELFARR_SETTING_OIDC_DEFAULT_ROLE: user
+  SHELFARR_SETTING_OIDC_LINK_EXISTING_USERS: "true"
+  SHELFARR_SETTING_OIDC_AUTO_REDIRECT: "false"
+
+  # --- Prowlarr --------------------------------------------------------------
+  #
+  # LinuxServer's tags are the upstream version plus their own build suffix
+  # (`1.37.0.5076-ls117`), and the suffix can't be derived from the release
+  # number — so a pin has to be read off the registry rather than guessed.
+  # `latest` until then; check
+  # https://github.com/linuxserver/docker-prowlarr/pkgs/container/prowlarr and
+  # replace this with the full tag, which is what every other version in this
+  # repo does.
+  PROWLARR_VERSION: latest
+
+  # Published port. 9696 is upstream's default and what the container listens
+  # on internally; kept the same on the host so the WebUI label, the LAN URL
+  # and the docs all read alike.
+  PROWLARR_HTTP_PORT: "9696"
+
+  # PROWLARR_API_KEY is a secret — Vault (homelab/arr), never set here.
+  # Generate a 32-character hex string:  openssl rand -hex 16
+  # Prowlarr generates its own into /config/config.xml if this is left empty,
+  # which works fine but means the value is only discoverable from the UI.
+
+  # STACK_ICON_PROWLARR is supplied by the compose_stack role, from the
+  # committed ../ansible/unraid/icon-prowlarr.png — not set here. Uncomment to
+  # use a hosted icon instead, exactly as with STACK_ICON:
+  #
+  #   STACK_ICON_PROWLARR: https://example.org/prowlarr.png

src/arr/terraform/README.md

@@ -0,0 +1,11 @@
+# terraform/ — arr
+
+Not built, and not expected to be. Terraform's only job in this repo is
+creating Proxmox LXCs; Shelfarr runs as a Compose stack on nas1, where the
+host already exists.
+
+This folder is a placeholder so the app keeps the same
+`common/` + `ansible/` + `terraform/` shape as every other one. If Shelfarr
+ever moves to Proxmox it would need an LXC defined here and wired into
+`build/config/terraform/main.tf` — see `../ansible/proxmox/README.md` first
+for why that move is less straightforward than it looks.

src/authentik/ansible/kubernetes/helmchart.yaml.j2

@@ -0,0 +1,130 @@
+{#
+  Managed by Ansible (roles/k3s_app) — do not edit on the node.
+
+  A HelmChart CR for k3s's bundled helm-controller, the same mechanism every
+  cluster service here already uses (roles/k3s_metallb, k3s_monitoring,
+  k3s_postgres, k3s_cert_manager). The difference is only in who owns the
+  file: those roles each carry their own template, whereas this one belongs
+  to the app and roles/k3s_app renders whatever *.yaml.j2 it finds here.
+
+  This is the k3s analogue of src/<app>/ansible/unraid/docker-compose.yml.
+  It is not static the way the compose files are — Helm values have no
+  ${VAR} interpolation to defer to a rendered .env, so config is substituted
+  at render time from `app_config` (common/vars.yml + this platform's
+  vars.yml). What survives from the Unraid convention is the part that
+  matters: no secret is ever interpolated into this file. Secrets are
+  referenced by secretKeyRef against the Secret roles/k3s_app renders from
+  Vault, so this manifest stays 0644 on the node and safe to read.
+
+  No Redis anywhere below, deliberately: recent Authentik no longer requires
+  it, and the chart carries no Redis dependency (its only conditional
+  subchart is Bitnami Postgres, which is disabled here in favour of the
+  cluster's shared CNPG instance).
+-#}
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: authentik
+  namespace: kube-system
+spec:
+  chart: authentik
+  repo: https://charts.goauthentik.io
+  # Chart version and app version are the same string upstream, so
+  # AUTHENTIK_VERSION pins both. See common/vars.yml — this must match the
+  # version the database was dumped from at cutover time.
+  version: "{{ app_config.AUTHENTIK_VERSION }}"
+  targetNamespace: {{ app_config.K8S_NAMESPACE }}
+  createNamespace: true
+  valuesContent: |-
+    global:
+      # Explicit `env` entries rather than an `envFrom: [secretRef]`, for a
+      # precedence reason rather than a stylistic one: the chart builds its
+      # own env for the values under `authentik:` below, and container `env`
+      # deterministically wins over anything arriving via `envFrom`, whereas
+      # two envFrom sources resolve by list order the chart controls. This
+      # way the non-secret half stays readable as ordinary Helm values and
+      # the secret half is unambiguously authoritative.
+      env:
+        # Signs session cookies. Carried over from the Unraid deployment, not
+        # generated — a new value logs every user out the moment DNS flips.
+        - name: AUTHENTIK_SECRET_KEY
+          valueFrom:
+            secretKeyRef:
+              name: authentik-secrets
+              key: AUTHENTIK_SECRET_KEY
+        - name: AUTHENTIK_POSTGRESQL__PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: authentik-secrets
+              key: AUTHENTIK_POSTGRESQL__PASSWORD
+
+    authentik:
+      log_level: info
+      # Off: this is a homelab, and the default ships crash reports to
+      # Sentry.
+      error_reporting:
+        enabled: false
+      postgresql:
+        host: {{ app_config.DB_HOST }}
+        port: {{ app_config.DB_PORT }}
+        name: {{ app_config.DB_NAME }}
+        user: {{ app_config.DB_USER }}
+        # password comes from global.env above, never from here.
+
+    # The chart's bundled Bitnami Postgres. Off — this cluster has a shared
+    # CNPG instance (roles/k3s_postgres) and an app getting its own database
+    # server would defeat the point of having one.
+    postgresql:
+      enabled: false
+
+    server:
+      replicas: 1
+      resources:
+        requests:
+          cpu: 100m
+          memory: {{ app_config.SERVER_MEMORY_REQUEST }}
+        limits:
+          memory: {{ app_config.SERVER_MEMORY_LIMIT }}
+
+      ingress:
+        enabled: true
+        ingressClassName: {{ app_config.INGRESS_CLASS }}
+        annotations:
+          # cert-manager watches Ingresses for this annotation and creates
+          # the Certificate (and hence the Secret named under tls: below) on
+          # its own — no Certificate resource of ours to keep in sync.
+          cert-manager.io/cluster-issuer: {{ app_config.CERT_ISSUER }}
+        hosts:
+          - {{ app_config.AUTHENTIK_HOST }}
+        tls:
+          - secretName: authentik-tls
+            hosts:
+              - {{ app_config.AUTHENTIK_HOST }}
+
+      # /media — uploaded icons and flow backgrounds, the only Authentik
+      # state that isn't in Postgres. The same PVC is mounted by the worker
+      # below; see vars.yml for why one ReadWriteOnce volume across two pods
+      # is fine here and what it costs.
+      volumes:
+        - name: media
+          persistentVolumeClaim:
+            claimName: authentik-media
+      volumeMounts:
+        - name: media
+          mountPath: /media
+
+    worker:
+      replicas: 1
+      resources:
+        requests:
+          cpu: 100m
+          memory: {{ app_config.WORKER_MEMORY_REQUEST }}
+        limits:
+          memory: {{ app_config.WORKER_MEMORY_LIMIT }}
+      volumes:
+        - name: media
+          persistentVolumeClaim:
+            claimName: authentik-media
+      volumeMounts:
+        - name: media
+          mountPath: /media

src/authentik/ansible/kubernetes/media-pvc.yaml.j2

@@ -0,0 +1,33 @@
+{#
+  Managed by Ansible (roles/k3s_app) — do not edit on the node.
+
+  Authentik's /media volume, claimed here rather than left to the chart: the
+  chart has no key that creates a PVC, only `volumes:`/`volumeMounts:` that
+  reference one, so something has to declare it.
+
+  Owning it separately turns out to be the safer arrangement anyway. A PVC
+  created by the chart would carry Helm's ownership metadata and be a
+  candidate for deletion whenever the HelmChart CR is uninstalled or the
+  chart's templates change shape across an upgrade. This one is a plain
+  manifest whose lifecycle is its own, so `state: absent` on the app — which
+  deletes the manifests and lets k3s garbage-collect them — is the only thing
+  that touches it, and roles/k3s_app/tasks/remove.yml documents PVCs as
+  deliberately surviving a default teardown.
+
+  WaitForFirstConsumer on the local-path StorageClass means this stays
+  Pending until the first Authentik pod is scheduled; that is the provisioner
+  working as intended, not a failure, and it's what pins the volume — and
+  therefore both pods — to a single worker Pi.
+-#}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: authentik-media
+  namespace: {{ app_config.K8S_NAMESPACE }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: {{ app_config.MEDIA_STORAGE_CLASS }}
+  resources:
+    requests:
+      storage: {{ app_config.MEDIA_STORAGE_SIZE }}

src/authentik/ansible/kubernetes/vars.yml

@@ -0,0 +1,63 @@
+---
+# Kubernetes-specific overrides, merged over ../../common/vars.yml (these
+# win). Consumed by roles/k3s_app — the same "portable config + platform
+# overrides" split compose_stack and lxc_app apply, on the third platform.
+env_defaults:
+  K8S_NAMESPACE: authentik
+
+  # --- Database ---------------------------------------------------------
+  #
+  # CNPG's read-write Service, by in-cluster DNS. This is the *primary*, and
+  # it follows a failover — CNPG re-points the selector when it promotes the
+  # replica, so the name stays correct without Authentik being restarted.
+  #
+  # Deliberately not the LoadBalancer address (192.168.50.81): that exists so
+  # the Ansible controller can provision databases from off-cluster, and
+  # sending in-cluster traffic out to the LAN and back would put MetalLB's
+  # L2 speaker in the path of every query for no benefit. See
+  # roles/k3s_app/tasks/deploy.yml, which uses the LB address for exactly the
+  # one thing that needs it.
+  DB_HOST: shared-postgres-rw.shared-postgres.svc.cluster.local
+  DB_PORT: "5432"
+
+  # --- Ingress ----------------------------------------------------------
+  #
+  # k3s's bundled Traefik, already running on 192.168.50.80 (MetalLB's first
+  # pool address). Nothing here pins that IP — the Ingress attaches to the
+  # class and Traefik's own Service owns the address.
+  INGRESS_CLASS: traefik
+
+  # The ClusterIssuer roles/k3s_cert_manager creates. Must match
+  # k3s_cert_manager_issuer_name in inventory/group_vars/k3s_cluster.yml.
+  CERT_ISSUER: letsencrypt
+
+  # --- Storage ----------------------------------------------------------
+  #
+  # /media holds uploaded application icons and flow backgrounds — the things
+  # that make the login page look like yours rather than stock. Small, but
+  # the one piece of Authentik's state that isn't in Postgres, so it has to
+  # be copied across at cutover (docs/authentik-migration.md) or the migration
+  # is visibly incomplete.
+  #
+  # ReadWriteOnce on k3s's local-path provisioner, and both the server and
+  # worker pods mount it. That works without any affinity rules of our own:
+  # local-path PVs carry node affinity, so once the volume binds to whichever
+  # worker the first pod lands on, the scheduler is obliged to place the
+  # second pod on that same node — and RWO permits multiple pods per node.
+  # The cost is that both pods are pinned to one Pi and neither can be
+  # rescheduled while it's down. Acceptable for a homelab SSO that is already
+  # a single logical instance; the fix, if it ever matters, is
+  # ReadWriteMany-capable storage, not a second PVC.
+  MEDIA_STORAGE_SIZE: 2Gi
+  MEDIA_STORAGE_CLASS: local-path
+
+  # --- Sizing -----------------------------------------------------------
+  #
+  # Pi 4, 3.8Gi usable, already running CNPG + Prometheus + MetalLB. The
+  # worker gets more headroom than the server because it's the half that runs
+  # migrations on startup and processes outposts/policies in the background;
+  # it is the one that gets OOM-killed if these are set naively equal.
+  SERVER_MEMORY_REQUEST: 384Mi
+  SERVER_MEMORY_LIMIT: 768Mi
+  WORKER_MEMORY_REQUEST: 512Mi
+  WORKER_MEMORY_LIMIT: 1Gi

src/authentik/ansible/proxmox/README.md

@@ -0,0 +1,15 @@
+# Authentik on Proxmox — placeholder
+
+Not deployed on Proxmox. Authentik runs on the k3s cluster
+(`../kubernetes/`), having moved off a hand-made Unraid container — see
+`docs/authentik-migration.md`.
+
+To add it here later: `vars.yml` of Proxmox-specific overrides, `install.yml`
+of install steps, and templates for the app's config and systemd unit, plus
+`src/authentik/terraform/` defining its LXC. See `src/forgejo/ansible/proxmox/`
+for a worked example, and CLAUDE.md → "Adding a new app" — in particular that
+an app must only ever be live on one platform at a time.
+
+This folder is kept empty-but-present on purpose: every app carries the same
+`common/` + `ansible/` + `terraform/` shape whether or not each platform is
+in use, so adding one later doesn't mean restructuring.

src/authentik/ansible/unraid/README.md

@@ -0,0 +1,23 @@
+# Authentik on Unraid — deliberately not managed here
+
+Authentik **does** currently run on Unraid, but it was created by hand
+through the Unraid UI, not by this repo. There is no `docker-compose.yml`
+here and there should not be one: adding it would put Authentik in `nas01`'s
+`stacks:` list, and CLAUDE.md → "Adding a new app" is explicit that an app
+must never be declared on two platforms at once — they would share a Vault
+path and a database, and two live deployments would corrupt each other's
+state.
+
+The k3s deployment under `../kubernetes/` is where Authentik is going. See
+`docs/authentik-migration.md` for the cutover: dump the database off the
+Unraid Postgres, restore onto the cluster's CNPG instance, bring Authentik up
+on k3s on the *same* version, verify, then move DNS.
+
+Once DNS has moved and the k3s instance has been trusted for long enough,
+stop and delete the hand-made Unraid container. That teardown is a UI action,
+not a `state: absent` run — this repo never deployed it, so it has nothing to
+tear down.
+
+This folder stays as a placeholder rather than being deleted, per CLAUDE.md:
+the shape is the point, and Authentik moving back to Unraid one day should
+mean writing a compose file here, not restructuring the app.

src/authentik/common/vars.yml

@@ -0,0 +1,38 @@
+---
+# Portable config — values that hold regardless of where Authentik runs.
+# Platform-specific values (namespace, storage class, ingress class, how the
+# database is reached) live in ../ansible/<platform>/vars.yml and are merged
+# over these at deploy time.
+env_defaults:
+  # >>> CONFIRM BEFORE THE CUTOVER <<<
+  #
+  # This must match the version currently running on Unraid at the moment
+  # the database is dumped. Authentik runs its Django migrations on startup
+  # against whatever schema it finds, and those migrations are one-way: a
+  # newer Authentik pointed at an older dump will silently upgrade the
+  # schema, and there is no downgrade path if the cutover then has to be
+  # rolled back. Deploy on the *same* version, confirm it works, flip DNS,
+  # and only then bump this in a separate commit.
+  #
+  # The chart version and the app version are the same string upstream (see
+  # ../ansible/kubernetes/helmchart.yaml.j2, which uses this for both), so
+  # this is the only place a version is written down.
+  AUTHENTIK_VERSION: "2026.5.4"
+
+  AUTHENTIK_HOST: auth.turtlesystems.uk
+  AUTHENTIK_URL: "https://auth.turtlesystems.uk"
+
+  # Authentik's own HTTP port inside the container. Not the published port —
+  # on k3s the Ingress terminates and forwards here; there is no host port.
+  AUTHENTIK_HTTP_PORT: "9000"
+
+  DB_NAME: authentik
+  DB_USER: authentik
+  # AUTHENTIK_SECRET_KEY and AUTHENTIK_POSTGRESQL__PASSWORD are secrets —
+  # sourced from Vault (homelab/authentik), never set here.
+  #
+  # AUTHENTIK_SECRET_KEY specifically must be carried over from the existing
+  # Unraid deployment rather than generated fresh: it signs session cookies,
+  # so a new value logs every user out at the moment DNS flips, and the
+  # cutover stops looking like the no-op it's meant to be. See
+  # docs/authentik-migration.md.

src/authentik/terraform/README.md

@@ -0,0 +1,11 @@
+# Authentik Terraform — placeholder
+
+No Terraform for Authentik. It runs on the k3s cluster, and the k3s cluster
+is 4 Raspberry Pis that already exist — there is no guest for Terraform to
+provision, the same reason `homelab-utils` has no `src/<app>/terraform/`
+entries of its own (CLAUDE.md → "K3s cluster").
+
+Terraform would only come into this if Authentik moved to Proxmox, where it
+would need a module defining its LXC (template, cores, memory, disk, IP),
+called from `build/config/terraform/main.tf`. See `src/forgejo/terraform/`
+for the shape.

src/forgejo/ansible/proxmox/README.md

@@ -0,0 +1,94 @@
+# ansible/proxmox/ — forgejo
+
+Native install into the Terraform-provisioned LXC (`../../terraform/`). Run by
+the shared `lxc_app` role, which merges `../../common/vars.yml` with `vars.yml`
+here, fetches `homelab/forgejo` from Vault, provisions the database, then
+includes `install.yml`.
+
+| File | What it is |
+|---|---|
+| `vars.yml` | Proxmox-specific overrides — paths, service user, exact release, LAN address of the shared Postgres |
+| `install.yml` | Packages, `git` user, directory layout, binary download, config/unit rendering, dump schedule |
+| `templates/app.ini.j2` | Forgejo's config — the equivalent of the `FORGEJO__section__KEY` env vars in `../unraid/docker-compose.yml` |
+| `templates/forgejo.service.j2` | systemd unit, replacing Compose's `restart: unless-stopped` |
+| `templates/forgejo-dbdump.*` | Nightly `pg_dump`, script + service + timer |
+| `templates/pgpass.j2` | Credentials for that dump, mode 0600 |
+
+## How this is backed up
+
+Nothing is bind-mounted in from the NAS. Every path Forgejo writes to —
+repositories, LFS, attachments, indexers, queues, sessions — is left at its
+default under `APP_DATA_PATH` on the container's own disk, so a vzdump of the
+guest is a complete copy of its filesystem state. Proxmox excludes bind mounts
+from vzdump, so an NFS-mounted repository directory would have been the one
+thing missing from the backup that was made to protect it.
+
+The off-box copy is the vzdump job in
+`build/config/ansible/roles/pve_backup`, writing to an NFS storage on the NAS.
+
+**The database is the other half.** Postgres runs elsewhere — the shared
+Postgres LXC at 192.168.50.54, a different guest and therefore a different
+archive — so a restored container would come back with every repository and no
+issues, pull requests, users or permissions. `forgejo-dbdump.timer` runs
+`pg_dump` into `FORGEJO_BACKUP_PATH` on the container's own disk, which puts
+the dump inside the same vzdump archive as the repositories it belongs to. One
+artifact, one restore.
+
+That only holds if the dump finishes before the backup window, so the two
+schedules are a pair:
+
+| | Set in | Default |
+|---|---|---|
+| Database dump | `vars.yml` → `FORGEJO_DB_DUMP_ONCALENDAR` | 01:30 |
+| vzdump job | `group_vars/proxmox_nodes.yml` → `pve_backup_schedule` | 02:00 |
+
+Move one and move the other. Retention is independent: `pg_dump` output is
+pruned inside the container by `FORGEJO_BACKUP_KEEP_DAYS`, while the archives
+themselves are pruned by the storage's `prune-backups` settings — Proxmox
+prunes archives, not their contents.
+
+### Restoring
+
+1. Restore the container from the archive (`pct restore`, or the UI).
+2. Recreate the database and role if they're gone —
+   `ansible-playbook playbooks/proxmox.yml -e only_stacks=forgejo` does it
+   idempotently.
+3. Load the newest dump from `/var/lib/forgejo/backups`:
+   ```sh
+   systemctl stop forgejo
+   zcat /var/lib/forgejo/backups/forgejo-<stamp>.sql.gz \
+     | psql -h <postgres-host> -U forgejo -d forgejo
+   systemctl start forgejo
+   ```
+
+## Upgrading Forgejo
+
+`FORGEJO_VERSION` in `../../common/vars.yml` is `10` — a Docker tag, a rolling
+pointer at the latest 10.x, which is all the Compose stack needs. A native
+install downloads one specific artifact, so `vars.yml` here carries
+`FORGEJO_RELEASE` with the exact version. Bump it and redeploy: the new binary
+lands alongside the running one, the `/usr/local/bin/forgejo` symlink flips,
+and the service restarts. The old binary stays on disk, so rolling back is
+editing `FORGEJO_RELEASE` and redeploying.
+
+## Differences from the Unraid stack
+
+- **SSH port 22, not 2222.** On Unraid, 2222 was a published Docker port
+  dodging the host's own sshd. Here the container has its own IP and its own
+  sshd, which Forgejo uses directly (`START_SSH_SERVER = false`) — it manages
+  the `git` user's `authorized_keys` rather than running its own server. This
+  changes the SSH clone URLs Forgejo advertises, so existing remotes need
+  updating after a migration.
+- **`INSTALL_LOCK = true`.** `app.ini` is rendered in full from Vault, so the
+  web installer is skipped entirely. Nothing is left for Forgejo to generate
+  on first boot — which is why `LFS_JWT_SECRET` has to exist in Vault here
+  even though the Compose stack lets Forgejo invent one.
+- **`DB_HOST` is a LAN address**, not the `shared-postgres` Docker network
+  alias — there's no `unraid_shared` network to resolve it through.
+
+## One platform at a time
+
+Both platforms use the same Vault path and the same database, so they must not
+run at once. This was why `host_vars/forgejo.yml` carried an empty `apps:`
+list for so long; nas2 has since been retired, the list is live, and the
+bring-up runbook is `docs/forgejo-proxmox.md`.

src/forgejo/ansible/proxmox/install.yml

@@ -0,0 +1,190 @@
+---
+# App-specific install steps, included by the `lxc_app` role after it has
+# merged ../../common/vars.yml with ./vars.yml into `app_config` and fetched
+# `vault_secrets` from the app's Vault path. The role handles enabling and
+# starting the service once this file has put the unit in place.
+#
+# Template paths are absolute rather than relative: this file is included from
+# a role, so a bare `src:` would be looked up against that role's templates/
+# directory, not this app's.
+
+# A native install renders app.ini in full and sets INSTALL_LOCK, so nothing
+# is left for Forgejo to generate on first boot. A missing key would otherwise
+# surface as an undefined-variable error three tasks later.
+- name: Check that every required Vault key is present
+  ansible.builtin.assert:
+    that:
+      - vault_secrets.DB_PASSWORD is defined
+      - vault_secrets.SECRET_KEY is defined
+      - vault_secrets.INTERNAL_TOKEN is defined
+      - vault_secrets.JWT_SECRET is defined
+      - vault_secrets.LFS_JWT_SECRET is defined
+    fail_msg: >-
+      Vault path {{ app.vault_path }} is missing one or more of DB_PASSWORD,
+      SECRET_KEY, INTERNAL_TOKEN, JWT_SECRET, LFS_JWT_SECRET. See
+      docs/vault-secrets.md — LFS_JWT_SECRET is needed by the native install
+      even though the Unraid Compose stack lets Forgejo generate it.
+    quiet: true
+
+- name: Install packages Forgejo needs
+  ansible.builtin.apt:
+    name:
+      - git
+      - gzip
+      - ca-certificates
+      # pg_dump for the nightly database dump at the end of this file. The
+      # distro client and the shared Postgres LXC (17, from the PGDG archive)
+      # are close enough that either direction works, but the supported one is
+      # a client at least as new as the server — check this if the dump ever
+      # starts failing after a Postgres major upgrade over there.
+      - postgresql-client
+    state: present
+    update_cache: true
+    cache_valid_time: 3600
+
+- name: Create the Forgejo service group
+  ansible.builtin.group:
+    name: "{{ app_config.FORGEJO_USER }}"
+    system: true
+    state: present
+
+- name: Create the Forgejo service user
+  ansible.builtin.user:
+    name: "{{ app_config.FORGEJO_USER }}"
+    group: "{{ app_config.FORGEJO_USER }}"
+    system: true
+    # A real shell, not nologin: this account is also the SSH login clients
+    # use for git@host:owner/repo.git.
+    shell: /bin/bash
+    home: "{{ app_config.FORGEJO_HOME }}"
+    create_home: true
+    comment: Forgejo
+
+- name: Create Forgejo directories
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    state: directory
+    owner: "{{ item.owner | default(app_config.FORGEJO_USER) }}"
+    group: "{{ item.group | default(app_config.FORGEJO_USER) }}"
+    mode: "{{ item.mode }}"
+  loop:
+    - { path: "{{ app_config.FORGEJO_HOME }}", mode: "0750" }
+    - { path: "{{ app_config.FORGEJO_DATA_PATH }}", mode: "0750" }
+    - { path: "{{ app_config.FORGEJO_LOG_PATH }}", mode: "0750" }
+    # Forgejo writes the git user's authorized_keys here itself.
+    - { path: "{{ app_config.FORGEJO_HOME }}/.ssh", mode: "0700" }
+    # Nightly database dumps. On the container's own disk on purpose — that is
+    # what gets them into the vzdump archive.
+    - { path: "{{ app_config.FORGEJO_BACKUP_PATH }}", mode: "0700" }
+    # Config is root-owned and group-readable by Forgejo: it holds the
+    # database password and the three signing secrets, and Forgejo has no
+    # business rewriting it — everything in it is rendered from Vault below.
+    - { path: "{{ app_config.FORGEJO_CONFIG_DIR }}", owner: root, mode: "0750" }
+    - { path: "{{ app_config.FORGEJO_INSTALL_DIR }}", owner: root, group: root, mode: "0755" }
+  loop_control:
+    label: "{{ item.path }}"
+
+- name: Set Forgejo release facts
+  ansible.builtin.set_fact:
+    forgejo_artifact: >-
+      forgejo-{{ app_config.FORGEJO_RELEASE }}-linux-{{ app_config.FORGEJO_ARCH }}
+    forgejo_release_url: >-
+      https://codeberg.org/forgejo/forgejo/releases/download/v{{ app_config.FORGEJO_RELEASE }}
+
+- name: Download the Forgejo binary
+  ansible.builtin.get_url:
+    url: "{{ forgejo_release_url }}/{{ forgejo_artifact }}"
+    dest: "{{ app_config.FORGEJO_INSTALL_DIR }}/{{ forgejo_artifact }}"
+    # Checksum from the release's own .sha256 file. Ansible fetches it and
+    # matches the line for this artifact — which is why `dest` keeps the
+    # upstream filename rather than being renamed to plain `forgejo`.
+    checksum: "sha256:{{ forgejo_release_url }}/{{ forgejo_artifact }}.sha256"
+    owner: root
+    group: root
+    mode: "0755"
+  register: forgejo_binary
+
+# Versioned file plus a symlink, rather than overwriting one path: an upgrade
+# lands the new binary alongside the running one and flips the link, so a
+# failed download can't leave a half-written executable in place.
+- name: Link the current Forgejo binary onto PATH
+  ansible.builtin.file:
+    src: "{{ app_config.FORGEJO_INSTALL_DIR }}/{{ forgejo_artifact }}"
+    dest: /usr/local/bin/forgejo
+    state: link
+  register: forgejo_link
+
+- name: Render Forgejo configuration
+  ansible.builtin.template:
+    src: "{{ app_local_dir }}/ansible/proxmox/templates/app.ini.j2"
+    dest: "{{ app_config.FORGEJO_CONFIG_PATH }}"
+    owner: root
+    group: "{{ app_config.FORGEJO_USER }}"
+    mode: "0640"
+  register: forgejo_config
+  no_log: true
+
+- name: Install the Forgejo systemd unit
+  ansible.builtin.template:
+    src: "{{ app_local_dir }}/ansible/proxmox/templates/forgejo.service.j2"
+    dest: /etc/systemd/system/forgejo.service
+    owner: root
+    group: root
+    mode: "0644"
+  register: forgejo_unit
+
+# `lxc_app` starts the service after this file, but starting an already-running
+# service is a no-op — it would not pick up a changed app.ini or a new binary.
+- name: Restart Forgejo if its binary or configuration changed
+  ansible.builtin.systemd_service:
+    name: forgejo
+    state: restarted
+    daemon_reload: true
+  when: forgejo_binary.changed or forgejo_link.changed
+        or forgejo_config.changed or forgejo_unit.changed
+
+# --- Database dump -------------------------------------------------------
+#
+# The vzdump job on the Proxmox node captures this container's filesystem, but
+# Forgejo's database is not in it — Postgres runs elsewhere, in the shared
+# Postgres LXC at 192.168.50.54, which is a *different guest* and so a
+# different archive. Restoring the container alone would give back every repository with
+# no issues, pull requests, users or permissions. So dump the database into the
+# container's own filesystem on a schedule that finishes before the vzdump
+# window, and the one archive holds both halves.
+
+- name: Render the Postgres password file for the dump job
+  ansible.builtin.template:
+    src: "{{ app_local_dir }}/ansible/proxmox/templates/pgpass.j2"
+    dest: "{{ app_config.FORGEJO_HOME }}/.pgpass"
+    owner: "{{ app_config.FORGEJO_USER }}"
+    group: "{{ app_config.FORGEJO_USER }}"
+    mode: "0600"
+  no_log: true
+
+- name: Install the database dump script
+  ansible.builtin.template:
+    src: "{{ app_local_dir }}/ansible/proxmox/templates/forgejo-dbdump.sh.j2"
+    dest: /usr/local/bin/forgejo-dbdump
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Install the database dump systemd units
+  ansible.builtin.template:
+    src: "{{ app_local_dir }}/ansible/proxmox/templates/{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    owner: root
+    group: root
+    mode: "0644"
+  loop:
+    - forgejo-dbdump.service
+    - forgejo-dbdump.timer
+  register: forgejo_dbdump_units
+
+- name: Enable the nightly database dump timer
+  ansible.builtin.systemd_service:
+    name: forgejo-dbdump.timer
+    enabled: true
+    state: started
+    daemon_reload: "{{ forgejo_dbdump_units.changed }}"

src/forgejo/ansible/proxmox/templates/app.ini.j2

@@ -0,0 +1,116 @@
+{#
+  Rendered by Ansible from src/forgejo/common/vars.yml merged with
+  ../vars.yml (as `app_config`), plus Vault path homelab/forgejo (as
+  `vault_secrets`). The native-install equivalent of the
+  FORGEJO__section__KEY environment variables in ../../unraid/docker-compose.yml.
+
+  Do not edit this file on the host — the next deploy overwrites it. It is
+  root-owned and group-readable by Forgejo precisely so Forgejo cannot rewrite
+  it behind Ansible's back.
+
+  Note what is NOT configured here: every path Forgejo stores state under is
+  left at its default beneath APP_DATA_PATH — indexers, queues, sessions,
+  avatars, packages. That is deliberate. All of it sitting on the container's
+  own disk is what makes a vzdump of this guest a complete copy of Forgejo's
+  filesystem state.
+#}
+APP_NAME = Forgejo
+RUN_USER = {{ app_config.FORGEJO_USER }}
+RUN_MODE = prod
+WORK_PATH = {{ app_config.FORGEJO_WORK_PATH }}
+
+[server]
+PROTOCOL = http
+DOMAIN = {{ app_config.FORGEJO_DOMAIN }}
+ROOT_URL = {{ app_config.FORGEJO_ROOT_URL }}
+HTTP_ADDR = 0.0.0.0
+HTTP_PORT = {{ app_config.FORGEJO_HTTP_PORT }}
+APP_DATA_PATH = {{ app_config.FORGEJO_DATA_PATH }}
+{# The container's own sshd serves git over SSH, so Forgejo only advertises
+   the address and manages the git user's authorized_keys. On Unraid this was
+   Forgejo's built-in server behind a published Docker port. #}
+SSH_DOMAIN = {{ app_config.FORGEJO_DOMAIN }}
+SSH_PORT = {{ app_config.FORGEJO_SSH_PORT }}
+START_SSH_SERVER = false
+DISABLE_SSH = false
+LFS_START_SERVER = true
+LFS_JWT_SECRET = {{ vault_secrets.LFS_JWT_SECRET }}
+
+[database]
+DB_TYPE = postgres
+HOST = {{ app_config.DB_HOST }}:{{ app_config.DB_PORT }}
+NAME = {{ app_config.DB_NAME }}
+USER = {{ app_config.DB_USER }}
+PASSWD = {{ vault_secrets.DB_PASSWORD }}
+{# The shared Postgres publishes a plain port on the LAN and has no TLS
+   configured — see src/shared/postgres/ansible/unraid/docker-compose.yml.
+   Change both together if that ever gains a certificate. #}
+SSL_MODE = disable
+
+[repository]
+ROOT = {{ app_config.FORGEJO_DATA_PATH }}/forgejo-repositories
+
+[security]
+{# Fully configured from Vault, so skip the web installer entirely — without
+   this, a fresh container serves the setup wizard on first boot and would
+   write its own app.ini over this one. #}
+INSTALL_LOCK = true
+SECRET_KEY = {{ vault_secrets.SECRET_KEY }}
+INTERNAL_TOKEN = {{ vault_secrets.INTERNAL_TOKEN }}
+
+[oauth2]
+JWT_SECRET = {{ vault_secrets.JWT_SECRET }}
+
+[oauth2_client]
+{# Policy for signing in through an external OIDC provider (Authentik). The
+   provider itself is NOT configured here and cannot be: Forgejo keeps
+   authentication sources in its database, added once with
+   `forgejo admin auth add-oauth` — see step 8 of docs/forgejo-proxmox.md.
+   This section only decides what happens to a user who has just
+   authenticated there. #}
+
+{# Create a Forgejo account on first successful OIDC login. Without this,
+   Authentik authenticates the user and Forgejo then has nobody to log in as.
+   Gated by the registration settings below, not independent of them. #}
+ENABLE_AUTO_REGISTRATION = true
+
+{# Attach an OIDC identity to an existing local account when the email
+   matches, rather than creating a second one alongside it. This is what lets
+   the break-glass admin from step 4b become the same account you arrive as
+   through Authentik, instead of demoting you to a fresh unprivileged user on
+   first SSO login. `login` would ask the user to confirm by entering their
+   local password; `auto` links silently, which is only safe because
+   Authentik is the sole source of verified addresses here. #}
+ACCOUNT_LINKING = auto
+
+{# Take the Forgejo username from the provider's preferred_username claim.
+   The alternative, `userid`, would name accounts after Authentik's opaque
+   subject UUID. #}
+USERNAME = nickname
+
+[service]
+{# INSTALL_LOCK skips the wizard, which is also where these would have been
+   chosen.
+
+   These two are a pair and the combination is deliberate. DISABLE_REGISTRATION
+   cannot stay `true` once SSO is wanted: it blocks OIDC auto-registration as
+   well as the local signup form, so Authentik logins authenticate correctly
+   and are then refused an account — which reads like a broken provider rather
+   than a policy setting. ALLOW_ONLY_EXTERNAL_REGISTRATION restores exactly
+   the property that was wanted: no self-service signup, accounts only through
+   a configured provider.
+
+   Safe to have in place before the Authentik source exists. With no external
+   provider configured, "only external registration" permits nothing, so this
+   is not a window during which the forge is open — flipping the first value
+   alone would have been. #}
+DISABLE_REGISTRATION = false
+ALLOW_ONLY_EXTERNAL_REGISTRATION = true
+REQUIRE_SIGNIN_VIEW = false
+
+[log]
+{# journald, via systemd capturing stdout — `journalctl -u forgejo`. ROOT_PATH
+   still matters: some subsystems write their own files regardless. #}
+MODE = console
+LEVEL = info
+ROOT_PATH = {{ app_config.FORGEJO_LOG_PATH }}

src/forgejo/ansible/proxmox/templates/forgejo-dbdump.service.j2

@@ -0,0 +1,16 @@
+{# Rendered by Ansible — see ../install.yml. #}
+[Unit]
+Description=Dump the Forgejo database into this container's filesystem
+Documentation=file://{{ app_config.FORGEJO_CONFIG_PATH }}
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+User={{ app_config.FORGEJO_USER }}
+Group={{ app_config.FORGEJO_USER }}
+ExecStart=/usr/local/bin/forgejo-dbdump
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectSystem=full
+ReadWritePaths={{ app_config.FORGEJO_BACKUP_PATH }}

src/forgejo/ansible/proxmox/templates/forgejo-dbdump.sh.j2

@@ -0,0 +1,50 @@
+#!/bin/bash
+{# Rendered by Ansible — see ../install.yml. Do not edit on the host. #}
+#
+# Dumps the Forgejo database into this container's filesystem, where the
+# Proxmox vzdump job will pick it up along with everything else. Postgres runs
+# elsewhere (shared, on Unraid), so without this a restored container would
+# have every repository and no issues, pull requests, users or permissions.
+#
+# Run by forgejo-dbdump.timer, not by hand — though running it by hand is
+# harmless and is the quickest way to check the credentials work.
+
+# pipefail matters more than usual here: pg_dump feeds gzip, and without it a
+# failed dump still exits 0 through gzip and gets published as a valid-looking
+# but truncated archive.
+set -euo pipefail
+
+readonly BACKUP_DIR="{{ app_config.FORGEJO_BACKUP_PATH }}"
+readonly KEEP_DAYS="{{ app_config.FORGEJO_BACKUP_KEEP_DAYS }}"
+
+export PGPASSFILE="{{ app_config.FORGEJO_HOME }}/.pgpass"
+
+umask 077
+
+dest="${BACKUP_DIR}/{{ app_config.DB_NAME }}-$(date -u +%Y%m%dT%H%M%SZ).sql.gz"
+
+# Write to .part and rename only on success, so a dump interrupted by the
+# vzdump window (or a reboot) never leaves a partial file that looks complete
+# to the retention sweep below.
+pg_dump \
+  --host="{{ app_config.DB_HOST }}" \
+  --port="{{ app_config.DB_PORT }}" \
+  --username="{{ app_config.DB_USER }}" \
+  --dbname="{{ app_config.DB_NAME }}" \
+  --format=plain \
+  --no-owner \
+  --no-privileges \
+  | gzip -9 > "${dest}.part"
+
+mv "${dest}.part" "${dest}"
+
+# Retention is here rather than in the storage's prune settings because these
+# live inside the container: Proxmox prunes backup archives, not their
+# contents. Only complete dumps are counted, so a stale .part never displaces
+# a good one.
+find "${BACKUP_DIR}" -maxdepth 1 -type f -name '*.sql.gz' \
+  -mtime "+${KEEP_DAYS}" -delete
+find "${BACKUP_DIR}" -maxdepth 1 -type f -name '*.sql.gz.part' \
+  -mtime +1 -delete
+
+echo "wrote ${dest}"

src/forgejo/ansible/proxmox/templates/forgejo-dbdump.timer.j2

@@ -0,0 +1,16 @@
+{# Rendered by Ansible — see ../install.yml. #}
+[Unit]
+Description=Nightly Forgejo database dump
+
+[Timer]
+{# Must land before the vzdump window on the Proxmox node
+   (build/config/ansible/inventory/host_vars/pve.yml). The archive is only a
+   complete restore point if the dump inside it is from the same night. #}
+OnCalendar={{ app_config.FORGEJO_DB_DUMP_ONCALENDAR }}
+{# Catch up after a reboot rather than skipping a night silently. #}
+Persistent=true
+{# No randomised delay: the gap to the backup window is the whole point. #}
+AccuracySec=1min
+
+[Install]
+WantedBy=timers.target

src/forgejo/ansible/proxmox/templates/forgejo.service.j2

@@ -0,0 +1,33 @@
+{# Rendered by Ansible — see ../install.yml. The native-install replacement for
+   Compose's `restart: unless-stopped`. #}
+[Unit]
+Description=Forgejo
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User={{ app_config.FORGEJO_USER }}
+Group={{ app_config.FORGEJO_USER }}
+WorkingDirectory={{ app_config.FORGEJO_WORK_PATH }}
+ExecStart=/usr/local/bin/forgejo web --config {{ app_config.FORGEJO_CONFIG_PATH }}
+Restart=always
+RestartSec=2s
+Environment=USER={{ app_config.FORGEJO_USER }}
+Environment=HOME={{ app_config.FORGEJO_HOME }}
+Environment=GITEA_WORK_DIR={{ app_config.FORGEJO_WORK_PATH }}
+
+# HTTP_PORT is 3000, so no CAP_NET_BIND_SERVICE is needed. Forgejo does need
+# to write the git user's authorized_keys and its own data, so the filesystem
+# is not made read-only wholesale.
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectSystem=full
+ProtectHome=false
+ReadWritePaths={{ app_config.FORGEJO_HOME }}
+{# Restarting mid-push should not orphan git subprocesses. #}
+KillMode=control-group
+TimeoutStopSec=30s
+
+[Install]
+WantedBy=multi-user.target