commit 1250c9cef61f662aad27636ffc831b9c3ea77cfc Author: Russell Seymour Date: Mon Aug 24 20:52:36 2026 +0100 Initial checkin of code for managing homelab diff --git a/.forgejo/workflows/deploy.yml b/.forgejo/workflows/deploy.yml new file mode 100644 index 0000000..2ddc8a3 --- /dev/null +++ b/.forgejo/workflows/deploy.yml @@ -0,0 +1,116 @@ +name: deploy + +on: + push: + branches: [main] + paths: + # No build/config/terraform/** — this workflow only deploys to Unraid, + # and `terraform apply` is run by hand. Add it if that ever changes. + - "src/**" + - "build/config/ansible/**" + +jobs: + deploy: + # Requires a self-hosted Forgejo Actions runner on your LAN (labelled + # `unraid-deploy`) with: network access to Vault and to the Unraid hosts' + # SSH + Postgres ports, and Python + pip available. See README.md + # "Bootstrapping" — this workflow can only run once that runner exists, + # which is why the first deploy has to happen by hand. The SSH key + # `ansible_user` (root) authenticates with is fetched from Vault below, + # not stored as a runner or repo secret — see README.md "SSH access". + runs-on: unraid-deploy + env: + VAULT_ADDR: ${{ secrets.VAULT_ADDR }} + VAULT_AUTH_METHOD: approle + VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID }} + VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # need history to diff — a shallow clone can't compute this + + - name: Install Ansible and Python dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install ansible hvac psycopg2-binary + + - name: Install Ansible collections + working-directory: build/config/ansible + run: ansible-galaxy collection install -r requirements.yml + + - name: Fetch SSH deploy key from Vault + run: | + set -eu + python3 - <<'PY' + import os + import hvac + + client = hvac.Client(url=os.environ["VAULT_ADDR"]) + client.auth.approle.login( + role_id=os.environ["VAULT_ROLE_ID"], + secret_id=os.environ["VAULT_SECRET_ID"], + ) + secret = client.secrets.kv.v2.read_secret_version( + path="homelab/ci/ssh", mount_point="kv" + )["data"]["data"] + + key_path = os.path.join(os.environ["RUNNER_TEMP"], "unraid_ansible_key") + with open(key_path, "w") as f: + f.write(secret["PRIVATE_KEY"].rstrip() + "\n") + os.chmod(key_path, 0o600) + PY + echo "ANSIBLE_PRIVATE_KEY_FILE=$RUNNER_TEMP/unraid_ansible_key" >> "$GITHUB_ENV" + + - name: Determine changed stacks + id: changed + run: | + set -eu + # Three possible outcomes, and they are not the same thing: + # all=true deploy every stack on every host + # only_stacks=a,b deploy just those + # skip=true nothing here affects the Unraid deployment + # An empty only_stacks must never be read as "deploy everything" — + # that is exactly what a Proxmox-only or Terraform-only push + # produces. + # + # A change under build/config/ansible/ can affect how every stack + # is deployed (the role, inventory, playbook), so don't try to + # narrow it. Same if the diff can't be computed at all (force-push, + # or a first push where `before` is all-zeros): fall back to a full + # deploy rather than silently deploying nothing. + if ! changed_files=$(git diff --name-only "${{ github.event.before }}" "${{ github.sha }}" 2>/dev/null); then + echo "all=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + if echo "$changed_files" | grep -q '^build/config/ansible/'; then + echo "all=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + # Only changes that affect the Unraid deployment count, since that + # is all this job runs. A stack's common/ and ansible/unraid/ do; + # its terraform/ and ansible/proxmox/ don't. + stacks=$(echo "$changed_files" \ + | { grep '^src/' || true; } \ + | { grep -Ev '/(terraform|ansible/proxmox)/' || true; } \ + | awk -F/ '{ print ($2 == "shared") ? $3 : $2 }' \ + | sort -u | paste -sd, -) + if [ -z "$stacks" ]; then + echo "skip=true" >> "$GITHUB_OUTPUT" + else + echo "only_stacks=$stacks" >> "$GITHUB_OUTPUT" + fi + + # Unraid only. Deploying to Proxmox from CI would need the runner to + # reach the Proxmox API and the guests, and `terraform apply` is + # deliberately kept manual — creating and destroying LXCs on a push is + # a bigger blast radius than restarting a Compose stack. Run + # playbooks/proxmox.yml by hand for now. + - name: Run deploy playbook + if: steps.changed.outputs.skip != 'true' + working-directory: build/config/ansible + run: | + if [ "${{ steps.changed.outputs.all }}" = "true" ]; then + ansible-playbook playbooks/unraid.yml + else + ansible-playbook playbooks/unraid.yml -e only_stacks="${{ steps.changed.outputs.only_stacks }}" + fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e5a72a3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,33 @@ +# Ansible +*.retry +build/config/ansible/inventory/*.retry + +# Rendered secrets — only .env.example is ever committed +**/.env +!**/.env.example + +# Terraform +# .terraform/ holds the provider cache and, after +# `terraform init -backend-config=...`, the Postgres connection string with +# its password — never commit it. .terraform.lock.hcl is deliberately NOT +# ignored: it pins provider versions and belongs in git. +.terraform/ +*.tfstate +*.tfstate.* +*.tfvars +!*.tfvars.example +# A saved plan (`terraform plan -out=…`) is not just a diff: it embeds a copy +# of the state it was made against *and* the values of every input variable, +# `sensitive` ones included — so build/config/terraform/deploy.plan contains +# the Proxmox API token in the clear. Ignored so that a `git add -A` can't +# publish it. +*.plan + +# OS cruft +.DS_Store +Thumbs.db + +# Scratch output that isn't state and isn't secret-free — e.g. the k3s +# kubeconfig playbooks/k3s.yml fetches to local/k3s/.kubeconfig, +# which carries a client cert. +local/ \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..6a70136 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,597 @@ +# Homelab IaC — Project Notes + +Manages apps as infrastructure-as-code across three platforms — Unraid +server(s), Proxmox, and a bare-metal k3s cluster — with secrets sourced from +HashiCorp Vault and CI/CD via Forgejo Actions (self-hosted, on the Forgejo +instance this repo deploys). + +Ansible is the deployment tool on **all three**; what differs is the +deployment primitive and who creates the host: + +| | Unraid | Proxmox | k3s | +|---|---|---|---| +| Provision the host | already exists | Terraform (`bpg/proxmox`) creates an LXC | already exists (4 Raspberry Pis) | +| Install the app | Ansible + `compose_stack` → `docker compose up -d` | Ansible + `lxc_app` → native install + systemd | Ansible + `k3s_app` → manifests into k3s's auto-deploy dir | +| App config source | `common/vars.yml` + `ansible/unraid/vars.yml` | `common/vars.yml` + `ansible/proxmox/vars.yml` | `common/vars.yml` + `ansible/kubernetes/vars.yml` | +| App list lives in | `host_vars/.yml` `stacks:` | `host_vars/.yml` `apps:` | `group_vars/k3s_cluster.yml` `k3s_apps:` | + +The k3s column is the newest and reached that shape late: `homelab-utils` +started as cluster-bootstrap-only, with no `src//` entry, and the third +column was added when Authentik moved onto it (see +`docs/authentik-migration.md`). Two things about it don't reduce to the +table: + +- **The cluster itself has no `src/` entry.** Terraform never provisions it + and it isn't an app, so it gets an inventory group (`k3s_cluster`) and + `roles/k3s_node`, the same way the Proxmox *node* gets `roles/pve_backup` + while its guests get `lxc_app`. +- **Cluster services aren't apps.** `k3s_metallb`, `k3s_monitoring`, + `k3s_postgres` and `k3s_cert_manager` are the cluster's own + infrastructure — one hard-coded role each, config in `group_vars`. Apps go + through the generic `k3s_app` and live in `src/`. Both end up as + `HelmChart` CRs, so they look alike on disk; the distinction is whether + anything would still need it if no app were deployed. See "K3s cluster + (homelab-utils)" below. + +### Hosts + +Two Unraid entries are gone from the original plan: `nas2` was retired, and its +address (192.168.50.2) was reassigned to `turtle-proxmox-01` — so a stale copy +of it fails by reaching the wrong machine rather than by timing out. What +remains: `nas01` (Unraid, 192.168.50.1), `turtle-proxmox-01`/`-02` (Proxmox, +.2/.3), the guests on them from the API inventory, and the four k3s Pis +(.60–.63, MetalLB pool .80–.99). Gateway is 192.168.50.254. + +**Proxmox LXC guests live in 192.168.50.50–.59** — Forgejo .52, shared +Postgres .54. Addresses here are assigned by hand from that band, not derived +from anything, so a new guest's `ip_address` is a question for whoever owns +the network rather than a value to pick. + +The shared Postgres for everything outside the cluster is now the Proxmox LXC +at 192.168.50.54 — see `docs/postgres-proxmox.md`. The Unraid Compose +deployment under `src/shared/postgres/ansible/unraid/` is kept for the +platform-shape reason every unused platform folder is kept, not because it is +running anywhere. + +This repo supersedes the planning done in `../homelab/CLAUDE.md` — that file +has the original decision log if you want the "why" behind the architecture +in more detail. Multi-host support, the shared-Postgres pattern, and the +Forgejo-Actions-hosts-itself bootstrap problem were decided after that doc +was written; this file and `README.md` are the current source of truth. + +## Layout + +- `src//common/vars.yml` — `env_defaults:` that hold regardless of where + the app runs (version, ports, domain, DB name/user). +- `src//ansible/unraid/` — the Compose deployment: `docker-compose.yml`, + `.env.example`, a `vars.yml` of Unraid-specific overrides (appdata paths, + the `shared-postgres` Docker network alias), and optionally an `icon.png` + for the Unraid Docker page. +- `src//ansible/proxmox/` — the native install: `vars.yml` of + Proxmox-specific overrides, `install.yml` of app-specific install steps, + and templates for the app's own config file and systemd unit. Two apps + ship one: `src/forgejo/` (binary + systemd unit, the shape the role was + written for) and `src/shared/postgres/` (distro packaging, so the unit + comes from the package and `install.yml` only reconfigures the cluster). +- `src//ansible/kubernetes/` — the k3s deployment: `vars.yml` of + cluster-specific overrides, and one or more `*.yaml.j2` manifests (normally + a `HelmChart` CR, plus anything the chart won't create — see + `src/authentik/`). "kubernetes" rather than "k3s": the contents are plain + Kubernetes apart from the `HelmChart` CR. +- `src//terraform/` — a module defining this app's LXC (template, cores, + memory, disk, IP), called from `build/config/terraform/main.tf`. +- `src/shared//` — shared services (databases, etc.) used by + multiple apps, same shape. +- `build/config/ansible/` — inventory, playbooks, and the roles. Three of + them deploy *apps*, one per platform, and share the same layering and + `state: present|absent` contract: `compose_stack` (Unraid), `lxc_app` + (Proxmox guests), `k3s_app` (the cluster). `compose_stack` and `k3s_app` + each split into `deploy.yml`/`remove.yml`, dispatched by `main.yml` on the + entry's `state:`. The rest configure infrastructure rather than apps: + `k3s_node` (bootstraps a Pi as k3s server or agent, picked by inventory + group), `k3s_metallb`, `k3s_monitoring`, `k3s_postgres`, + `k3s_cert_manager` (cluster services), `k3s_maintenance` (drain/reboot/ + uncordon), `unattended_upgrades`, and `pve_backup` (the Proxmox node's + backup storage and vzdump schedule). +- `build/config/terraform/` — Proxmox provider + `backend "pg"` state config, + and the module calls saying what infrastructure exists. +- `.forgejo/workflows/` — CI/CD, runs once Forgejo + a self-hosted runner + exist (see "Bootstrapping" in README.md — chicken-and-egg on the first + deploy). + +## Key decisions + +- Compose files are static and reference `${VAR}`; only the rendered `.env` + changes per deploy. Nothing secret is ever committed. +- Config is layered, not duplicated: `common/vars.yml` merged with + `ansible//vars.yml` (platform wins), then Vault secrets on top + (secrets win). Both roles do this the same way, so moving an app between + platforms means writing a new platform vars file, not restating its config. +- One Vault KV path per app (`homelab/`), fetched as a whole dict via + `community.hashi_vault.vault_kv2_get`. The prefix is `homelab/`, not + `unraid/`, because an app's secrets don't change with the platform it lands + on. +- Unraid draws a Compose deployment's stack row and its containers from + unrelated sources, so one committed `src//ansible/unraid/icon.png` is + copied to two places. The **stack row** is Compose Manager serving + `/icon.png` off disk — a file or nothing, no label and no URL, + and only in the maintained fork (Compose Manager Plus); on the original + plugin the copy is inert. The **containers** get icon, WebUI link and + console shell from the dockerMan template that created them, which a Compose + stack doesn't have, so each service carries `net.unraid.docker.*` labels + (6.10+ falls back to them). The label path is a second copy under + `/mnt/user/appdata/icons/` rather than the project-folder one, because the + webgui resolves it on every Docker page load and that shouldn't hit the + flash drive. +- Of those labels only the icon is role-supplied: `compose_stack` injects + `STACK_ICON` *underneath* both vars.yml layers, so an app can override it + with a hosted URL. WebUI and shell stay literal in the compose file — the + WebUI port has to be the published one and the shell is a property of the + base image, so both belong next to `ports:` and `image:` rather than in a + variable. Details, including where SVG does and doesn't work, in README.md → + "Icons on the Unraid Docker page". +- Shared Postgres is provisioned once per host; per-app DB/user creation + is idempotent via `community.postgresql`, not `docker-entrypoint-initdb.d` + (which only runs once on an empty volume — wrong fit for a shared service + apps get added to over time). The Unraid instance was pinned to 13 because + its data directory predated this repo; the Proxmox one is 17 from the PGDG + archive, a fresh cluster with nothing to stay compatible with. +- The Proxmox shared Postgres lives on the `AppData` ZFS pool with a + `proxmox_replication` job to the second node, and **everything is on one + volume** — no `mount_point` splitting PGDATA from the WAL. Unlike Forgejo's + "no bind mounts", which is about what `vzdump` includes, this is a + correctness requirement: replication snapshots each of a guest's volumes + independently, so two volumes replicate as two snapshots taken at different + instants, and the far node can end up with a data directory and a WAL that + disagree. One volume means one atomic snapshot — exactly the + crash-consistent image WAL replay is designed for. It also means running + out of space has one fix and not two: raise `disk_size` on the module call + (256 GiB as of this writing, up from the original 32 once Forgejo's + database made the OS-sized rootfs look optimistic), never add a second + disk. Cheap to do — on ZFS the resize is a refquota change applied to a + running guest — but one-way, since shrinking is a replacement and + `prevent_destroy` blocks it. Runbook in `docs/postgres-proxmox.md`. Two + further consequences worth keeping straight: replication is configured *per guest*, not per pool, so a + container created on `AppData` without a job is a single point of failure + that looks identical in the storage view (hence Terraform owning the job + rather than a hand-run `pvesr`); and it's asynchronous, so a failover loses + up to `replication_schedule` — five minutes — of commits. It is not a + backup; `pve_backup`'s vzdump is. +- `full_page_writes = off` on the Proxmox instance, and only there. It guards + against a torn page, which copy-on-write storage cannot produce, so on ZFS + the full-page images are pure write amplification that also becomes + replication traffic. The same setting on ext4 or xfs risks unrecoverable + corruption — which is why it's in `ansible/proxmox/vars.yml` and must never + migrate to `common/vars.yml`. +- Terraform provisions the guest and stops there — it has no way to deploy an + app into an LXC, so Ansible picks up from the point the container exists. +- Proxmox guests keep all their state on their own rootfs; no bind mounts, no + NFS shares mounted into a container. Counter-intuitive if the goal is "back + the repos up to the NAS", but `vzdump` excludes bind mounts by design, so a + bind-mounted repo directory is the one thing *missing* from the archive. + Everything local means one archive is a complete filesystem restore point. + Getting a copy off-box is the node's job (`pve_backup` → NFS storage on the + NAS + a vzdump schedule), not the container's. +- State an app keeps *outside* its container is that app's problem. Forgejo's + database is on the shared Postgres LXC — a different guest, and so a + different vzdump archive — so a systemd timer in the + container `pg_dump`s it onto the container's own disk, and the vzdump archive + carries both halves. The dump schedule and the vzdump window are a pair — + move one, move the other. +- Proxmox hosts come from the `community.proxmox` dynamic inventory plugin + querying the Proxmox API, so `terraform apply` creating an LXC doesn't also + require an inventory edit. +- The `terraform` tag is what marks a Proxmox guest as this repo's to manage. + The dynamic inventory returns every LXC on the cluster, hand-made ones + included, so `inventory/proxmox.yml` filters on that tag — both Terraform + modules set it (`tags` in their `variables.tf`), nothing else does. It + filters rather than narrowing the `proxmox_guests` condition, so an + unmanaged guest is absent from `all` as well and no future play can reach + it by accident. The failure mode is silent in one direction: a new module + that forgets the tag creates a container Ansible simply never visits, with + no error anywhere — same shape as an omitted `stacks:` entry, and the first + thing to check if a freshly-applied guest is never deployed to. +- Terraform state lives in Postgres (`backend "pg"`), not a local file, so CI + and a workstation see the same state — specifically the **CloudNativePG + cluster on k3s**, not the shared Postgres this configuration provisions on + Proxmox. That would be circular: Terraform would need the database to exist + in order to create the container the database runs in. State goes somewhere + Terraform has no hand in building, which breaks the cycle outright at the + cost of a dependency the rest of the repo avoids — `terraform apply` now + needs the Pis up. Worth it over the alternative (local state, apply, + `init -migrate-state`), which has to be got right exactly once. Bootstrap + ordering is `docs/postgres-proxmox.md`. +- Every app keeps the same `common/` + `ansible/` + `terraform/` shape even + when only one platform is in use, so adding the other later doesn't require + restructuring. +- Removal on Unraid is explicit (`state: absent` on the stack entry), never + implied by deleting the entry. Ansible stores no record of the last run, so + an omitted stack isn't removed — it's just never visited again, and keeps + running unmanaged. The entry stays as a tombstone documenting that the app + was deliberately removed. Volumes, images and the app's database are + further opt-ins (`remove_volumes`, `remove_images`, `remove_database`) + rather than being implied by `state: absent`, and the Vault path is never + touched — a default teardown is meant to be reversible. `lxc_app` has no + counterpart yet; it would need a per-app `uninstall.yml`. +- The k3s cluster's join token is a fixed value from Vault + (`homelab/k3s-homelab-utils` → `K3S_TOKEN`), not the one `k3s server` + generates for itself on first install. Both the control-plane play and the + worker play look it up independently, so a full rebuild — wipe both SD + cards, reinstall — reproduces the same cluster identity instead of needing + a freshly-generated token fetched off the server and redistributed by hand. + Same "generate once, store it, never let a redeploy invent a new one" + reasoning as Forgejo's fixed secrets, applied to cluster identity instead + of app state. +- The SSH key the k3s plays connect with is fetched from Vault + (`homelab/ci/ssh-k3s`) by `playbooks/k3s_ssh_key.yml` and written to + `local/`, which both `k3s.yml` and `k3s_maintenance.yml` import as their + first play — rather than the operator loading it into `ssh-agent` first, as + the Unraid manual flow still expects. Same arrangement CI already uses for + the Unraid key, just moved into the playbook because these two are + manual-only and there's no workflow to hold the fetch step. It's a separate + playbook, not a `pre_tasks` block, because both entry points need it and + neither can assume the other ran. It targets `k3s_cluster` rather than + `localhost` for a non-obvious reason: the implicit localhost isn't a member + of `all`, so it wouldn't inherit `vault_addr`/`vault_kv_mount`/ + `vault_auth_method` from `group_vars/all.yml`. Nothing in it connects to a + Pi — `gather_facts: false` plus `delegate_to: localhost`, necessarily, since + the key isn't on disk yet at that point. Falling back to a local key takes + *two* overrides (`-e k3s_ssh_key_fetch=false` alongside + `-e ansible_ssh_private_key_file=...`): pointing at another key alone + doesn't help if Vault is what's broken, because the fetch play fails before + any other play runs. +- `playbooks/k3s.yml` is deliberately **not** wired into `deploy.yml` or + `.forgejo/workflows/deploy.yml`. Converging 4 physical nodes on every push + is a bigger blast radius than restarting a Compose stack — the same + reasoning that keeps `terraform apply` manual for Proxmox. Run it by hand. +- `playbooks/k3s.yml`'s plays are tagged so one piece can be converged on its + own — `nodes`, `metallb`, `monitoring`, `postgres`, `services` (those three + cluster services together), `upgrades`, `kubeconfig`. Play-level tags, one + per play, not per-task tags inside the roles: the plays already are the + units, and tagging tasks would mean every role carrying tags for a + selectivity only this playbook needs. This is the k3s equivalent of + `only_stacks` for Unraid — a cluster service isn't a `stacks:` entry, so + `only_stacks` can't reach it. The SSH-key import play is tagged `always` + rather than getting a name of its own, since it's the prerequisite for + anything else connecting and has to survive every `--tags` filter. Note + what a tagged service run does *not* cover: the roles only drop manifests + for k3s's controllers to reconcile, so a green playbook means the files + landed, not that the workload came up. +- `k3s_node` is one role, not two, even though a server and an agent install + differently. Both need the same OS prep (cgroups, swap), and which install + path runs is a group var (`k3s_node_role: server`/`agent`, set per + inventory group) rather than a second role — so the shared prep can't drift + between the two nodes types the way copy-pasted tasks eventually would. +- Extra `INSTALL_K3S_EXEC` flags split across two vars, not one: + `k3s_extra_args` (both roles) and `k3s_server_extra_args` (control-plane + only). `k3s agent` doesn't understand server-only flags like `--disable` + and fails to start if handed one, so a flag like `--disable=servicelb` + (which `k3s_metallb` needs to avoid fighting k3s's bundled ServiceLB for + the same IPs) can't safely live in the shared list. +- `k3s_node`'s install tasks track the exec line they last installed a node + with (a small marker file next to k3s's own config) and reinstall — + restarting the `k3s`/`k3s-agent` service, not the node — whenever it + drifts from what `k3s_extra_args`/`k3s_server_extra_args`/`k3s_api_tls_san` + currently say, even if `k3s_version` hasn't changed. Deliberately folded + into the ordinary install task rather than a separate maintenance + playbook: `playbooks/k3s.yml` is already manual-only and documented as + safe to re-run, and unlike the reboots `k3s_maintenance` guards, bouncing + the k3s process doesn't take pods down — containerd keeps them running + underneath it. The worker play still runs with `serial: 1` so at most one + node's kubelet is ever mid-restart at once. +- `k3s server`/`k3s agent` are installed with an explicit `--node-name + {{ inventory_hostname }}`, not left to default to the OS hostname. This is + what lets `roles/k3s_maintenance` address a node by + `inventory_hostname` when draining/uncordoning — the k8s node object and + the Ansible host are guaranteed to be the same string. +- Patching a k3s Pi is split into two roles that don't know about each other + directly, only through a file: `unattended_upgrades` installs updates + hands-off but with `Automatic-Reboot "false"`, and `k3s_maintenance` is the + only thing that actually reboots a node — triggered by the presence of + `/var/run/reboot-required`, which is the OS's own signal, not something + either role invents. Splitting it this way means the risky half (taking a + node out of the cluster) is exactly one thing (`playbooks/k3s_maintenance.yml`, + `serial: 1`), not entangled with the routine half (installing packages), + which runs unattended every day on every node without anyone thinking about + it. +- `k3s_maintenance`'s kubectl calls (`drain`, `wait`, `uncordon`) are + delegated to the control-plane host and run as `k3s kubectl`, k3s's own + bundled client, rather than requiring a kubectl install or a kubeconfig + anywhere else — including on whatever eventually runs this on a schedule. + This also has to work when the node currently being processed *is* the + control plane: delegating a host to itself is just a normal SSH connection, + used right up until the moment that connection reboots out from under it. +- `k3s_monitoring` (a lean kube-prometheus-stack, for OpenLens and similar + tools) is installed as a `HelmChart` custom resource dropped into k3s's own + auto-deploying manifests directory, not run through a `helm` binary or the + `kubernetes.core.helm` collection — k3s ships a helm-controller that + reconciles anything found there, the same mechanism it uses to install its + own bundled Traefik and ServiceLB. That keeps the pattern this repo already + uses everywhere else (Ansible renders a file, something else converges on + it) instead of adding a second, unrelated way to reach the cluster from the + controller. Grafana and Alertmanager are left disabled — OpenLens brings + its own dashboards and this cluster doesn't page anyone — and the + control-plane scrape targets (`kubeControllerManager`, `kubeScheduler`, + `kubeProxy`, `kubeEtcd`) are disabled too, since k3s bundles those into one + static binary instead of exposing them the way the chart expects; leaving + them on just produces permanently-"down" targets, not a working scrape. +- `k3s_metallb` (MetalLB, L2 mode) follows the same `HelmChart` CR pattern as + `k3s_monitoring`, plus a second, plain manifest for its + `IPAddressPool`/`L2Advertisement` config dropped in the same + auto-deploying directory — k3s's deploy controller applies any manifest it + finds there, not only `HelmChart` CRs, and retries one referencing CRDs + that don't exist yet rather than failing outright, so the config doesn't + need to wait on the chart install finishing first. It replaces k3s's + bundled ServiceLB rather than running next to it — both would otherwise + hand out IPs for the same `LoadBalancer` Services — so + `k3s_server_extra_args` in `inventory/group_vars/k3s_cluster.yml` + carries `--disable=servicelb` (server-only — `k3s_extra_args`, applied to + both server and agent, can't carry a server-only flag like `--disable` + without breaking agent installs). The IP pool (`k3s_metallb_address_range`, same + file) has no built-in default; the role fails fast rather than silently + advertising an empty pool. +- `k3s_postgres` (CloudNativePG) is the k3s cluster's own shared Postgres — + a third platform for `src/shared/postgres/`, alongside Unraid and Proxmox, + deployed via Helm rather than Compose or a native install. Chosen over + Bitnami's postgresql-ha (repmgr+pgpool, and Bitnami's free chart/image + catalog was restructured into a "legacy" repo in 2025) and the Zalando + operator (Patroni-based, heavier) because it fits the same `HelmChart` CR + pattern already established: the operator installs as a chart CR, same + shape as `k3s_metallb`/`k3s_monitoring`, and the actual cluster is a plain + `Cluster` CR manifest — same "config manifest that outlives the CRDs it + references" trick as `k3s_metallb`'s `IPAddressPool`. It's a separate + physical instance from the Docker-based shared/postgres (pinned to 13), + but reuses the same Vault path (`homelab/shared/postgres`) rather than a + k3s-specific one — one superuser identity for the "shared postgres" + concept regardless of which platform it's running on, consistent with how + every host already draws from that same path. 1 primary + 1 replica, + scheduled on worker Pis only via `nodeAffinity` — the control plane stays + free of app pods, same boundary `k3s_metallb`/`k3s_monitoring` already + keep. HA replication only for now, no backups. +- The CNPG cluster is published on the LAN through MetalLB at a pinned + `k3s_postgres_loadbalancer_ip`, on top of the three `ClusterIP` Services + CNPG makes for every `Cluster`. Declared inside the `Cluster` CR as a + managed service (`.spec.managed.services.additional`, `selectorType: rw`) + rather than as a Service manifest of this repo's own, so the operator owns + the selector and the address follows a failover instead of needing to be + re-pointed by hand. Pinned rather than auto-assigned because the address is + meant to be written down in config elsewhere; it comes out of the bottom of + `k3s_metallb_address_range` and would need a second `autoAssign: false` + pool if anything else on this cluster ever wants a fixed IP. This is what + closes the reachability half of per-app DB provisioning — the + `community.postgresql` tasks `compose_stack`/`lxc_app` use can now reach + this instance the same way they reach the Unraid one. `roles/k3s_app` is + the other half, and uses exactly those tasks; note the asymmetry it + relies on, which is easy to misread as a mistake. The *provisioning* runs + from the Ansible controller and therefore uses the LoadBalancer address + (`k3s_postgres_loadbalancer_ip`), because `delegate_to: localhost` is off + the cluster and can't route to a ClusterIP. The *app* uses the in-cluster + `shared-postgres-rw` DNS name, because sending pod traffic out to the LAN + and back would put MetalLB's L2 speaker in the path of every query for no + benefit. Two addresses for one database, each correct for its caller. +- `roles/k3s_app` is the cluster's counterpart to `compose_stack` and + `lxc_app`: the generic role that deploys an *app*, driven by `k3s_apps:` + in `group_vars/k3s_cluster.yml`, as distinct from the `k3s_*` roles that + each deploy one cluster service. It renders whatever `*.yaml.j2` the app + ships under `src//ansible/kubernetes/` into k3s's auto-deploying + manifests directory, so adding a manifest to an app is dropping a file + next to the others rather than a role change. Secrets go into a separate + Kubernetes Secret rendered from Vault, never interpolated into the + manifests — the same committed-config/generated-secrets split as Unraid's + static compose file plus rendered `.env`, which is why the manifests can + stay 0644 on the node while the Secret is 0600. +- `k3s_apps:` lives in `group_vars/k3s_cluster.yml`, not `host_vars/`, unlike + both other platforms. An app is deployed to the cluster, not to a node: + the play runs against `k3s_control_plane` only because that's where the + manifests directory is, which is an implementation detail of how k3s is + reached rather than a statement about where the app runs. +- Removal on k3s is one pass, not two. `compose_stack` needs the compose file + still on disk to run `docker compose down`, so `state: absent` and deleting + `src//` have to be separate commits (see "Removing an app" below). + k3s's deploy controller tracks which resources each manifest file created, + so deleting the file *is* the teardown and `src//` can go in the same + commit. The tombstone convention still applies for the same reason as + everywhere else — an entry dropped from `k3s_apps:` is never visited again + and keeps running unmanaged. +- `k3s_cert_manager` is the fourth cluster service and the first that exists + purely for apps rather than for the cluster — nothing in the cluster needs + a certificate, `k3s_app`'s tenants do. It's still a cluster service rather + than an app, on the same reasoning as the shared Postgres: one + `ClusterIssuer` that every app's Ingress annotates itself against, rather + than each app carrying its own ACME account and DNS credentials. DNS-01, + not HTTP-01, because HTTP-01 needs Let's Encrypt to reach the cluster from + the internet on port 80 and this LAN deliberately isn't reachable — which + has the useful side effect that a certificate can be issued *before* DNS + points at the cluster, so a migration's TLS is settled before its cutover. + The solver stanza itself has no default and the role fails fast without + one: it depends on who runs the DNS, which the repo can't know. Same + fail-fast-on-unset treatment as `k3s_metallb_address_range`. +- Apps on k3s reach the outside through the Traefik k3s already bundles, + on the MetalLB address its Service holds, rather than each getting its own + LoadBalancer IP. One entrypoint and host-based routing means adding an app + costs a DNS record, not a pool address — the opposite of the choice made + for `k3s_postgres`, which is pinned to its own IP precisely because + Postgres isn't HTTP and can't be name-routed. +- `k3s_traefik` is the one cluster-service role that installs nothing: k3s + installs Traefik itself, so the role only adjusts it, via a + `HelmChartConfig` merged over k3s's own `HelmChart` rather than an edit to + the `traefik.yaml` k3s rewrites on every server start. Its scope is the + dashboard, which a stock k3s 404s — Traefik still builds it, but the chart + stopped shipping the router that reaches it in v28. The role restores that + router on Traefik's internal `traefik` entrypoint (port 9000, unpublished → + port-forward only), and optionally publishes the dashboard on a hostname + behind an Authentik forward-auth middleware when + `k3s_traefik_dashboard_host` is set. Three things worth keeping straight: + the internal route stays on even when the hostname one exists, because + authenticated access depends on Authentik → CNPG → a healthy cluster, + exactly what you'd open the dashboard to diagnose; the published route is + an `IngressRoute` with an explicit cert-manager `Certificate` rather than + an annotated `Ingress`, because the dashboard is `api@internal` and has no + Service to point an `Ingress` at; and the role fails fast if a hostname is + set without an auth address, since the failure mode is publishing every + router, service and middleware on the cluster to the LAN unauthenticated. + Authentik's provider must be **forward auth (domain level)** — + single-application mode needs `/outpost.goauthentik.io/` routed on the + dashboard's own host, a cross-namespace service reference Traefik rejects + unless `allowCrossNamespace` is on. + +## Adding a new app + +1. `src//common/vars.yml` — `env_defaults:` for the portable config. +2. Whichever platform(s) it targets: + - **Unraid:** `src//ansible/unraid/` with `docker-compose.yml` + (static, `${VAR}`-driven), `vars.yml` of overrides, `.env.example` + documenting every var. + - **Proxmox:** `src//ansible/proxmox/` with `vars.yml`, + `install.yml`, and config/systemd templates; plus + `src//terraform/` defining its LXC, wired into + `build/config/terraform/main.tf`. + - **k3s:** `src//ansible/kubernetes/` with `vars.yml` and at least + one `*.yaml.j2` manifest — normally a `HelmChart` CR, plus anything the + chart won't create for you. See `src/authentik/` for a worked example, + including the two things a chart usually leaves out: a PVC, and secrets + referenced by `secretKeyRef` rather than set as values. +3. Populate its Vault path (`homelab/`) with real secret values. +4. Declare it on the target: + - Unraid → `host_vars/.yml` `stacks:` list. + - Proxmox → `host_vars/.yml` `apps:` list. + - k3s → `group_vars/k3s_cluster.yml` `k3s_apps:` list. + All three take the same entry shape (name, src, vault_path, optional + `db:`, optional `state:`). Never declare the same app on more than one at + once — they share a Vault path and a database, so two live deployments + corrupt each other's state. Moving an app *between* platforms is + therefore a cutover with a verification step in the middle, not an edit; + `docs/authentik-migration.md` is the worked example. +5. Leave the unused platforms' folders as README placeholders rather than + deleting them — the shape is the point. + +## Removing an app + +**Unraid.** Set `state: absent` on its `stacks:` entry and run +`playbooks/unraid.yml` — don't delete the entry, which removes nothing. Then +delete `src//` in a *second* pass, never before: `docker compose down` +needs the compose file to know what it's tearing down. Full detail, including +the opt-in flags for volumes/images/database, is in README.md → "Removing an +app". + +**k3s.** Same `state: absent` tombstone rule, but one pass — deleting the +manifests is the teardown, because k3s's deploy controller garbage-collects +what each file created, so `src//` can go in the same commit. The +database, PVCs and Vault path deliberately survive; there's no +`remove_volumes`/`remove_database` equivalent yet, and adding one is a +"when it's actually wanted" job rather than a guess at the shape. + +**Proxmox.** No teardown path at all — `lxc_app` would need a per-app +`uninstall.yml`, and neither app that ships an `install.yml` has one. Note +this is worse for the shared Postgres than for an ordinary app: tearing it +down means every other app's database as well. + +## K3s cluster (homelab-utils) + +4 Raspberry Pis, `inventory/hosts.yml` → `k3s_cluster` (`k3s_control_plane`: +1 host, `k3s_workers`: 3). No Terraform — the Pis already exist. The cluster +itself has no `src//` entry (it isn't an app), but apps now land on it: +`k3s_apps:` in `group_vars/k3s_cluster.yml`, deployed by `roles/k3s_app` from +`src//ansible/kubernetes/`. `playbooks/k3s.yml` is manual-only (see "Key +decisions" above); run it with: + +```sh +cd build/config/ansible +ansible-playbook playbooks/k3s.yml + +# or one piece of it — see "Key decisions" above for the tag list +ansible-playbook playbooks/k3s.yml --tags postgres + +# apps only, or one app +ansible-playbook playbooks/k3s.yml --tags apps +ansible-playbook playbooks/k3s.yml --tags apps -e only_apps=authentik +``` + +`--tags services` covers the cluster's own infrastructure and `--tags apps` +what runs on top of it; neither implies the other, so "everything except +reinstalling k3s" is `--tags services,apps`. + +Prerequisites, once, before the first run: the `ansible` user exists on every +Pi with NOPASSWD sudo and the **public** half of this repo's k3s SSH key +installed, and `homelab/k3s-homelab-utils` → `K3S_TOKEN` plus +`homelab/ci/ssh-k3s` → `PRIVATE_KEY` are set in Vault. Nothing needs setting +up on the controller beyond `VAULT_ADDR`/`VAULT_TOKEN` — the private key comes +from Vault at run time (see "Key decisions" above). Full detail in README.md → +"K3s (Raspberry Pi)" and `docs/vault-secrets.md`. + +**Rebuilding**: reimage the SD card(s), redo the `ansible` user + SSH key +step, then re-run the playbook — the fixed Vault token means the rebuilt +node(s) rejoin the same cluster identity rather than needing a token hunted +down from a live server. **Adding a fifth Pi**: add it under +`k3s_control_plane` or `k3s_workers` in `hosts.yml`; no `host_vars/` entry +needed, `k3s_node_role` comes from the group. This repo's `k3s_node/server.yml` +assumes a single control-plane node — adding a second one for HA needs +changes there, not just an inventory edit. + +**Patching**: `playbooks/k3s.yml` also applies `roles/unattended_upgrades` to +every node, so updates install themselves daily with no login required — but +never reboot themselves (`Automatic-Reboot "false"`). Run +`playbooks/k3s_maintenance.yml` to actually apply a pending reboot: it checks +`/var/run/reboot-required` per node and, only where set, drains → reboots → +waits for `Ready` → uncordons, `serial: 1` so only one node is ever down. +Manual-only for now, same as `playbooks/k3s.yml` — meant to eventually run on +a schedule from a self-hosted Forgejo Actions runner rather than by hand. + +**Metrics**: `playbooks/k3s.yml` also applies `roles/k3s_monitoring`, a lean +kube-prometheus-stack (no Grafana/Alertmanager) so OpenLens and similar tools +can show node/pod metrics — see "Key decisions" above for why it's a +`HelmChart` CR rather than a `helm` install, and README.md → "Metrics +(Prometheus)" for pointing OpenLens at it. + +**LoadBalancer IPs**: `playbooks/k3s.yml` also applies `roles/k3s_metallb`, +MetalLB in L2 mode, so `type: LoadBalancer` Services get real LAN IPs instead +of staying `ClusterIP`-only. Set the pool it hands out from — +`k3s_metallb_address_range` in `inventory/group_vars/k3s_cluster.yml` — to a +range your DHCP scope and every static assignment in `hosts.yml` steer clear +of before the first run; see "Key decisions" above for why it replaces +ServiceLB rather than running alongside it, and README.md → "LoadBalancer +IPs (MetalLB)" for more. + +There's no *node* teardown path, unlike `state: absent` for apps — removing a +Pi today means wiping its SD card and deleting its inventory entry by hand. + +**Shared Postgres**: `playbooks/k3s.yml` also applies `roles/k3s_postgres`, +CloudNativePG — the cluster's own shared Postgres, a third platform for +`src/shared/postgres/` alongside Unraid and Proxmox. See "Key decisions" +above for why CloudNativePG and the topology, and README.md → "Postgres +(CloudNativePG)" for connecting to it. Per-app database provisioning works +now (`roles/k3s_app`, `db:` on a `k3s_apps:` entry) — note the two-address +asymmetry described in "Key decisions". Still **no backups**: HA replication +only, which protects against a dead SD card and not against a dropped table. +That gap matters more now that Authentik's database lives here rather than +inside whatever covers Unraid's `/mnt/user/appdata`; a CNPG `ScheduledBackup` +is the obvious next piece of work. + +**TLS**: `playbooks/k3s.yml` also applies `roles/k3s_cert_manager` — +cert-manager plus one `ClusterIssuer` that every app's Ingress annotates +itself against. Two values have no default and the role refuses to run +without them: `k3s_cert_manager_acme_email` and `k3s_cert_manager_solver` +(the DNS-01 stanza, which depends on your DNS provider — a commented +Cloudflare example is in `group_vars/k3s_cluster.yml`). Use the Let's Encrypt +staging directory while working a solver out; production allows five failed +validations per hostname per hour. + +**Traefik dashboard**: `playbooks/k3s.yml` also applies `roles/k3s_traefik` — +the only cluster-service role that installs nothing, since k3s installs +Traefik itself. Out of the box the dashboard is reachable only by +port-forward (`kubectl -n kube-system port-forward deploy/traefik 9000:9000`, +then `http://127.0.0.1:9000/dashboard/` — trailing slash required; without +this role a stock k3s 404s there). Publishing it on a hostname behind +Authentik is one commented-out line, `k3s_traefik_dashboard_host` in +`group_vars/k3s_cluster.yml`, plus a domain-level forward-auth Proxy Provider +on the Authentik side. See "Key decisions" above and README.md → "Dashboard +(Traefik)". + +**Apps**: `k3s_apps:` in `group_vars/k3s_cluster.yml`, deployed by +`roles/k3s_app`. Authentik is the first and currently only one — it moved off +a hand-made Unraid container rather than being deployed fresh, so +`src/authentik/ansible/unraid/` is a README explaining why there's no compose +file there. The cutover procedure is `docs/authentik-migration.md`; the part +worth knowing without reading it is that the deployed `AUTHENTIK_VERSION` +must match the version a restored database was dumped from, because +Authentik's migrations run on startup and are one-way. diff --git a/README.md b/README.md new file mode 100644 index 0000000..0fcab00 --- /dev/null +++ b/README.md @@ -0,0 +1,966 @@ +# Homelab IaC + +Deploy apps as code across two platforms: Unraid server(s), where apps run as +Docker Compose stacks, and Proxmox, where Terraform creates an LXC and the +app is installed into it natively. Ansible is the deployment tool on both. +Secrets are pulled from HashiCorp Vault at deploy time; CI/CD is Forgejo +Actions once Forgejo itself is running. + +There's also a third piece of infrastructure alongside those two: a bare-metal +k3s cluster (`homelab-utils`) across 4 Raspberry Pis, bootstrapped by Ansible +too. It doesn't fit the `src//` table above the way Unraid/Proxmox apps +do — Terraform never provisions it, and no `stacks:`/`apps:` list drives +what's on it — so it gets its own inventory group and role instead. It does +run shared services deployed via Helm, though: `k3s_monitoring`, `k3s_metallb`, +and the cluster's own shared Postgres (`k3s_postgres`, a third platform for +`src/shared/postgres/`). See `playbooks/k3s.yml` and "K3s (Raspberry Pi)" +below. + +See `CLAUDE.md` for the short version of the architecture decisions. + +## Layout + +``` +src/ + / + common/vars.yml config that holds on either platform + ansible/ + unraid/ docker-compose.yml + .env.example + platform overrides + (+ optional icon.png for the Unraid Docker page) + proxmox/ install.yml + config/systemd templates + platform overrides + kubernetes/ vars.yml + *.yaml.j2 manifests (normally a HelmChart CR) + terraform/ this app's LXC definition (Proxmox only) + shared// same shape, for services multiple apps use +build/config/ + ansible/ + inventory/ + hosts.yml static: unraid_servers, proxmox_nodes, k3s_cluster + proxmox.yml dynamic: proxmox_guests, from the Proxmox API + group_vars/all.yml Vault connection, Compose Manager project path + group_vars/k3s_*.yml k3s cluster name/version, per-role vars (server/agent) + host_vars/.yml which stacks/apps deploy there + (k3s apps are in group_vars/k3s_cluster.yml instead) + roles/compose_stack/ Unraid: sync compose + render .env + provision DB + up + (deploy.yml / remove.yml, picked by the entry's state:) + roles/lxc_app/ Proxmox: install natively + render config + systemd + roles/k3s_app/ k3s: render app manifests + Secret + provision DB + (deploy.yml / remove.yml, picked by the entry's state:) + roles/pve_backup/ Proxmox node: NAS backup storage + vzdump schedule + roles/k3s_node/ k3s Pi: cgroups/swap prep + install server or agent + roles/k3s_monitoring/ k3s control plane: lean kube-prometheus-stack via HelmChart CR + roles/k3s_metallb/ k3s control plane: MetalLB (LoadBalancer IPs) via HelmChart CR + roles/k3s_postgres/ k3s control plane: shared Postgres (CloudNativePG) via HelmChart CR + roles/k3s_cert_manager/ k3s control plane: cert-manager + ClusterIssuer via HelmChart CR + roles/unattended_upgrades/ any apt host: hands-off patching, no auto-reboot + roles/k3s_maintenance/ k3s Pi: drain + reboot-if-required + uncordon, one at a time + playbooks/ + deploy.yml everything (Unraid + Proxmox — not k3s, see below) + unraid.yml just Unraid + proxmox.yml just the Proxmox guests + pve_host.yml just the Proxmox node (backups) + k3s.yml bootstraps the k3s cluster (manual-only, not in deploy.yml) + k3s_maintenance.yml rolls pending reboots across it, serial: 1 (manual-only) + terraform/ Proxmox provider, pg state backend, LXC module calls +.forgejo/workflows/ CI/CD (needs a self-hosted runner, see below) +docs/vault-secrets.md Vault path/key reference +docs/authentik-migration.md hand-run cutover: Authentik from Unraid to k3s +``` + +Terraform's scope stops at the guest — it creates the LXC and nothing more, +because the Proxmox provider has no way to install an application. Ansible +picks up from there, which is why it's the common tool and Terraform is +Proxmox-only. + +## How config is layered + +An app's settings are split by whether they survive a change of platform: + +- `src//common/vars.yml` — version, ports, domain, database name/user. +- `src//ansible//vars.yml` — the rest. Data paths differ + (`/mnt/user/appdata/forgejo` vs `/var/lib/forgejo`), and so does how the + database is reached (the `shared-postgres` Docker network alias only exists + on Unraid). + +Both roles merge them the same way: common, then platform overrides, then +Vault secrets last. Nothing is restated between platforms. + +## How a deploy works + +**Unraid.** For each stack in a host's `stacks:` list, `compose_stack`: + +1. Copies `src//ansible/unraid/docker-compose.yml` to the host's + Compose Manager project folder, unmodified — it's static and only + references `${VAR}`. +2. Merges the two `vars.yml` layers with the stack's Vault KV path and + renders the project's `.env`. Nothing secret ever touches git. +3. If the stack declares a `db:` block, ensures its database/role exist on + the shared Postgres instance (idempotent — safe to add new apps to the + shared service over time without touching existing data). +4. Copies `icon.png`, if the stack ships one, to both the project folder and + `/mnt/user/appdata/icons/` (see "Icons on the Unraid Docker page" below). +5. Runs `docker compose up -d` for the stack. + +**Proxmox.** `terraform apply` creates the LXC first. Then, for each app in a +guest's `apps:` list, `lxc_app` does the same config merge and database +provisioning, runs that app's `ansible/proxmox/install.yml`, and manages its +systemd unit instead of a Compose project. + +**k3s.** For each app in `k3s_apps:` (`group_vars/k3s_cluster.yml`, not +host_vars — an app belongs to the cluster, not a node), `k3s_app`: + +1. Does the same two-layer config merge, using + `src//ansible/kubernetes/vars.yml` as the platform layer. +2. Provisions the app's database/role on the CloudNativePG cluster if it + declares a `db:` block — the same `community.postgresql` tasks the other + two platforms use, pointed at MetalLB's LoadBalancer address for the CNPG + primary. +3. Renders the app's Vault path into a Kubernetes `Secret` (plus the + `Namespace`), keys passed through verbatim. +4. Renders every `*.yaml.j2` the app ships into + `/var/lib/rancher/k3s/server/manifests/`, where k3s's own deploy and helm + controllers pick them up. + +Nothing here talks to the Kubernetes API — no `helm`, no kubeconfig, no +`kubernetes.core`. Ansible writes files; k3s reconciles them. The consequence +worth remembering: **a green run means the manifests landed, not that the +workload came up.** Check with `kubectl -n get pods`. + +Secrets land on disk in the rendered `.env` (root-readable, on the Unraid +box) or, on k3s, in a 0600 manifest and then in etcd as an ordinary +Kubernetes Secret — this is "fetch from Vault at deploy time," not a +zero-secrets-at-rest model. Fine for a homelab; revisit if that changes. + +## Icons on the Unraid Docker page + +There are two icons per stack, and they come from completely different places. +Commit **one** `src//ansible/unraid/icon.png` and the deploy feeds both. +A stack running more than one container can commit +`icon-.png` alongside it for the extra services — see "More than one +container" below. + +### The stack row + +Compose Manager draws the collapsible header row for the whole project, and +serves its icon straight off disk from `icon.png` in the project folder — +`/boot/config/plugins/compose.manager/projects//icon.png`, the same +directory the compose file and rendered `.env` land in. No label, no +template, no URL: the file is there or the row has no icon. `.jpg`, `.gif`, +`.svg` and an extensionless `icon` also work, since the plugin serves the file +itself rather than handing a path to the webgui — but stick to PNG so the same +file can serve the labels below. + +This is a feature of **Compose Manager Plus** (`mstrhakr/compose_plugin`), the +maintained fork. The original Docker Compose Manager plugin (`dcflachs`, +archived April 2026) has no icon support at all — on that one the copied file +is simply ignored, so the deploy is safe either way, you just won't see it. + +### The containers under it + +Unraid's Docker page gets each *container's* icon, WebUI link and console +shell out of the dockerMan template that created it, under +`/boot/config/plugins/dockerMan/templates-user/`. A Compose stack never +creates one, which is why compose-deployed containers show a question mark and +no WebUI entry. Unraid 6.10+ falls back to Docker labels when there's no +template, so each service sets them itself: + +```yaml +labels: + net.unraid.docker.icon: ${STACK_ICON} + net.unraid.docker.webui: "http://[IP]:${FORGEJO_HTTP_PORT}/" + net.unraid.docker.shell: sh +``` + +They're per **service**, not per stack — a stack with two containers labels +each one separately, and can point them at different icons. + +- **icon** — `${STACK_ICON}` is set by the role, not by hand. The same + committed `icon.png` is copied a second time to + `/mnt/user/appdata/icons/.png` and the label points there; ship no + `icon.png` and the variable renders empty, leaving the placeholder. To use a + hosted image instead, set `STACK_ICON` in the app's + `ansible/unraid/vars.yml` — it's merged over the role's value. + + A second copy rather than reusing the project-folder one because this label + is a path the *webgui* resolves on every Docker page load, and that path + shouldn't run through the flash drive. + + **PNG only here** — unlike the stack row, this goes through the webgui's own + icon handling, where SVG renders as the question-mark fallback and WebP + renders nothing at all. A remote URL is fetched when the page renders, so an + unreachable host breaks the icon — the reason a committed file is the + default. + +- **webui** — `[IP]` is substituted with the host's address. The port must be + the *published* one, so interpolate the stack's own port variable rather + than the container port. Omit the label entirely for something with no web + interface (as `shared/postgres` does) — an empty value still draws a WebUI + entry that goes nowhere. + +- **shell** — `sh` or `bash`, whichever the image actually has. Alpine-based + images (Forgejo) need `sh`; Debian-based ones (Postgres, Shelfarr) can take + `bash`. + +One caveat: editing a compose container's labels through the Unraid UI writes +a template for it, and the template then wins over the labels until you clear +it. Change these in the compose file and redeploy, not in the webgui. + +### More than one container + +`${STACK_ICON}` is a single value — the stack's `icon.png` — so a stack with +two services would otherwise have to give both the same image. Since the +labels are per-service, commit `icon-.png` next to `icon.png` for +each additional container, named for the **compose service**: + +``` +src/arr/ansible/unraid/ + icon.png → ${STACK_ICON} → /mnt/user/appdata/icons/arr.png + icon-prowlarr.png → ${STACK_ICON_PROWLARR} → /mnt/user/appdata/icons/arr-prowlarr.png +``` + +The variable is the service name uppercased with `-` folded to `_` +(`icon-shelfarr-libation.png` → `${STACK_ICON_SHELFARR_LIBATION}`). These get +the appdata copy only — there is one Compose Manager stack row and `icon.png` +already has it — and, like `STACK_ICON`, each can be overridden with a hosted +URL by setting the same name in the app's `ansible/unraid/vars.yml`. + +The variable only exists when the file does. Interpolating +`${STACK_ICON_FOO}` with no `icon-foo.png` committed gets Compose's +unset-variable warning and the question-mark placeholder — so if the icon is +meant to be a URL, set it in `vars.yml` rather than relying on the file. + +## Backups (Proxmox) + +Proxmox guests keep **all** their state on their own rootfs. Nothing is +bind-mounted in from the NAS, which is the opposite of the obvious instinct — +the reason is that `vzdump` deliberately excludes bind mounts, so a +bind-mounted repository directory would be the one thing missing from the +backup that was made to protect it. Keeping everything local means one archive +is a complete copy of the guest's filesystem. + +`playbooks/pve_host.yml` (role: `pve_backup`) configures the off-box copy on +the node: an NFS storage pointing at the NAS, and a scheduled `vzdump` job +writing to it. Retention is the storage's `prune-backups` settings. Point it at +a Proxmox Backup Server instead by setting `pve_backup_storage_type: pbs` in +`host_vars/.yml`. + +**Data that lives outside a container is that app's problem, not vzdump's.** +Forgejo's database is on the shared Postgres over on Unraid, so restoring its +container alone would give back every repository with no issues, pull requests, +users or permissions. The fix is a `pg_dump` on a systemd timer *inside* the +container, writing into its own filesystem — which puts the dump inside the +same archive as the repositories it belongs to. That only works if it finishes +before the backup window, so the two schedules are a pair: + +| | Set in | Default | +|---|---|---| +| Forgejo database dump | `src/forgejo/ansible/proxmox/vars.yml` → `FORGEJO_DB_DUMP_ONCALENDAR` | 01:30 | +| vzdump job | `inventory/host_vars/pve.yml` → `pve_backup_schedule` | 02:00 | + +Any future app with external state should follow the same pattern. Restore +steps are in `src/forgejo/ansible/proxmox/README.md`. + +None of this covers the Unraid side, which has its own backup arrangements +outside this repo. + +## Prerequisites + +### Both platforms + +- Ansible control node needs to be Linux — **on Windows, run it from WSL2**, + not natively. +- If this repo is checked out on a Windows mount (e.g. under `/mnt/c/...`, + which is where a Seafile-synced folder ends up) rather than the native WSL + filesystem, Ansible will warn that `ansible.cfg` is being ignored because + the directory looks world-writable — that's DrvFs reporting `777` on + everything, not a real permissions problem. Either `export + ANSIBLE_CONFIG="$(pwd)/ansible.cfg"` before running (from the + `build/config/ansible/` dir), or fix it permanently by adding to + `/etc/wsl.conf` inside WSL: + ```ini + [automount] + options = "metadata,umask=22,fmask=11" + ``` + then `wsl --shutdown` from PowerShell and reopen the shell. +- `pip install ansible hvac psycopg2-binary` +- `ansible-galaxy collection install -r build/config/ansible/requirements.yml` +- A reachable Vault instance, with secrets populated per + `docs/vault-secrets.md`. + +### Unraid + +- Root SSH enabled on the Unraid box(es) (Settings → Management Access). +- Compose Manager (Docker Compose plugin) installed on Unraid — already + done. +- **Python3 on each Unraid host itself**, not just the controller: the + `compose_stack` role's tasks (`file`, `copy`, `template`, + `docker_compose_v2`) run on the target, and Ansible modules — unlike + `raw`/ad hoc shell — need a Python interpreter there to execute at all. + Stock Unraid doesn't ship one. Installing it by hand into the running + system doesn't survive a reboot, same reason as the SSH-key gotcha above — + Unraid rebuilds its root filesystem from the flash drive on every boot. + Install the **NerdTools** plugin (Community Applications → search + "NerdTools", formerly "NerdPack") and enable `python3` in its package + list — it reinstalls whatever you've selected on every boot itself, same + mechanism Compose Manager already relies on. `ansible.cfg` already sets + `interpreter_python = auto_silent`, so Ansible finds it automatically + wherever NerdTools puts it; no path to hardcode. Verify after installing: + ```sh + ansible unraid_servers -m ping + ``` + +### Proxmox + +- Terraform CLI (>= 1.6) on whatever runs `terraform apply`. +- A Proxmox API token with rights to create containers, stored in Vault at + `homelab/ci/proxmox` (see `docs/vault-secrets.md`). Both the Terraform + provider and Ansible's dynamic inventory authenticate with it. +- A `terraform_state` database and `terraform` role on the **CloudNativePG + cluster on k3s** (192.168.50.81), for `backend "pg"` — not on either shared + Postgres this repo deploys, which would be circular. See "Bootstrapping" + below. +- Container templates downloaded on the Proxmox node for whatever OS the + LXCs are built from. +- An NFS export on the NAS for backup archives, reachable from the Proxmox + node — set it in `inventory/host_vars/pve.yml`. See "Backups (Proxmox)". +- Root SSH from the Ansible controller to the Proxmox node itself, not just + the guests: `playbooks/pve_host.yml` configures the node over SSH. +- Unlike Unraid, nothing special is needed *inside* the guests: they're + ordinary Linux containers with Python already present, so Ansible works + without the NerdTools workaround above. + +### K3s (Raspberry Pi) + +- 4 Raspberry Pis running Ubuntu Server (64-bit), already imaged and on the + network at the addresses in `inventory/hosts.yml` → `k3s_cluster`. +- An `ansible` user on each Pi with **NOPASSWD sudo** and this repo's k3s SSH + key installed as an authorized key — see "SSH access" below. Not automated + by this repo; create it by hand (or via cloud-init at image time) before the + first run of `playbooks/k3s.yml`. +- `homelab/k3s-homelab-utils` → `K3S_TOKEN` set in Vault *before* the first + run — see `docs/vault-secrets.md`. Both the server and every agent read + this same fixed value rather than one generating it and handing it to the + other, which is what makes a rebuild reproducible. +- Nothing else special: `python3` ships with Ubuntu Server, and the k3s + install script (`get.k3s.io`) handles containerd, the systemd unit, and + everything else that isn't Pi-specific. The one Pi-specific thing — + ensuring the memory cgroup controller is on — is handled by + `roles/k3s_node` itself, not a prerequisite here. + +#### Keeping it patched + +`playbooks/k3s.yml` also applies `roles/unattended_upgrades` to every node — +apt updates install themselves on their own daily schedule, no login +required. It sets `Unattended-Upgrade::Automatic-Reboot "false"` though, so a +kernel or containerd update that needs a reboot to take effect just sits +applied-but-inactive until one happens; blindly auto-rebooting a k3s node +takes its pods down with no warning. + +`playbooks/k3s_maintenance.yml` is the other half: it checks +`/var/run/reboot-required` on each node and, only where it's set, cordons and +drains the node, reboots it, waits for it to report `Ready` again, then +uncordons it — one node at a time (`serial: 1`), so patching the cluster never +means dropping every workload at once. Manual-only for now, same as +`playbooks/k3s.yml`: + +```sh +ansible-playbook playbooks/k3s_maintenance.yml +``` + +Eventually this is meant to run on a schedule from a self-hosted Forgejo +Actions runner rather than by hand — not wired up yet, see the header comment +in the playbook. + +#### Metrics (Prometheus) + +`playbooks/k3s.yml` also deploys `roles/k3s_monitoring`: a lean +kube-prometheus-stack (Prometheus + prometheus-operator + node-exporter + +kube-state-metrics — no Grafana, no Alertmanager) so tools like OpenLens can +show node/pod metrics. It's installed as a `HelmChart` custom resource +dropped into k3s's own auto-deploying manifests directory +(`/var/lib/rancher/k3s/server/manifests/`) rather than run through a `helm` +binary — k3s's bundled helm-controller reconciles it the same way it +installs its own Traefik and ServiceLB, so this needs no extra Ansible +collection or kubeconfig on the controller. See +`roles/k3s_monitoring/defaults/main.yml` for the resource sizing (tuned for +a Raspberry Pi 4, not a real node) and why the control-plane component +scrapers (`kubeControllerManager`, `kubeScheduler`, `kubeProxy`, `kubeEtcd`) +are disabled — k3s doesn't expose those the way kube-prometheus-stack +expects, so leaving them on just produces permanently-"down" targets. + +To point OpenLens at it: open the cluster's Settings → Metrics, set +Prometheus to "Auto detect" or explicitly to the `Operator` provider — it +should find the `kube-prometheus-stack-prometheus` service in the +`monitoring` namespace via the API server proxy, the same path OpenLens +already uses to reach the cluster, so nothing needs to be exposed outside +it. + +#### LoadBalancer IPs (MetalLB) + +`playbooks/k3s.yml` also deploys `roles/k3s_metallb`: MetalLB in L2 mode, +handing out real LAN IPs to `type: LoadBalancer` Services instead of the +`ClusterIP`-only world k3s would otherwise leave homelab-utils apps in. It's +installed the same way as monitoring — a `HelmChart` CR for k3s's +helm-controller to reconcile — plus a plain `IPAddressPool`/`L2Advertisement` +manifest dropped in the same directory; k3s's deploy controller applies both +kinds of file and retries the config manifest until the HelmChart's CRDs +exist. + +MetalLB replaces k3s's bundled ServiceLB (Klipper), it doesn't run alongside +it — both would otherwise compete to satisfy the same Services. That's why +`inventory/group_vars/k3s_cluster.yml` sets `k3s_server_extra_args: +['--disable=servicelb']` — a separate, server-only var from `k3s_extra_args`, +since `k3s agent` doesn't understand `--disable` and would fail to start if +it were passed there too. The IP pool itself is +`k3s_metallb_address_range` in the same file — a range on the cluster's LAN +(`192.168.50.0/24`) that DHCP and every static assignment in +`inventory/hosts.yml` steer clear of. There's no built-in default: the role +fails fast if it's still empty. + +Enabling `--disable=servicelb` on an already-running cluster doesn't need a +separate step: `roles/k3s_node` notices its exec line has changed and +reinstalls (restarting just the `k3s` service, not the node) the next time +`playbooks/k3s.yml` runs — see `CLAUDE.md` → "Key decisions". + +#### Postgres (CloudNativePG) + +`playbooks/k3s.yml` also deploys `roles/k3s_postgres`: CloudNativePG (CNPG), +a Postgres operator, installed the same way as monitoring and MetalLB — a +`HelmChart` CR for k3s's helm-controller to reconcile — plus a plain +`Cluster` CR (the actual database) and a `Secret` (superuser credentials), +dropped in the same auto-deploying manifests directory. This is a third +platform for `src/shared/postgres/`, alongside the Unraid Compose stack and +the Proxmox placeholder: `roles/k3s_postgres` layers +`src/shared/postgres/common/vars.yml` with +`src/shared/postgres/ansible/kubernetes/vars.yml` the same way +`compose_stack`/`lxc_app` layer an app's config, and pulls the superuser +password from the same `homelab/shared/postgres` Vault path those platforms +already use (see `docs/vault-secrets.md`) — a separate physical instance, +not a shared login across platforms, just the same identity. + +1 primary + 1 replica, kept off the control-plane Pi via `nodeAffinity` (see +`roles/k3s_postgres/templates/postgres-cluster.yaml.j2`) — CNPG's own pod +anti-affinity then spreads the two across the 3 worker Pis. Storage is k3s's +default `local-path` StorageClass; that's node-local, but resilience here +comes from CNPG's own streaming replication between instances, not from +shared storage, so losing one instance's disk doesn't lose the data as long +as the other instance is up. HA replication only for now — no +`ScheduledBackup` to the NAS or anywhere else yet. + +To connect: `kubectl -n shared-postgres get svc` lists four Services. Three +are CNPG's own, all `ClusterIP` — `shared-postgres-rw` (the current primary), +`-ro` (replicas only) and `-r` (any instance). An app running on the cluster +uses the first of those by DNS and needs nothing else: + +``` +shared-postgres-rw.shared-postgres.svc.cluster.local:5432 +``` + +The fourth, `shared-postgres-lb`, is a `LoadBalancer` on +`k3s_postgres_loadbalancer_ip` (`inventory/group_vars/k3s_cluster.yml` — +inside the MetalLB pool above), which is how the cluster is reached from +outside it: + +```sh +psql -h 192.168.50.81 -U postgres # password: homelab/shared/postgres +``` + +It's declared in the `Cluster` CR as a CNPG *managed service* rather than as +a Service manifest of our own, so the operator keeps its selector pointed at +whichever instance is currently primary — a hand-written Service would need +re-pointing by hand after a failover. Credentials are the +`shared-postgres-superuser-vault` Secret Ansible renders; CNPG separately +auto-creates `shared-postgres-app` for the default `app` database it +bootstraps, which nothing uses yet. + +The `-vault` suffix keeps that Secret clear of `-superuser`, the +name CNPG uses for the superuser secret it generates itself. Name our own +object that and the operator treats it as one it already authored, so a +rotated password reaches the Secret and never reaches the database — the +Secret reads correctly, `psql` from the LAN keeps failing authentication, and +only `select rolpassword is null from pg_authid where rolname='postgres'` +inside the pod shows the disagreement. + +That LoadBalancer is what makes the controller-reachable `host:port` the +`community.postgresql` tasks need — the same ones `compose_stack` and +`lxc_app` use to create an app's database and role — available here too, and +`roles/k3s_app` now uses exactly those tasks for any app declaring a `db:` +block. + +Note the deliberate asymmetry, which reads like a mistake until you know why: +provisioning uses `192.168.50.81` (the LoadBalancer) because it runs on the +Ansible controller, which is off-cluster and can't route to a ClusterIP; +apps use `shared-postgres-rw.shared-postgres.svc.cluster.local` because +sending pod traffic out to the LAN and back would put MetalLB's L2 speaker in +the path of every query for nothing. Two addresses, one database, each +correct for its caller. + +Without `kubectl` to hand, or if MetalLB is having a bad day, the in-cluster +routes still work: + +```sh +kubectl -n shared-postgres port-forward svc/shared-postgres-rw 5432:5432 +kubectl -n shared-postgres exec -it shared-postgres-1 -- psql -U postgres +``` + +**No backups.** HA replication only — a streaming replica on another Pi, +which covers a dead SD card and not a dropped table, a bad migration, or a +mistyped `DROP`. This mattered less when the cluster held nothing; it matters +now that Authentik's database lives here rather than on Unraid. A CNPG +`ScheduledBackup` to the NAS is the obvious next piece of work and isn't +built yet. + +#### TLS (cert-manager) + +`playbooks/k3s.yml` also applies `roles/k3s_cert_manager`: cert-manager plus +one `ClusterIssuer` named `letsencrypt`, both via the usual `HelmChart` CR + +plain manifest pair. An app's Ingress opts in with a single annotation — + +```yaml +annotations: + cert-manager.io/cluster-issuer: letsencrypt +``` + +— and cert-manager creates and renews the certificate on its own. + +Two values have no default and the role refuses to run without them, both in +`inventory/group_vars/k3s_cluster.yml`: + +- `k3s_cert_manager_acme_email` — an ACME account is registered against it. +- `k3s_cert_manager_solver` — the DNS-01 stanza for your DNS provider, + rendered into the issuer as-is. A commented Cloudflare example is in that + file; for anything else take the stanza from + [cert-manager's docs](https://cert-manager.io/docs/configuration/acme/dns01/). + Its API token goes in Vault at `homelab/k3s-cert-manager`. + +DNS-01 rather than HTTP-01 because HTTP-01 needs Let's Encrypt to reach this +cluster from the internet on port 80, which it can't. That has a useful +consequence: a certificate can be issued **before** DNS points at the +cluster, so a migration's TLS is settled before its cutover rather than +after. + +While working out a solver, point `k3s_cert_manager_acme_server` at Let's +Encrypt staging — production allows five failed validations per hostname per +hour, and exhausting it means waiting, not retrying. Check it registered: + +```sh +kubectl get clusterissuer letsencrypt -o jsonpath='{.status.conditions[*].message}' +``` + +#### Dashboard (Traefik) + +Traefik is the one chart here this repo doesn't install — k3s installs it +itself. `roles/k3s_traefik` only adjusts it, through a `HelmChartConfig` +merged over k3s's own `HelmChart`. Editing +`/var/lib/rancher/k3s/server/manifests/traefik.yaml` on the node instead +works until the next server restart rewrites it. + +A stock k3s answers **404** on the dashboard, which looks like a broken +install and isn't: Traefik still builds the dashboard, but the Traefik chart +stopped creating the router that reaches it in v28. The role puts that router +back on Traefik's internal `traefik` entrypoint, which isn't published on the +Service — so it's reachable by port-forward and nothing else: + +```sh +kubectl -n kube-system port-forward deploy/traefik 9000:9000 +# then http://127.0.0.1:9000/dashboard/ — the trailing slash is required +``` + +Two things to check if that still 404s. `curl -i http://127.0.0.1:9000/ping` +returning 200 means the port-forward is fine and only the router is missing +(so the role hasn't run, or its `HelmChartConfig` hasn't reconciled yet — +`kubectl -n kube-system get ingressroute` should list `traefik-dashboard`). +And forward to `deploy/traefik`, not `svc/traefik`: the Service only publishes +80/443, so going through it lands you on the `web` entrypoint, where an +unmatched request also returns 404. + +**Publishing it on a hostname** is one commented line in +`inventory/group_vars/k3s_cluster.yml`: + +```yaml +k3s_traefik_dashboard_host: traefik.turtlesystems.uk +``` + +That renders an `IngressRoute` on `websecure` with a cert-manager certificate +and an Authentik forward-auth middleware in front. `k3s_traefik_dashboard_auth_address` +must be set too — the role refuses to run otherwise rather than publishing an +unauthenticated view of every router, service and middleware on the cluster. +It's already set in that file, so in practice this is the single line above +plus a DNS record pointing at Traefik's MetalLB address +(`kubectl -n kube-system get svc traefik`). + +On the Authentik side, first: a Proxy Provider in **forward auth (domain +level)** mode, assigned to an application, added to the embedded outpost. +Domain level rather than single-application because the latter needs +`/outpost.goauthentik.io/` routed to Authentik on the dashboard's own +hostname, which from `kube-system` is a cross-namespace service reference +Traefik rejects by default. + +An `IngressRoute` rather than an `Ingress`, unlike every app here, because +the dashboard is served by Traefik's internal `api@internal` and has no +Kubernetes Service for an `Ingress` to point at. That's also why its +certificate is an explicit `Certificate` resource — cert-manager watches +`Ingress`, not `IngressRoute`. + +Clearing the hostname again is a real teardown: the role deletes the +manifest, and k3s's deploy controller garbage-collects the route, middleware +and certificate it created. + +The port-forward stays enabled alongside the published route on purpose. Once +the dashboard is behind Authentik it depends on Authentik, which depends on +CNPG, which depends on a healthy cluster — the things you'd open the +dashboard to diagnose. Port-forward talks to the pod and traverses none of +it, so it's the break-glass path, not a leftover. + +#### Apps on the cluster + +Apps go in `k3s_apps:` in `inventory/group_vars/k3s_cluster.yml` — the k3s +equivalent of a host's `stacks:`/`apps:` list, in group_vars because an app +is deployed to the cluster rather than to a node. Same entry shape as the +other platforms (name, src, vault_path, optional `db:`, optional `state:`). + +```sh +ansible-playbook playbooks/k3s.yml --tags apps +ansible-playbook playbooks/k3s.yml --tags apps -e only_apps=authentik +``` + +Ingress goes through the Traefik k3s already bundles, on the MetalLB address +its Service holds (`kubectl -n kube-system get svc traefik`) — so adding an +app costs a DNS record pointed at that one address, not a pool IP each. + +Removal is `state: absent` on the entry, same tombstone rule as Unraid: keep +the entry, don't delete it, or nothing is torn down and the workload keeps +running unmanaged. Unlike Unraid it's a single pass — deleting the manifests +*is* the teardown, because k3s's deploy controller garbage-collects what each +file created — so `src//` can go in the same commit. The database, PVCs +and Vault path deliberately survive. + +**Authentik** is currently the only app, and it got here by migrating off a +hand-made Unraid container rather than being deployed fresh. If you're +repeating that for something else, `docs/authentik-migration.md` is the +worked procedure; the part worth knowing up front is that the deployed +version must match the version a restored database was dumped from, because +Authentik runs its migrations on startup and they don't go backwards. + +## SSH access + +Ansible connects as `ansible_user: root` (set in +`build/config/ansible/inventory/hosts.yml`) over SSH. Password auth would mean either an interactive prompt every run — +which doesn't work from CI at all — or a plaintext password somewhere, so +use a dedicated key pair instead: + +```sh +ssh-keygen -t ed25519 -f ~/.ssh/unraid_ansible -C "ansible@unraid-iac" -N "" +``` + +Install the public half as an authorized key on **every** Unraid host this +repo targets. Don't just `ssh-copy-id` it into `~/.ssh/authorized_keys` — +Unraid boots from the flash drive and `/root` lives on a RAM-backed overlay, +so anything written there is gone on the next reboot. Persist it through +`/boot/config/ssh/root.authorized_keys` instead, which Unraid copies into +place at every boot: + +```sh +cat ~/.ssh/unraid_ansible.pub | ssh root@ \ + 'cat >> /boot/config/ssh/root.authorized_keys' +``` + +For manual/bootstrap runs, load the key into `ssh-agent` and Ansible picks it +up automatically — no config file changes needed: + +```sh +eval "$(ssh-agent -s)" +ssh-add ~/.ssh/unraid_ansible +``` + +For CI, rather than adding yet another place secrets live, the private key +is stored in Vault (`homelab/ci/ssh`, see `docs/vault-secrets.md`) and the +workflow fetches it at the start of each run: + +```sh +vault kv put kv/homelab/ci/ssh PRIVATE_KEY=@~/.ssh/unraid_ansible +``` + +**K3s Pis** work the same way but as a non-root `ansible` user, not root, and +with their own key pair (`~/.ssh/k3s_ansible`, Vault path +`homelab/ci/ssh-k3s`) — see `docs/vault-secrets.md` for the full commands. +Three differences from the Unraid steps above. The public key goes in the +ordinary `~/.ssh/authorized_keys` for the `ansible` user (Ubuntu's root +filesystem isn't rebuilt from flash on every boot, so there's no persistence +quirk to work around), and that user needs NOPASSWD sudo configured in +`/etc/sudoers.d/` — `playbooks/k3s.yml` installs k3s via `become: true`. + +And the private half doesn't need loading into `ssh-agent` at all, even for a +manual run: `playbooks/k3s_ssh_key.yml` fetches it from Vault and writes it to +`local/k3s/homelab-utils.key`, and both `playbooks/k3s.yml` and +`playbooks/k3s_maintenance.yml` import that as their first play. So a k3s run +needs `VAULT_ADDR`/`VAULT_TOKEN` in the environment and nothing else — the +same "Vault is the only place secrets live" arrangement CI already uses for +the Unraid key, rather than a second setup step to remember. To fall back to a +local copy (Vault down, say), skip the fetch **and** point at the key — +overriding the path alone isn't enough, since the fetch play would still fail +before anything else ran: + +```sh +ansible-playbook playbooks/k3s.yml \ + -e k3s_ssh_key_fetch=false \ + -e ansible_ssh_private_key_file=~/.ssh/k3s_ansible +``` + +## Bootstrapping (chicken-and-egg on the very first deploy) + +CI/CD runs on Forgejo Actions — but Forgejo doesn't exist yet on the first +run, and Actions needs a self-hosted runner besides. So the first deploy is +manual: + +1. Fill in `build/config/ansible/inventory/hosts.yml` with nas2's real + address. +2. Set up the SSH key pair per "SSH access" above and install the public key + on nas2 (and nas1, once you migrate anything to it). +3. `vault kv put` the secrets in `docs/vault-secrets.md` for + `homelab/shared/postgres`, `homelab/forgejo`, and `homelab/ci/ssh`. +4. From WSL, with `VAULT_ADDR` / `VAULT_TOKEN` exported (`vault login` first) + and the key loaded in `ssh-agent`: + ```sh + cd build/config/ansible + ansible-playbook playbooks/unraid.yml --limit nas2 + ``` + This brings up shared Postgres, provisions the `forgejo` database/role, + and brings up Forgejo. Use `unraid.yml` rather than `deploy.yml` at this + point — no Proxmox guests exist yet, so the Proxmox play has nothing to + target. +5. Log into Forgejo, create this repo there, push it. +6. Register a self-hosted Forgejo Actions runner reachable to Vault and to + both Unraid hosts (labelled `unraid-deploy` — see + `.forgejo/workflows/deploy.yml`). Give it an AppRole (`VAULT_ROLE_ID` / + `VAULT_SECRET_ID`) scoped to read the `kv/homelab/*` paths it needs + (including `homelab/ci/ssh`), stored as Forgejo Actions secrets alongside + `VAULT_ADDR`. +7. From then on, pushes to `main` touching `src/**` or `build/config/ansible/**` + deploy automatically — and only the stack(s) whose `src//` folder + actually changed (see "Deploying one platform, one host, one stack" + below); a change under `build/config/ansible/` still triggers a full + deploy of every stack on every host, since that's a change to how *all* of + them get deployed. + +### Bringing Proxmox online later + +The shared Postgres and its LXC are the first thing to stand up here, and the +ordering is fiddly enough to have its own runbook: **`docs/postgres-proxmox.md`**. +The short version, and the reason it isn't just "run Terraform": + +Terraform stores its state in Postgres, so provisioning the Postgres LXC with +Terraform would be circular. It's broken by keeping state on the +CloudNativePG cluster on the k3s Pis instead — something this configuration +has no hand in building — which means `terraform apply` now depends on the +cluster being up. That's a cross-platform dependency the rest of the repo +avoids, and it's worth it because the alternative is a local-state-then-migrate +dance that has to be got right exactly once. + +After that, `docs/postgres-proxmox.md` walks through the template download, +`terraform apply -target=module.postgres`, checking the ZFS replication job +actually exists, installing Postgres with +`ansible-playbook playbooks/proxmox.yml -e only_stacks=postgres`, and +`playbooks/pve_host.yml` for node backups. It also covers failing over between +the two nodes and repointing clients afterwards. + +Forgejo comes after all of that, on **turtle-proxmox-02** — its database is +the shared Postgres the runbook above creates, so none of it works until that +one is finished. `docs/forgejo-proxmox.md` walks through the extra Vault key +the native install needs, `terraform apply -target=module.forgejo`, +`ansible-playbook playbooks/proxmox.yml -e only_stacks=forgejo`, and the parts +Ansible deliberately does not do — moving repository data, and the DNS/SSH-port +cutover. + +## Deploying one platform, one host, one stack + +`playbooks/deploy.yml` runs everything; `playbooks/unraid.yml`, +`playbooks/proxmox.yml` and `playbooks/pve_host.yml` do one slice each +(Unraid stacks, Proxmox guests, and the Proxmox node's backup config +respectively). Within any of them, `--limit ` +narrows by host and `-e only_stacks=[,...]` narrows by stack or +app (name = the `name:` in the host's `stacks:`/`apps:` list, e.g. `forgejo` +or `postgres` — not the `src:` path): + +```sh +# everything, both platforms +ansible-playbook playbooks/deploy.yml + +# everything declared for nas2 +ansible-playbook playbooks/unraid.yml --limit nas2 + +# just forgejo on nas2 +ansible-playbook playbooks/unraid.yml --limit nas2 -e only_stacks=forgejo + +# forgejo and its database, skipping any other stack on nas2 +ansible-playbook playbooks/unraid.yml --limit nas2 -e only_stacks=forgejo,postgres +``` + +Note `only_stacks` doesn't resolve dependencies for you — if you limit to +`forgejo` before `postgres` has ever been deployed, the DB provisioning step +will fail because there's nothing to connect to. CI doesn't hit this: it +computes `only_stacks` from which `src/**` paths actually changed in the +push, so an unrelated app's push never touches Postgres or Forgejo. + +`deploy.yml` runs Unraid before Proxmox by convention rather than necessity — +the two platforms no longer depend on each other. Proxmox-side apps provision +their databases against the Proxmox shared Postgres (192.168.50.54), and +Terraform keeps its state on the CloudNativePG cluster on k3s; neither goes +through Unraid. The node's backup config goes last, since a vzdump job pinned +to specific VMIDs needs those guests to exist. Note that `only_stacks` has no +meaning for `pve_host.yml` — it configures a node, not an app, and ignores it. + +`playbooks/k3s.yml` is separate from all of the above — it isn't included in +`deploy.yml` and doesn't take `only_stacks`, because a cluster service there +isn't a `stacks:` entry to filter down to. It has two selectors of its own +instead: `--limit` narrows *which nodes*, and `--tags` narrows *which piece*. + +```sh +# the whole cluster: nodes, then the cluster services, then the kubeconfig +ansible-playbook playbooks/k3s.yml + +# re-converge one worker, e.g. after reimaging it +ansible-playbook playbooks/k3s.yml --limit k3s-node-02 + +# just one cluster service, no node install across 4 Pis +ansible-playbook playbooks/k3s.yml --tags postgres + +# every cluster service, still no node install +ansible-playbook playbooks/k3s.yml --tags services +``` + +The tags, one per play (see the playbook's header comment): `nodes` (both +node plays), `metallb`, `monitoring`, `postgres`, `cert-manager`, `traefik`, +`services` (all five of those together), `apps`, `upgrades`, `kubeconfig`. +`services` and `apps` don't imply each other, so "everything except +reinstalling k3s" is `--tags services,apps`. The SSH-key fetch play is tagged +`always` rather than named, so it survives every filter — it's what any of +the others connect with, not something you'd pick. + +Deploying a single service this way skips the `kubeconfig` play too, so +`local/`'s copy isn't refreshed. That only matters on a cluster rebuild; +add `--tags postgres,kubeconfig` if you want both. + +Worth knowing what a tagged run does and doesn't tell you: every service role +just templates manifests into `/var/lib/rancher/k3s/server/manifests/` +for k3s's own controllers to reconcile, so the playbook finishing means the +files landed, not that the workload is up. Watch that separately, e.g. +`kubectl -n shared-postgres get cluster,pods -w`. + +`playbooks/k3s_maintenance.yml` is separate again — see "Keeping it patched" +above. It ignores `--limit` grouping in one sense worth knowing: `serial: 1` +still applies to whatever `--limit` narrows the run to, so limiting to two +nodes still patches them one at a time, not together. + +## Adding a new app + +1. `src//common/vars.yml` — `env_defaults:` for the config that doesn't + depend on where it runs. +2. Build the platform side(s) you need: + + **Unraid** — `src//ansible/unraid/` containing + `docker-compose.yml` (static, `${VAR}`-driven, joining `unraid_shared` + with `external: true` if it needs the shared Postgres), `vars.yml` of + overrides, and `.env.example` documenting every var with secrets blank. + Optionally an `icon.png` and the `net.unraid.docker.*` labels, so the + Docker page shows something other than a question mark. + + **Proxmox** — `src//ansible/proxmox/` containing `vars.yml`, + `install.yml`, and templates for the app's config file and systemd unit; + plus `src//terraform/` defining its LXC, called from + `build/config/terraform/main.tf`. + +3. Add its `homelab/` path to Vault (`docs/vault-secrets.md`). +4. Declare it on the target host — `stacks:` in an Unraid host's + `host_vars/.yml`, or `apps:` in a Proxmox guest's. Same entry shape + either way (name, src, vault_path, optional `db:` block). +5. Push — or run the relevant playbook manually before CI exists. +6. Leave the platform you didn't build as a README placeholder rather than + deleting the folder. + +## Removing an app (Unraid) + +**Deleting the entry from `stacks:` does not remove anything.** Ansible keeps +no record of what it deployed last run, so an app that disappears from the +list is simply never visited again — its containers keep running on the host, +now unmanaged and invisible to the playbook. This is the one place the Unraid +side doesn't behave like the Terraform side, where deleting a module call does +destroy the LXC. + +Removal is therefore an instruction, not an absence. Set `state: absent` on +the entry and leave it in place: + +```yaml +stacks: + - name: arr + src: arr + vault_path: homelab/arr + state: absent +``` + +The next run of `playbooks/unraid.yml` runs `docker compose down`, then +deletes the project folder (and with it the rendered `.env`). Because a +`host_vars/` edit is a change under `build/config/ansible/`, CI treats it as a +full converge — so pushing that change is enough to action the teardown. + +Keep the entry as a tombstone until you're sure. It's the only record that +the app was deliberately removed rather than never deployed, and flipping +`state:` back to `present` redeploys it. + +### What survives, and how to remove the rest + +Bind mounts are untouched, so `/mnt/user/appdata/` — which for a +SQLite-backed app like Shelfarr *is* the application — survives a teardown and +makes it reversible. External networks (`caddy-net`, `unraid_shared`) are left +alone too; they belong to Unraid or to another stack. Both copies of the +stack's icon *are* deleted without needing to be asked — the project-folder +one goes with the folder, and the one under `/mnt/user/appdata/icons/` is +removed explicitly. Neither is state; they're copies of a file in the repo, +and a redeploy puts them back. + +Three things are destructive enough to stay opt-in, per entry: + +| Key | Effect | +|---|---| +| `remove_volumes: true` | Also delete the project's named volumes. Bind mounts are unaffected either way. | +| `remove_images: local` | Also delete its images (`local` or `all`, as `docker compose down --rmi`). | +| `remove_database: true` | Drop the Postgres database **and** its role. Only for stacks with a `db:` block. | + +Nothing removes the app's `homelab/` path from Vault — that has its own +lifecycle, and destroying it would make the teardown irreversible. Delete it +by hand with `vault kv metadata delete` once you're finished with the app. + +### Order of operations + +`docker compose down` reads the compose file to know what it's removing, so +the teardown has to run **before** you delete `src//` or the host's +project folder. Delete those first and there's nothing left to tell Docker +what belonged to the project — you're cleaning up by hand with `docker rm` and +`docker network rm` instead. So: set `state: absent`, run the playbook, then +delete files in a second pass. + +There's no equivalent on the Proxmox side yet. `lxc_app` only ever installs, +and a symmetric teardown needs a per-app `uninstall.yml` convention that no +app implements — the role is still a skeleton. For now, removing a Proxmox app +means stopping and disabling its systemd unit by hand, or destroying the +container via Terraform. + +## Adding a host + +**Unraid:** add it under `unraid_servers` in +`build/config/ansible/inventory/hosts.yml`, then create +`build/config/ansible/inventory/host_vars/.yml` with its own `stacks:` +list — each host only runs what it's assigned. + +**Proxmox:** define the LXC as a module in `src//terraform/`, call it +from `build/config/terraform/main.tf`, and `terraform apply`. The guest +appears in inventory automatically via the API — no `hosts.yml` edit — but +it still needs a `host_vars/.yml` declaring its `apps:` list. + +**K3s:** add it as a new host under `k3s_control_plane` or `k3s_workers` (per +`inventory/hosts.yml` → `k3s_cluster`) — no `terraform apply`, the Pi has to +physically exist and have SSH access set up first (see "K3s (Raspberry Pi)" +above). No `host_vars/` entry needed beyond that: `roles/k3s_node` reads +`k3s_node_role` off the group, not the host, so which group a new Pi joins is +the only thing that decides whether it becomes another agent or a second +control-plane node — and this repo's role doesn't support the latter (single +server, no HA etcd) without changes to `k3s_node/tasks/server.yml`. diff --git a/build/config/ansible/ansible.cfg b/build/config/ansible/ansible.cfg new file mode 100644 index 0000000..927dbe1 --- /dev/null +++ b/build/config/ansible/ansible.cfg @@ -0,0 +1,18 @@ +[defaults] +# A directory, not a single file: static Unraid hosts (hosts.yml) and the +# Proxmox API-backed dynamic inventory (proxmox.yml) are both sources. +inventory = inventory/ +roles_path = roles +host_key_checking = False +retry_files_enabled = False +interpreter_python = auto_silent + +[inventory] +# `auto` must come before `yaml`: it reads the `plugin:` key in +# inventory/proxmox.yml and dispatches to the named plugin. With `yaml` +# first, that file gets parsed as a static host list instead, and `plugin:` +# is read as a group name. +enable_plugins = auto, yaml, ini, community.proxmox.proxmox + +[ssh_connection] +pipelining = True diff --git a/build/config/ansible/inventory/group_vars/all.yml b/build/config/ansible/inventory/group_vars/all.yml new file mode 100644 index 0000000..5b353b0 --- /dev/null +++ b/build/config/ansible/inventory/group_vars/all.yml @@ -0,0 +1,18 @@ +--- +# HashiCorp Vault connection. VAULT_ADDR/VAULT_TOKEN (or VAULT_ROLE_ID / +# VAULT_SECRET_ID for AppRole) are read from the environment of whatever is +# running ansible-playbook — a local shell during bootstrap, or Forgejo +# Actions secrets once CI is wired up. Never hardcode a token here. +vault_addr: "{{ lookup('env', 'VAULT_ADDR') }}" +vault_kv_mount: kv + +# AppRole auth (used by CI). Leave unset for local token-based auth +# (`vault login` + VAULT_TOKEN env var) during bootstrap. +vault_auth_method: "{{ lookup('env', 'VAULT_AUTH_METHOD') | default('token', true) }}" +vault_role_id: "{{ lookup('env', 'VAULT_ROLE_ID') | default(omit) }}" +vault_secret_id: "{{ lookup('env', 'VAULT_SECRET_ID') | default(omit) }}" + +# Where Compose Manager (the Unraid Docker Compose plugin) expects project +# folders. Verify this against your plugin version — override per-host in +# host_vars if yours differs. +compose_projects_root: /boot/config/plugins/compose.manager/projects diff --git a/build/config/ansible/inventory/group_vars/k3s_cluster.yml b/build/config/ansible/inventory/group_vars/k3s_cluster.yml new file mode 100644 index 0000000..56d9e14 --- /dev/null +++ b/build/config/ansible/inventory/group_vars/k3s_cluster.yml @@ -0,0 +1,205 @@ +--- +# Shared config for the whole k3s_cluster group (both k3s_control_plane and +# k3s_workers) — see roles/k3s_node and playbooks/k3s.yml. + +# Used as the kubeconfig context/cluster name and as the Vault path suffix +# below, so the cluster has one name in every place it shows up. +k3s_cluster_name: homelab-utils + +# Pinned, not "latest" — a rebuild months from now should produce the same +# cluster it does today, same reasoning as pinning an app's image tag in +# common/vars.yml. Bump deliberately; check the current stable release at +# https://github.com/k3s-io/k3s/releases first. +k3s_version: v1.31.4+k3s1 + +# One Vault path for the whole cluster, same "one path per stack" convention +# as everything under homelab/ — see docs/vault-secrets.md. +k3s_vault_path: "homelab/k3s-{{ k3s_cluster_name }}" + +# The SSH key this group is reached with. A different path from k3s_vault_path +# above because it's a different kind of secret: that one is cluster identity +# (the join token), this one is access to the hosts, and it lives alongside +# the Unraid key under homelab/ci/ rather than with the cluster's own data. +# Fetched by playbooks/k3s_ssh_key.yml, which both k3s playbooks import. +k3s_ssh_key_vault_path: homelab/ci/ssh-k3s + +# Where that play writes it, and what every play here then connects with — so +# a manual run needs Vault credentials in the environment and nothing else, +# rather than the key pre-loaded into ssh-agent. Override on the command line +# (-e ansible_ssh_private_key_file=~/.ssh/k3s_ansible) to use a local copy +# instead, e.g. if Vault itself is down. +# +# Under local/, which .gitignore already excludes — same as the kubeconfig +# below, and for the same reason: generated credential, never committed. Built +# from playbook_dir rather than repo_root, unlike k3s_kubeconfig_local_path: +# repo_root is a *play* var set in k3s.yml, so it only resolves inside the two +# plays that set it, whereas this is a connection var and has to resolve in +# every play in the file. +k3s_ssh_key_local_path: "{{ playbook_dir }}/../../../../local/k3s/{{ k3s_cluster_name }}.key" +ansible_ssh_private_key_file: "{{ k3s_ssh_key_local_path }}" + +# Extra INSTALL_K3S_EXEC args appended on every node, both roles. Empty by +# default. Only flags valid on *both* `k3s server` and `k3s agent` belong +# here — see k3s_server_extra_args below for server-only flags like +# `--disable`, which `k3s agent` doesn't understand and will fail to start +# with. +k3s_extra_args: [] + +# Extra INSTALL_K3S_EXEC args appended on the control-plane node only. +# --disable=servicelb turns off k3s's bundled LoadBalancer controller +# (Klipper) in favour of roles/k3s_metallb — the two would otherwise both +# try to satisfy the same LoadBalancer Services. Add e.g. '--disable=traefik' +# here too if homelab-utils apps end up wanting their own ingress instead. +# +# Changing this (or k3s_extra_args) on an already-installed cluster is picked +# up on the next `ansible-playbook playbooks/k3s.yml` run — roles/k3s_node +# tracks the exec line it last installed with and reinstalls (restarting the +# k3s/k3s-agent service, not the node) whenever it drifts from what's +# configured here. See tasks/server.yml and tasks/agent.yml. +k3s_server_extra_args: + - "--disable=servicelb" + +# MetalLB's LoadBalancer IP pool — see roles/k3s_metallb. Must be addresses +# on the LAN (192.168.50.0/24 — see inventory/hosts.yml) that nothing else, +# DHCP included, will ever hand out. Adjust to a range your DHCP scope +# doesn't cover before the first run. +k3s_metallb_address_range: "192.168.50.80-192.168.50.99" + +# The LAN address the shared Postgres primary answers on — see +# roles/k3s_postgres, which declares a LoadBalancer Service for it alongside +# CNPG's built-in ClusterIP ones. Pinned rather than auto-assigned because +# the address gets written down elsewhere (an app's vars.yml, a connection +# string in Vault) instead of only being looked up at runtime, so it mustn't +# move when the Service is recreated. +# +# Must be inside the pool above — MetalLB only assigns from its own pools, +# and a request for an address outside them leaves the Service pending +# forever rather than failing loudly. The caveat of pinning out of the same +# pool MetalLB auto-assigns from: it hands unpinned Services the lowest free +# address, so if some other LoadBalancer Service is created before this one +# and takes .80, this Service is the one left pending. Nothing else here +# requests an IP today. If that changes, the fix is a second +# IPAddressPool with autoAssign: false in roles/k3s_metallb reserved for +# pinned addresses, not a different number here. +k3s_postgres_loadbalancer_ip: "192.168.50.81" + +# Where playbooks/k3s.yml fetches the kubeconfig to, on the Ansible +# controller. Under local/, which .gitignore already excludes — it's a +# generated credential, never committed. +k3s_kubeconfig_local_path: "{{ repo_root }}/local/k3s/{{ k3s_cluster_name }}.kubeconfig" + +# --- cert-manager (roles/k3s_cert_manager) ---------------------------------- +# +# TLS for anything this cluster serves over Traefik. See that role's +# defaults/main.yml for why DNS-01 rather than HTTP-01, and what each of +# these does. + +# Where Let's Encrypt sends expiry warnings. An ACME account is registered +# against it. +k3s_cert_manager_acme_email: russell.seymour@turtlesystems.co.uk + +# >>> SET BEFORE THE FIRST RUN <<< +# +# How cert-manager proves control of the domain, rendered into the +# ClusterIssuer's solvers list as-is. Left empty deliberately — it depends on +# who runs DNS for turtlesystems.uk, which this repo has no way to know. +# roles/k3s_cert_manager fails fast while it's empty, rather than creating an +# issuer that can never satisfy an order. +# +# Cloudflare, as the most common case — the token needs Zone:DNS:Edit on the +# zone, and is stored in Vault (see k3s_cert_manager_vault_path below and +# docs/vault-secrets.md) rather than written here: +# +# k3s_cert_manager_solver: +# dns01: +# cloudflare: +# apiTokenSecretRef: +# name: cert-manager-dns-credentials +# key: CLOUDFLARE_API_TOKEN +# selector: +# dnsZones: +# - turtlesystems.uk +# +# For another provider, take the stanza from +# https://cert-manager.io/docs/configuration/acme/dns01/ and point its secret +# ref at whichever key you stored at that Vault path. +# +# Use the Let's Encrypt staging directory while working this out — override +# k3s_cert_manager_acme_server. Production allows 5 failed validations per +# hostname per hour, and exhausting it means waiting, not retrying. +k3s_cert_manager_solver: + dns01: + cloudflare: + apiTokenSecretRef: + name: cert-manager-dns-credentials + key: CLOUDFLARE_API_TOKEN + selector: + dnsZones: + - turtlesystems.uk + +# --- Traefik (roles/k3s_traefik) -------------------------------------------- +# +# The ingress controller k3s installs for itself; that role installs nothing, +# it only adjusts what's already there. Its defaults are enough for the +# dashboard over `kubectl port-forward` — see that role's defaults/main.yml +# and README.md → "Dashboard (Traefik)". + +# >>> SET WHEN AUTHENTIK IS READY TO PROTECT IT <<< +# +# Uncomment to publish the dashboard on this hostname, behind the forward-auth +# middleware below. Until then the port-forward is the only way in, which is +# the safe default rather than a limitation. Needs a DNS record pointing at +# Traefik's MetalLB address (`kubectl -n kube-system get svc traefik`). +# +# Before uncommenting, in Authentik: create a Proxy Provider in **forward auth +# (domain level)** mode covering turtlesystems.uk, assign it to an +# application, and add that application to the embedded outpost. Domain level +# rather than single-application for a Kubernetes-specific reason — see +# roles/k3s_traefik/defaults/main.yml. +# +# k3s_traefik_dashboard_host: traefik.turtlesystems.uk + +# Authentik's embedded outpost, by in-cluster DNS — the `authentik-server` +# Service in the `authentik` namespace, i.e. the chart's own service, not a +# separately deployed outpost. Unused while the host above is commented out; +# set here anyway so enabling the dashboard is one line rather than two. +# roles/k3s_traefik refuses to run if the host is set and this isn't. +k3s_traefik_dashboard_auth_address: "http://authentik-server.authentik.svc.cluster.local/outpost.goauthentik.io/auth/traefik" + +# --- Apps on the cluster (roles/k3s_app) ------------------------------------ +# +# The k3s equivalent of `stacks:` in host_vars/nas01.yml and `apps:` in a +# Proxmox guest's host_vars — the list of *applications* on this cluster, as +# opposed to the k3s_* cluster services above, which are the cluster's own +# infrastructure. +# +# In group_vars rather than host_vars because an app is deployed to the +# cluster, not to a node: playbooks/k3s.yml runs roles/k3s_app against +# k3s_control_plane only (that's where the manifests directory is), but +# nothing about an app belongs to that Pi specifically. +# +# Same entry shape as the other two platforms — name, src, vault_path, an +# optional `db:`, and an optional `state:`. And the same removal rule as +# Unraid: set `state: absent` to tear an app down and leave the entry here as +# a tombstone; deleting the entry removes nothing, it just stops Ansible +# visiting it. See roles/k3s_app/tasks/remove.yml. +k3s_apps: + - name: authentik + src: authentik + vault_path: homelab/authentik + db: + name: authentik + user: authentik + # The key at homelab/authentik holding this role's password. Named for + # the environment variable Authentik itself reads, because + # roles/k3s_app passes Vault keys through to the app's Secret verbatim + # — so one value serves both the provisioning step and the running app. + password_vault_key: AUTHENTIK_POSTGRESQL__PASSWORD + admin_vault_path: homelab/shared/postgres + # The CNPG LoadBalancer, not the in-cluster -rw Service: these tasks + # run on the Ansible controller (delegate_to: localhost), which is off + # the cluster and can't resolve or route to a ClusterIP. Authentik + # itself uses the ClusterIP — see + # src/authentik/ansible/kubernetes/vars.yml. + provision_host: "{{ k3s_postgres_loadbalancer_ip }}" + provision_port: "5432" diff --git a/build/config/ansible/inventory/group_vars/k3s_control_plane.yml b/build/config/ansible/inventory/group_vars/k3s_control_plane.yml new file mode 100644 index 0000000..d4b1afc --- /dev/null +++ b/build/config/ansible/inventory/group_vars/k3s_control_plane.yml @@ -0,0 +1,9 @@ +--- +# Selects the server branch of roles/k3s_node/tasks/main.yml. +k3s_node_role: server + +# The API server's cert needs this IP in its SAN list so kubectl can reach it +# at the same address agents join through. Defaults to the control-plane +# host's own address; override here if you later put a DNS name or VIP in +# front of it. +k3s_api_tls_san: "{{ ansible_host }}" diff --git a/build/config/ansible/inventory/group_vars/k3s_workers.yml b/build/config/ansible/inventory/group_vars/k3s_workers.yml new file mode 100644 index 0000000..72e5a5d --- /dev/null +++ b/build/config/ansible/inventory/group_vars/k3s_workers.yml @@ -0,0 +1,3 @@ +--- +# Selects the agent branch of roles/k3s_node/tasks/main.yml. +k3s_node_role: agent diff --git a/build/config/ansible/inventory/group_vars/proxmox_guests.yml b/build/config/ansible/inventory/group_vars/proxmox_guests.yml new file mode 100644 index 0000000..f48f891 --- /dev/null +++ b/build/config/ansible/inventory/group_vars/proxmox_guests.yml @@ -0,0 +1,19 @@ +--- +# Connection settings for the LXCs on the Proxmox nodes — the group +# inventory/proxmox.yml composes from the API, targeted by +# playbooks/proxmox.yml. As distinct from proxmox_nodes.yml, which is the +# hosts those guests run on. +# +# In group_vars rather than repeated per guest because it is a property of +# how every container this repo creates is built, not of any one app: the +# Terraform modules install `ssh_public_keys` for root (see +# build/config/terraform/variables.tf) and create no other account, so root +# is the only user Ansible can authenticate as. +# +# Needed explicitly because the other three groups set this in hosts.yml +# (`root` for unraid_servers and proxmox_nodes, `ansible` for k3s_cluster) +# and the guests have no static entry to set it in. Without it Ansible falls +# back to the *controller's* login name, which no container has an account +# for — a "Permission denied (publickey)" that names a user nothing in the +# repo mentions, so there is no obvious thread to pull. +ansible_user: root diff --git a/build/config/ansible/inventory/group_vars/proxmox_nodes.yml b/build/config/ansible/inventory/group_vars/proxmox_nodes.yml new file mode 100644 index 0000000..cc5942d --- /dev/null +++ b/build/config/ansible/inventory/group_vars/proxmox_nodes.yml @@ -0,0 +1,46 @@ +--- +# Configuration for the Proxmox nodes themselves — as distinct from the guests +# on them, which come from the API-backed dynamic inventory and are configured +# by playbooks/proxmox.yml. Applied by playbooks/pve_host.yml. +# +# In group_vars rather than a host_vars file per node: both nodes want the +# same backup storage and the same schedule, and each backs up whichever +# guests are running on it, so there is nothing here that differs between +# them. A node that ever does need to differ can still override in +# host_vars/.yml. + +# Where archives go. An export on the NAS, added to Proxmox as a storage so +# vzdump can write to it — the node mounts it, no guest ever sees it. +# +# nas01 (192.168.50.1). This used to point at nas2, which no longer exists — +# and note the address it used, 192.168.50.2, now belongs to +# turtle-proxmox-01, so a stale copy of this value would have had each node +# quietly trying to write its backups to itself. +pve_backup_storage: nas-backup +pve_backup_storage_type: nfs +pve_backup_nfs_server: 192.168.50.1 +pve_backup_nfs_export: /mnt/user/backups/proxmox + +# Every guest on the node, so anything added later is covered without an edit +# here. Set false and list VMIDs (postgres is 161, forgejo 160 — see the +# `backup_vmids` output in build/config/terraform/main.tf) to be selective. +# +# Applied per node, so with two nodes each one archives whatever is currently +# running on it. That matters for a replicated guest: it is a real guest on +# one node and an inactive replica volume on the other, so it is backed up +# once, by whichever node currently owns it — and after a failover, by the +# other one, without an edit here. +pve_backup_all: true + +# 02:00, half an hour after Forgejo's database dump at 01:30 +# (src/forgejo/ansible/proxmox/vars.yml). The archive is only a complete +# restore point if the dump inside it is from the same night — move one and +# move the other. +pve_backup_schedule: "02:00" + +# Archive retention, applied by Proxmox. Distinct from the retention inside a +# guest: Proxmox prunes archives, not their contents. +pve_backup_prune: + keep-daily: 7 + keep-weekly: 4 + keep-monthly: 6 diff --git a/build/config/ansible/inventory/host_vars/forgejo.yml b/build/config/ansible/inventory/host_vars/forgejo.yml new file mode 100644 index 0000000..30791b0 --- /dev/null +++ b/build/config/ansible/inventory/host_vars/forgejo.yml @@ -0,0 +1,52 @@ +--- +# The Proxmox LXC created by src/forgejo/terraform (VMID 160), on +# turtle-proxmox-02. Named for the container's hostname, which is how it +# appears in the API-backed dynamic inventory. +# +# This was empty for a long time while Forgejo also ran as a Compose stack on +# nas2 — two deployments sharing one Vault path and one database means the +# second to start wins and the other quietly serves stale state. nas2 has +# since been retired, so there is no longer a second copy to collide with and +# the `apps:` list below is live. +# +# WHAT THIS DOES NOT DO: nothing here moves repository data. Ansible +# provisions the database and installs the app; a Forgejo with no +# /var/lib/forgejo/data/forgejo-repositories is a working, empty forge. If the +# nas2 repositories still exist somewhere, restore them into +# `FORGEJO_DATA_PATH` and `chown -R git:git /var/lib/forgejo/data` before +# pointing DNS at this host — the original instructions here were an rsync +# off nas2, which is no longer a machine that can be read from. +# +# Also note SSH clone URLs move from port 2222 to 22 (see FORGEJO_SSH_PORT in +# src/forgejo/ansible/proxmox/vars.yml): existing remotes need editing, and +# git.turtlesystems.uk has to resolve to this container. + +# The address Terraform assigns this guest (build/config/terraform/main.tf). +# Pinned here rather than left to DNS: the API-backed dynamic inventory +# supplies no `ansible_host`, so without this Ansible SSHes to the bare +# inventory name `forgejo`, which the LAN's resolver has no record for at +# all. These addresses are hand-assigned from the 192.168.50.50-.59 band +# anyway, so restating one here duplicates nothing that was ever derived. +ansible_host: 192.168.50.52 + +apps: + - name: forgejo + src: forgejo + vault_path: homelab/forgejo + db: + # The shared Postgres LXC, reached over the LAN. Unlike the Unraid + # host_vars this can't be `{{ ansible_host }}` — that would be this + # container's own address, not the database's. Keep in step with + # DB_HOST in src/forgejo/ansible/proxmox/vars.yml; the authoritative + # copy of both is the `postgres_address` output in + # build/config/terraform/main.tf. + # + # Provisioning runs from the Ansible controller (`delegate_to: + # localhost` in roles/lxc_app), so this address has to be reachable + # from wherever you run the playbook, not just from this container. + provision_host: 192.168.50.54 + provision_port: 5432 + name: forgejo + user: forgejo + password_vault_key: DB_PASSWORD + admin_vault_path: homelab/shared/postgres diff --git a/build/config/ansible/inventory/host_vars/nas01.yml b/build/config/ansible/inventory/host_vars/nas01.yml new file mode 100644 index 0000000..dcacd1a --- /dev/null +++ b/build/config/ansible/inventory/host_vars/nas01.yml @@ -0,0 +1,19 @@ +--- +# Existing Unraid server. Stacks are applied in list order. +# +# Each entry takes an optional `state:` — `present` (the default) deploys it, +# `absent` tears it down. To remove a stack, set `state: absent` and leave the +# entry here; deleting it instead removes nothing, it just stops Ansible +# managing the containers. See README.md → "Removing an app". +stacks: + # The arr ecosystem as one stack: Shelfarr (library) plus Prowlarr (indexer + # manager) in a single Compose project, so they come up together and share a + # network, a rendered .env and a Vault path. + - name: arr + src: arr + vault_path: homelab/arr + # No `db:` block — neither container uses an external database. Shelfarr + # is SQLite-only (config/database.yml pins production to storage/*.sqlite3 + # with no DATABASE_URL support) and Prowlarr keeps its own SQLite under + # /config, so there is nothing to provision on the shared Postgres. Both + # databases live in the appdata bind mounts along with everything else. diff --git a/build/config/ansible/inventory/host_vars/postgres.yml b/build/config/ansible/inventory/host_vars/postgres.yml new file mode 100644 index 0000000..4d15c18 --- /dev/null +++ b/build/config/ansible/inventory/host_vars/postgres.yml @@ -0,0 +1,28 @@ +--- +# The shared Postgres LXC created by src/shared/postgres/terraform (VMID 161, +# 192.168.50.54). Named for the container's hostname, which is how it appears +# in the API-backed dynamic inventory. +# +# This is the one app with no `db:` block, because it *is* the database — the +# per-app databases are created against it by whichever compose_stack / +# lxc_app / k3s_app run declares them, using homelab/shared/postgres as +# `admin_vault_path`. + +# The address Terraform assigns this guest (build/config/terraform/main.tf). +# Pinned here rather than left to DNS: the API-backed dynamic inventory +# supplies no `ansible_host`, so without this Ansible SSHes to the bare +# inventory name and whatever the LAN's resolver happens to return. For +# `postgres` that resolver answers 192.168.50.53 — the *Unraid* Postgres +# container, a different instance of the same service — which fails as a +# connection refused rather than as anything that names the real problem. +# These addresses are hand-assigned from the 192.168.50.50-.59 band anyway, +# so restating one here duplicates nothing that was ever derived. +ansible_host: 192.168.50.54 + +apps: + - name: postgres + src: shared/postgres + vault_path: homelab/shared/postgres + # The packaging's umbrella unit, not postgresql@17-main — see + # POSTGRES_SERVICE_NAME in src/shared/postgres/ansible/proxmox/vars.yml. + service_name: postgresql diff --git a/build/config/ansible/inventory/hosts.yml b/build/config/ansible/inventory/hosts.yml new file mode 100644 index 0000000..99b38d3 --- /dev/null +++ b/build/config/ansible/inventory/hosts.yml @@ -0,0 +1,58 @@ +--- +all: + children: + unraid_servers: + hosts: + nas01: + ansible_host: 192.168.50.1 + vars: + ansible_user: root + + # The Proxmox host itself, not the guests on it — those come from the + # API-backed dynamic inventory (proxmox.yml) as `proxmox_guests`. Node-level + # configuration (backup storage, the vzdump schedule) has to be applied over + # SSH to the node, which the API inventory gives no way to address. + # + # VERIFY: depending on its configuration, the community.proxmox inventory + # plugin can also emit the node as a host. If it does, this static entry and + # the dynamic one merge under the same name — check with + # `ansible-inventory --graph` before assuming which vars won. + proxmox_nodes: + hosts: + turtle-proxmox-01: + ansible_host: 192.168.50.2 + turtle-proxmox-02: + ansible_host: 192.168.50.3 + vars: + ansible_user: root + + # Bare-metal K3s cluster (4 Raspberry Pis) — a third platform alongside + # Unraid and Proxmox, not an app deployed onto either of them. No + # Terraform: the hosts already exist. See playbooks/k3s.yml and + # roles/k3s_node. Split into two groups, not one, so the playbook can + # bootstrap the control plane before any agent tries to join it. + # + # ansible_user is `ansible`, not root: create this user on every Pi with + # NOPASSWD sudo and install the public half of `homelab/ci/ssh-k3s` + # (docs/vault-secrets.md) as its authorized key before the first run — + # see README.md "SSH access". The private half doesn't need setting up on + # the controller: group_vars/k3s_cluster.yml points + # ansible_ssh_private_key_file at a path playbooks/k3s_ssh_key.yml fetches + # from that same Vault entry. + k3s_cluster: + vars: + ansible_user: ansible + ansible_become: true + children: + k3s_control_plane: + hosts: + k3s-ctrl-01: + ansible_host: 192.168.50.60 + k3s_workers: + hosts: + k3s-wkr-01: + ansible_host: 192.168.50.61 + k3s-wkr-02: + ansible_host: 192.168.50.62 + k3s-wkr-03: + ansible_host: 192.168.50.63 diff --git a/build/config/ansible/inventory/proxmox.yml b/build/config/ansible/inventory/proxmox.yml new file mode 100644 index 0000000..ff3a4a0 --- /dev/null +++ b/build/config/ansible/inventory/proxmox.yml @@ -0,0 +1,56 @@ +--- +# Dynamic inventory: asks the Proxmox API which guests exist rather than +# listing them by hand, so LXCs created by `terraform apply` show up without +# a second edit here. +# +# VERIFY BEFORE FIRST USE — two things this config asserts that depend on +# your Proxmox and collection versions: +# +# 1. Collection name. Proxmox content was split out of `community.general` +# into its own `community.proxmox` collection. If your installed +# Ansible predates that split, this is `community.general.proxmox` +# instead — here, in requirements.yml, and in ansible.cfg's +# `enable_plugins`. Check with: +# ansible-doc -t inventory community.proxmox.proxmox +# 2. Auto-generated group names. The plugin builds groups from guest type +# and state; `proxmox_all_lxc` is the conventional one but confirm +# against your own output before relying on it: +# ansible-inventory -i inventory/proxmox.yml --graph +# +plugin: community.proxmox.proxmox + +url: "{{ lookup('env', 'PROXMOX_URL') }}" +user: "{{ lookup('env', 'PROXMOX_USER') }}" + +# API token, not a password — same reasoning as the Unraid SSH key: a +# password would mean an interactive prompt (impossible from CI) or a +# plaintext secret on disk. Store these in Vault under `homelab/ci/proxmox` +# and export them before running; see docs/vault-secrets.md. +token_id: "{{ lookup('env', 'PROXMOX_TOKEN_ID') }}" +token_secret: "{{ lookup('env', 'PROXMOX_TOKEN_SECRET') }}" + +# Homelab Proxmox typically has a self-signed cert. Flip to true once you +# put a real one on it. +validate_certs: false + +want_facts: true + +# Manually-created guests are not this repo's to touch. Both Terraform +# modules tag the LXCs they create with `terraform` (see +# src/*/terraform/variables.tf), so that tag is the line between "managed +# here" and "made by hand on the node". Filtering rather than narrowing +# the group below keeps hand-made guests out of the inventory entirely, +# so no future play can target them by accident either. +# +# `filters` needs a reasonably recent plugin version. If yours rejects the +# option, delete this block and put the same condition on the group: +# proxmox_guests: >- +# proxmox_vmtype == 'lxc' +# and 'terraform' in (proxmox_tags_parsed | default([])) +filters: + - "'terraform' in (proxmox_tags_parsed | default([]))" + +groups: + # The group playbooks/proxmox.yml targets. Containers only — VMs, if you + # ever add any, are not what `lxc_app` knows how to install into. + proxmox_guests: "proxmox_vmtype == 'lxc'" diff --git a/build/config/ansible/playbooks/deploy.yml b/build/config/ansible/playbooks/deploy.yml new file mode 100644 index 0000000..71ed74f --- /dev/null +++ b/build/config/ansible/playbooks/deploy.yml @@ -0,0 +1,21 @@ +--- +# Everything, both platforms. Unraid first only by convention now — the two +# platforms no longer depend on each other. Proxmox-side apps provision their +# databases against the Proxmox shared Postgres (192.168.50.54), and Terraform +# keeps its state on the CloudNativePG cluster on k3s, neither of which is on +# Unraid. +# +# Run a single platform with `ansible-playbook playbooks/unraid.yml` or +# `playbooks/proxmox.yml` instead. `--limit ` and `-e only_stacks=...` +# work the same way against any of the three. +- name: Deploy Unraid stacks + ansible.builtin.import_playbook: unraid.yml + +- name: Deploy Proxmox apps + ansible.builtin.import_playbook: proxmox.yml + +# Last, because the vzdump job can be pinned to specific VMIDs and those guests +# have to exist first. Harmless to re-run; skipped entirely if no node is +# defined in the `proxmox_nodes` group. +- name: Configure Proxmox node backups + ansible.builtin.import_playbook: pve_host.yml diff --git a/build/config/ansible/playbooks/k3s.yml b/build/config/ansible/playbooks/k3s.yml new file mode 100644 index 0000000..afde238 --- /dev/null +++ b/build/config/ansible/playbooks/k3s.yml @@ -0,0 +1,241 @@ +--- +# Bootstraps the homelab-utils k3s cluster from bare Pis: no Terraform, the +# hosts already exist (inventory/hosts.yml → k3s_cluster). Manual-only for +# now — not wired into .forgejo/workflows/deploy.yml, since converging 4 +# physical nodes on every push is a bigger blast radius than restarting a +# Compose stack (same reasoning the repo already applies to `terraform +# apply`). Run by hand: +# +# ansible-playbook playbooks/k3s.yml +# +# Every play below the SSH-key import is tagged so a single piece can be +# converged without touching the rest — deploying one cluster service +# shouldn't mean re-running the node install across 4 Pis: +# +# --tags nodes k3s install/join + exec-line drift (both node plays) +# --tags metallb MetalLB chart + IPAddressPool +# --tags monitoring kube-prometheus-stack chart +# --tags postgres CloudNativePG operator + Cluster +# --tags cert-manager cert-manager chart + ClusterIssuer +# --tags traefik dashboard route on k3s's bundled Traefik +# --tags services all five of the above, no node install +# --tags apps the k3s_apps: list (roles/k3s_app) — apps, not +# cluster services; narrow further with -e only_apps=… +# --tags upgrades unattended-upgrades config +# --tags kubeconfig re-fetch the admin kubeconfig +# +# e.g. `ansible-playbook playbooks/k3s.yml --tags postgres`. The SSH-key +# import is tagged `always` rather than given a name of its own: it's not a +# thing you'd select, it's the prerequisite for any of these connecting at +# all, so it has to survive every --tags filter. +# +# `services` covers the cluster's own infrastructure and `apps` covers what +# runs on top of it; neither implies the other, so a run meaning "everything +# except reinstalling k3s" is `--tags services,apps`. +# +# Prerequisites (see README.md "K3s (Raspberry Pi)"): the `ansible` user +# exists on every Pi with NOPASSWD sudo and this repo's SSH key installed, +# and homelab/k3s-homelab-utils/K3S_TOKEN is already set in Vault — see +# docs/vault-secrets.md. Both plays fetch that same fixed token independently +# rather than one generating it and handing it to the other, which is what +# makes a full rebuild (wipe both SD cards, reinstall) reproduce the same +# cluster identity. + +# First, before anything tries to connect: pull the SSH key the plays below +# authenticate with out of Vault and onto the controller. See +# k3s_ssh_key.yml — it's a separate file because k3s_maintenance.yml imports +# it too. +- ansible.builtin.import_playbook: k3s_ssh_key.yml + tags: always + +- name: Bootstrap the k3s control plane + hosts: k3s_control_plane + gather_facts: true + tags: nodes + roles: + - k3s_node + +# serial: 1 not for joining itself (idempotent, safe in parallel) but for +# roles/k3s_node's exec-line drift detection: if k3s_extra_args changes and +# every worker's k3s-agent restarts to pick it up, one at a time keeps more +# than one node's kubelet from bouncing simultaneously. See tasks/agent.yml. +- name: Join k3s worker nodes + hosts: k3s_workers + gather_facts: true + serial: 1 + tags: nodes + roles: + - k3s_node + +# Against the control plane only — it just drops manifests for k3s's own +# helm-controller and deploy controller to reconcile, so it doesn't need +# every node like the plays above. Runs after workers have joined so the +# resulting speaker DaemonSet schedules across the whole cluster on first +# reconcile, though this doesn't strictly matter: the controller picks up +# new nodes on its own regardless of ordering. Before monitoring so a +# LoadBalancer Service (e.g. exposing Prometheus/Grafana later) has +# somewhere to get an IP from as soon as it's requested. +- name: Deploy MetalLB (LoadBalancer IPs) + hosts: k3s_control_plane + gather_facts: false + tags: + - services + - metallb + roles: + - k3s_metallb + +# Against the control plane only — it just drops a HelmChart manifest for +# k3s's own helm-controller to reconcile, so it doesn't need every node like +# the plays above. Runs after workers have joined so the resulting +# node-exporter DaemonSet schedules across the whole cluster on first +# reconcile, though this doesn't strictly matter: the controller picks up +# new nodes on its own regardless of ordering. +- name: Deploy cluster monitoring (Prometheus) + hosts: k3s_control_plane + gather_facts: false + tags: + - services + - monitoring + roles: + - k3s_monitoring + +# Against the control plane only, same reasoning as monitoring/MetalLB above +# — it just drops manifests for k3s's own controllers to reconcile. Needs +# repo_root (unlike the other k3s roles) because it reads config from +# src/shared/postgres/, the same shared-service config Unraid/Proxmox +# already deploy from — see roles/k3s_postgres/tasks/main.yml. +- name: Deploy shared Postgres (CloudNativePG) + hosts: k3s_control_plane + gather_facts: false + tags: + - services + - postgres + vars: + repo_root: "{{ playbook_dir }}/../../../.." + roles: + - k3s_postgres + +# Against the control plane only, same reasoning as the services above. +# Before the apps play because an app's Ingress annotates itself against the +# ClusterIssuer this creates — not that ordering is load-bearing (k3s's +# deploy controller retries, and cert-manager picks up an Ingress whenever it +# appears), but an app deployed first would sit without a certificate until +# this ran, which reads as a broken deploy rather than a pending one. +- name: Deploy cert-manager (TLS certificates) + hosts: k3s_control_plane + gather_facts: false + tags: + - services + - cert-manager + roles: + - k3s_cert_manager + +# Against the control plane only, same reasoning as the services above. The +# odd one out among them: it installs nothing, because k3s installs Traefik +# itself — it only adjusts what k3s already put there, via a HelmChartConfig. +# After cert-manager because publishing the dashboard on a hostname asks for a +# Certificate from the ClusterIssuer that play creates; ordering isn't +# load-bearing (the deploy controller retries), it just avoids a route sitting +# without a certificate in between. +- name: Configure Traefik (ingress + dashboard) + hosts: k3s_control_plane + gather_facts: false + tags: + - services + - traefik + roles: + - k3s_traefik + +# Apps, as opposed to the cluster services above — see roles/k3s_app for the +# distinction and inventory/group_vars/k3s_cluster.yml for the list. Against +# the control plane because that's where the manifests directory is; nothing +# about an app belongs to that Pi in particular. +# +# Note what a green run here does and doesn't mean, the same caveat every +# service play above carries: the role renders manifests for k3s's +# controllers to reconcile, so success means the files landed and any +# database was provisioned — not that the workload came up. Check with +# `kubectl -n get pods`. +- name: Converge apps on the cluster + hosts: k3s_control_plane + gather_facts: false + tags: apps + vars: + repo_root: "{{ playbook_dir }}/../../../.." + # Comma-separated app names to restrict this run to, e.g. + # `-e only_apps=authentik`. Empty (the default) converges every app in + # k3s_apps. Same knob as only_stacks in playbooks/unraid.yml — --tags + # can select the apps play as a whole, but not one app within it. + only_apps: "" + + tasks: + # "Converge", not "deploy": an entry carrying `state: absent` is torn + # down rather than brought up. Removals have to stay in the list to be + # acted on — see roles/k3s_app/tasks/main.yml. + - name: Converge each declared app + ansible.builtin.include_role: + name: k3s_app + loop: >- + {{ k3s_apps | default([]) if only_apps == '' + else k3s_apps | default([]) + | selectattr('name', 'in', only_apps.split(',')) | list }} + loop_control: + loop_var: app + label: "{{ app.name }}" + +# Every node, both roles — hands-off patching so the Pis don't need logging +# into just to stay updated. Reboots are deliberately not part of this: see +# roles/unattended_upgrades and playbooks/k3s_maintenance.yml. +- name: Configure unattended upgrades + hosts: k3s_cluster + gather_facts: true + tags: upgrades + roles: + - unattended_upgrades + +# Last, and against the control plane specifically (there's only one) — pulls +# the admin kubeconfig k3s wrote for itself back to the controller so +# `kubectl` works from your workstation. +- name: Fetch the cluster kubeconfig + hosts: k3s_control_plane + gather_facts: false + tags: kubeconfig + vars: + repo_root: "{{ playbook_dir }}/../../../.." + + tasks: + - name: Read the cluster's kubeconfig + ansible.builtin.slurp: + src: /etc/rancher/k3s/k3s.yaml + register: k3s_kubeconfig_raw + + - name: Ensure the local kubeconfig directory exists + ansible.builtin.file: + path: "{{ k3s_kubeconfig_local_path | dirname }}" + state: directory + mode: "0700" + delegate_to: localhost + become: false + + # k3s.yaml points at 127.0.0.1 and names everything "default" — both + # correct only on the node itself. Rewritten so the file is usable + # straight off the controller: the server address becomes reachable from + # off-box, and the cluster/context/user names become this cluster's own + # rather than colliding with every other "default" in ~/.kube/config. + - name: Write the rewritten kubeconfig to the controller + ansible.builtin.copy: + dest: "{{ k3s_kubeconfig_local_path }}" + content: >- + {{ (k3s_kubeconfig_raw.content | b64decode) + | replace('127.0.0.1', ansible_host) + | regex_replace('\\bdefault\\b', k3s_cluster_name) }} + mode: "0600" + delegate_to: localhost + become: false + + - name: Show how to use the fetched kubeconfig + ansible.builtin.debug: + msg: >- + Kubeconfig written to {{ k3s_kubeconfig_local_path }}. Use it with + `export KUBECONFIG={{ k3s_kubeconfig_local_path }}`, or merge it + into ~/.kube/config by hand. diff --git a/build/config/ansible/playbooks/k3s_maintenance.yml b/build/config/ansible/playbooks/k3s_maintenance.yml new file mode 100644 index 0000000..06d7157 --- /dev/null +++ b/build/config/ansible/playbooks/k3s_maintenance.yml @@ -0,0 +1,37 @@ +--- +# Rolls a "reboot required" flag left by unattended-upgrades +# (roles/unattended_upgrades, Automatic-Reboot disabled) into an actual +# reboot, one node at a time — cordon, drain, reboot, wait for Ready, +# uncordon — so patching the OS never means dropping every workload on the +# cluster at once. A no-op on any node that isn't carrying a pending reboot. +# +# serial: 1 is the point of this playbook: exactly one node is ever down for +# maintenance at a time, whatever `--limit` narrows the run to. +# +# Host order matters here and isn't left to inventory.yml's own group +# order (control plane, then workers) — reversed below so workers are +# rebooted while the API server is still up on the control plane, and the +# control plane's own reboot (which drops the API for everyone, single-server +# cluster, no HA) happens last rather than first. +# +# Manual-only for now, like playbooks/k3s.yml — run by hand: +# +# ansible-playbook playbooks/k3s_maintenance.yml +# +# Intended to eventually run on a schedule from a self-hosted Forgejo Actions +# runner (a `schedule:`-triggered workflow, same runner as +# .forgejo/workflows/deploy.yml) rather than by hand — not wired up yet, but +# the SSH key import below is what makes that possible without also handing +# the runner an ssh-agent. + +# Same first play as k3s.yml — the key comes from Vault, and this playbook is +# routinely run on its own, so it can't rely on a k3s.yml run having fetched +# it. See k3s_ssh_key.yml. +- ansible.builtin.import_playbook: k3s_ssh_key.yml + +- name: Roll pending reboots across the k3s cluster + hosts: k3s_workers:k3s_control_plane + gather_facts: true + serial: 1 + roles: + - k3s_maintenance diff --git a/build/config/ansible/playbooks/k3s_ssh_key.yml b/build/config/ansible/playbooks/k3s_ssh_key.yml new file mode 100644 index 0000000..4576faf --- /dev/null +++ b/build/config/ansible/playbooks/k3s_ssh_key.yml @@ -0,0 +1,82 @@ +--- +# Materializes the SSH key the k3s plays authenticate with, from Vault +# (homelab/ci/ssh-k3s) onto the controller. Imported as the first play of both +# playbooks/k3s.yml and playbooks/k3s_maintenance.yml — a separate file rather +# than copied into each, since both target k3s_cluster and both would +# otherwise depend on a file only the other one creates. +# +# This is the same shape CI already uses for the Unraid key: +# .forgejo/workflows/deploy.yml fetches homelab/ci/ssh, writes it to disk and +# points ANSIBLE_PRIVATE_KEY_FILE at it. Vault is the one place secrets live, +# so a manual k3s run shouldn't need the key hand-loaded into ssh-agent first +# — which was the only way it worked before. +# +# hosts: k3s_cluster, not localhost, and deliberately so: the implicit +# localhost is not a member of `all`, so it doesn't inherit group_vars/all.yml +# — where vault_addr, vault_kv_mount and vault_auth_method live. Targeting the +# group picks those up along with k3s_ssh_key_vault_path/_local_path from +# group_vars/k3s_cluster.yml. Nothing here connects to a Pi (gather_facts is +# off and every task is delegated or connectionless), which is the whole +# point: at this stage the key isn't on disk yet. +# +# become: false on the delegated tasks because the k3s_cluster group sets +# ansible_become: true for the Pis (inventory/hosts.yml) — without it these +# would try to sudo on the controller. + +- name: Fetch the k3s SSH deploy key from Vault + hosts: k3s_cluster + gather_facts: false + + tasks: + # The escape hatch, and the reason it's a flag rather than just pointing + # ansible_ssh_private_key_file somewhere else: if Vault is unreachable, + # overriding the key path alone doesn't help — this play would still fail + # before the first real one runs. Skipping it is the only thing that lets + # a run proceed on a local key. Both overrides together: + # + # ansible-playbook playbooks/k3s.yml \ + # -e k3s_ssh_key_fetch=false \ + # -e ansible_ssh_private_key_file=~/.ssh/k3s_ansible + - name: Fetch and write the key + when: k3s_ssh_key_fetch | default(true) | bool + block: + # run_once because the key is per-cluster, not per-host: one Vault + # read for the whole group. Facts set by a run_once task apply to + # every host in the play, but nothing outside this play needs it. + - name: Look up the k3s SSH deploy key from Vault + ansible.builtin.set_fact: + k3s_ssh_key_secret: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + k3s_ssh_key_vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + run_once: true + no_log: true + + - name: Ensure the local key directory exists + ansible.builtin.file: + path: "{{ k3s_ssh_key_local_path | dirname }}" + state: directory + mode: "0700" + delegate_to: localhost + become: false + run_once: true + + # trim + an explicit trailing newline: OpenSSH rejects a key file + # whose final line isn't terminated, and `vault kv put + # PRIVATE_KEY=@file` is as likely to have stored one with trailing + # whitespace as not. 0600 for the same reason — ssh refuses a key file + # that's group- or world-readable. Both fail at connection time rather + # than here, which is a much less obvious error to read. + - name: Write the private key to the controller + ansible.builtin.copy: + dest: "{{ k3s_ssh_key_local_path }}" + content: "{{ k3s_ssh_key_secret.PRIVATE_KEY | trim }}\n" + mode: "0600" + delegate_to: localhost + become: false + run_once: true + no_log: true diff --git a/build/config/ansible/playbooks/proxmox.yml b/build/config/ansible/playbooks/proxmox.yml new file mode 100644 index 0000000..4cdbc73 --- /dev/null +++ b/build/config/ansible/playbooks/proxmox.yml @@ -0,0 +1,29 @@ +--- +# Installs apps natively into Proxmox LXCs. The LXCs themselves are created +# by Terraform (build/config/terraform) — run that first; this playbook +# assumes the guests already exist and are reachable. +# +# Hosts come from the community.proxmox inventory plugin +# (inventory/proxmox.yml), which queries the Proxmox API rather than listing +# guests by hand. `apps` is declared per guest in host_vars/.yml, +# mirroring how `stacks` works for Unraid hosts. +- name: Install apps into Proxmox LXCs + hosts: proxmox_guests + gather_facts: true + vars: + repo_root: "{{ playbook_dir }}/../../../.." + # Comma-separated app names to restrict this run to. Shares the + # `only_stacks` name with playbooks/unraid.yml so a single `-e` narrows + # both platforms in one invocation of deploy.yml. + only_stacks: "" + + tasks: + - name: Install each declared app + ansible.builtin.include_role: + name: lxc_app + loop: >- + {{ apps if only_stacks == '' + else apps | selectattr('name', 'in', only_stacks.split(',')) | list }} + loop_control: + loop_var: app + label: "{{ app.name }}" diff --git a/build/config/ansible/playbooks/pve_host.yml b/build/config/ansible/playbooks/pve_host.yml new file mode 100644 index 0000000..b2d81fe --- /dev/null +++ b/build/config/ansible/playbooks/pve_host.yml @@ -0,0 +1,14 @@ +--- +# Configures the Proxmox node itself — as distinct from the guests on it, +# which playbooks/proxmox.yml handles. Right now that means backups: the NAS +# storage archives are written to, and the vzdump schedule that fills it. +# +# Targets the static `proxmox_nodes` group in inventory/hosts.yml rather than +# the API-backed dynamic inventory, which enumerates guests. If no node is +# defined there yet, this play matches nothing and is skipped. +- name: Configure Proxmox nodes + hosts: proxmox_nodes + gather_facts: true + + roles: + - pve_backup diff --git a/build/config/ansible/playbooks/unraid.yml b/build/config/ansible/playbooks/unraid.yml new file mode 100644 index 0000000..df844ff --- /dev/null +++ b/build/config/ansible/playbooks/unraid.yml @@ -0,0 +1,24 @@ +--- +- name: Converge Unraid Docker Compose stacks + hosts: unraid_servers + gather_facts: true + vars: + repo_root: "{{ playbook_dir }}/../../../.." + # Comma-separated stack names to restrict this run to, e.g. + # `-e only_stacks=forgejo` or `-e only_stacks=forgejo,postgres`. + # Empty (the default) converges every stack declared for the host. + only_stacks: "" + + tasks: + # "Converge", not "deploy": an entry carrying `state: absent` is torn down + # rather than brought up. Removals have to stay in the list to be acted + # on — see the header comment in roles/compose_stack/tasks/main.yml. + - name: Converge each declared stack + ansible.builtin.include_role: + name: compose_stack + loop: >- + {{ stacks if only_stacks == '' + else stacks | selectattr('name', 'in', only_stacks.split(',')) | list }} + loop_control: + loop_var: stack + label: "{{ stack.name }}" diff --git a/build/config/ansible/requirements.yml b/build/config/ansible/requirements.yml new file mode 100644 index 0000000..25cf287 --- /dev/null +++ b/build/config/ansible/requirements.yml @@ -0,0 +1,15 @@ +--- +collections: + - name: community.docker + version: ">=3.10.0" + - name: community.hashi_vault + version: ">=6.2.0" + - name: community.postgresql + version: ">=3.0.0" + - name: ansible.posix + version: ">=1.5.0" + # Proxmox dynamic inventory (inventory/proxmox.yml). If your Ansible + # predates the split of Proxmox content out of community.general, drop + # this and use community.general instead — see the note in + # inventory/proxmox.yml. + - name: community.proxmox diff --git a/build/config/ansible/roles/compose_stack/defaults/main.yml b/build/config/ansible/roles/compose_stack/defaults/main.yml new file mode 100644 index 0000000..291c157 --- /dev/null +++ b/build/config/ansible/roles/compose_stack/defaults/main.yml @@ -0,0 +1,9 @@ +--- +compose_projects_root: /boot/config/plugins/compose.manager/projects +vault_kv_mount: kv + +# Where a stack's committed icon.png is copied to on the Unraid host, for the +# `net.unraid.docker.icon` label to point at. Under appdata rather than +# /boot — it's regenerated content, not configuration worth putting on the +# flash drive's write budget. +unraid_icons_root: /mnt/user/appdata/icons diff --git a/build/config/ansible/roles/compose_stack/tasks/deploy.yml b/build/config/ansible/roles/compose_stack/tasks/deploy.yml new file mode 100644 index 0000000..6093dcb --- /dev/null +++ b/build/config/ansible/roles/compose_stack/tasks/deploy.yml @@ -0,0 +1,183 @@ +--- +# Deploys one stack to the current Unraid host: sync the compose file, render +# its .env from Vault, provision its database if it declares one, then +# `docker compose up -d`. Compose files stay static/generic — only the +# rendered .env differs per run, which is what keeps re-runs idempotent. +# +# Included from main.yml when the stack's `state:` is `present`; the facts it +# relies on (stack_local_dir, stack_remote_dir) are set there. + +- name: Load portable stack variables + ansible.builtin.include_vars: + file: "{{ stack_local_dir }}/common/vars.yml" + name: stack_common_vars + +- name: Load Unraid-specific stack variables + ansible.builtin.include_vars: + file: "{{ stack_local_dir }}/ansible/unraid/vars.yml" + name: stack_platform_vars + +- name: Ensure remote project directory exists + ansible.builtin.file: + path: "{{ stack_remote_dir }}" + state: directory + mode: "0750" + +- name: Copy docker-compose.yml + ansible.builtin.copy: + src: "{{ stack_local_dir }}/ansible/unraid/docker-compose.yml" + dest: "{{ stack_remote_dir }}/docker-compose.yml" + mode: "0640" + +# --- Unraid Docker page presentation --------------------------------------- +# +# The committed icon.png has two destinations, because Unraid draws the stack +# row and the containers under it from entirely different sources: +# +# stack row — Compose Manager serves /icon.png directly +# (icon.php, which also accepts .jpg/.gif/.svg or a bare +# `icon`). No label, no template, no URL: a file or nothing. +# Present only in the maintained fork, Compose Manager Plus; +# the original plugin has no icon support at all, and there +# the copy is inert rather than harmful. +# containers — the Docker page reads a container's icon, WebUI link and +# console shell from the dockerMan template that created it. +# A Compose stack has no template, so 6.10+ falls back to the +# `net.unraid.docker.*` labels the compose files set, and +# STACK_ICON below is what the icon label resolves to. +# +# Hence the appdata copy as well: the label is a path the webgui resolves at +# page-render time, and pointing it into /boot to reuse the project-dir copy +# would put the flash drive in the path of every Docker page load. +# +# Only the second of those is per-container, so a stack running more than one +# service can also commit icon-.png files. Those get the appdata copy +# only — there is exactly one stack row and it already has its icon. See +# main.yml, which is where the names are resolved. +- name: Sync stack icons to the Unraid host + when: stack_icon_dest | length > 0 or stack_service_icon_names | length > 0 + block: + - name: Ensure the icon directory exists + ansible.builtin.file: + path: "{{ unraid_icons_root }}" + state: directory + mode: "0755" + + - name: Copy stack icon for the container labels + ansible.builtin.copy: + src: "{{ stack_local_dir }}/ansible/unraid/icon.png" + dest: "{{ stack_icon_dest }}" + mode: "0644" + when: stack_icon_dest | length > 0 + + # Into the project folder on the flash drive, next to the compose file. + # `copy` is checksum-based, so a redeploy that hasn't changed the icon + # writes nothing — this costs one flash write per icon, not one per run. + - name: Copy stack icon for the Compose Manager stack row + ansible.builtin.copy: + src: "{{ stack_local_dir }}/ansible/unraid/icon.png" + dest: "{{ stack_remote_dir }}/icon.png" + mode: "0644" + when: stack_icon_dest | length > 0 + + # Appdata only, and no flash-drive copy: these label individual containers, + # which is the one thing the Compose Manager stack row isn't. + - name: Copy per-service icons for the container labels + ansible.builtin.copy: + src: "{{ stack_local_dir }}/ansible/unraid/icon-{{ item }}.png" + dest: "{{ unraid_icons_root }}/{{ stack.name }}-{{ item }}.png" + mode: "0644" + loop: "{{ stack_service_icon_names }}" + +- name: Look up stack secrets from Vault + ansible.builtin.set_fact: + vault_secrets: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + stack.vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + no_log: true + +- name: Render .env file + ansible.builtin.template: + src: env.j2 + dest: "{{ stack_remote_dir }}/.env" + mode: "0600" + vars: + # The role-computed icon paths first, then portable values, then + # Unraid-specific overrides on top; Vault secrets win over all three (see + # env.j2). STACK_ICON and the per-service STACK_ICON_ keys sit at + # the bottom so either vars.yml layer can replace one with a hosted URL, + # and STACK_ICON is always defined so Compose never warns about an unset + # variable in the label block. + # + # STACK_ICON_ only exists when the matching icon-.png is + # committed. A compose file that interpolates one without the file present + # gets Compose's empty-value warning and the placeholder icon — set it in + # vars.yml instead if the icon is meant to be a URL. + env_defaults: >- + {{ {'STACK_ICON': stack_icon_dest} + | combine(stack_service_icons) + | combine(stack_common_vars.env_defaults | default({})) + | combine(stack_platform_vars.env_defaults | default({})) }} + no_log: true + +- name: Provision application database + when: stack.db is defined + no_log: true + block: + - name: Look up Postgres superuser credentials from Vault + ansible.builtin.set_fact: + pg_admin_secrets: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + stack.db.admin_vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + + - name: Ensure application database role exists + community.postgresql.postgresql_user: + name: "{{ stack.db.user }}" + password: "{{ vault_secrets[stack.db.password_vault_key] }}" + login_host: "{{ stack.db.provision_host }}" + login_port: "{{ stack.db.provision_port }}" + login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}" + login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}" + state: present + delegate_to: localhost + become: false + + # Role first, then the database with `owner:` — see the same pair of + # tasks in roles/lxc_app for the full reasoning. The short version is + # that `priv: ALL` on postgresql_user was removed from + # community.postgresql in 4.0.0, and ownership is what the replacement + # should have been anyway. + # + # This instance is pinned to Postgres 13, so unlike the Proxmox one it + # was never *also* broken by the Postgres 15 `public` schema change — + # the removal of `priv` is the only thing that forces the edit here. + # Kept identical to the other two roles regardless: the point of the + # three app roles sharing this block is that an app moving between + # platforms gets the same database either way. + - name: Ensure application database exists + community.postgresql.postgresql_db: + name: "{{ stack.db.name }}" + owner: "{{ stack.db.user }}" + login_host: "{{ stack.db.provision_host }}" + login_port: "{{ stack.db.provision_port }}" + login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}" + login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}" + state: present + delegate_to: localhost + become: false + +- name: Deploy stack with Docker Compose + community.docker.docker_compose_v2: + project_src: "{{ stack_remote_dir }}" + state: present + pull: policy diff --git a/build/config/ansible/roles/compose_stack/tasks/main.yml b/build/config/ansible/roles/compose_stack/tasks/main.yml new file mode 100644 index 0000000..0e58c46 --- /dev/null +++ b/build/config/ansible/roles/compose_stack/tasks/main.yml @@ -0,0 +1,75 @@ +--- +# Converges one stack (one loop iteration of `stack` from playbooks/unraid.yml) +# to the state its host_vars entry asks for. +# +# `state: present` (the default) deploys; `state: absent` tears down. Removal +# has to be asked for explicitly because Ansible keeps no record of what it +# deployed last run — deleting a stack from a host's `stacks:` list only stops +# the loop visiting it, leaving the containers running and unmanaged on the +# host forever. So the entry stays put as a tombstone, with `state: absent`, +# until the teardown has actually been applied. +# +# Unraid-only by design. Proxmox guests install apps natively rather than as +# Compose stacks — see the `lxc_app` role. + +- name: Set stack facts + ansible.builtin.set_fact: + stack_local_dir: "{{ repo_root }}/src/{{ stack.src }}" + stack_remote_dir: "{{ compose_projects_root }}/{{ stack.name }}" + stack_state: "{{ stack.state | default('present') }}" + +- name: Validate requested stack state + ansible.builtin.assert: + that: stack_state in ['present', 'absent'] + fail_msg: >- + Stack '{{ stack.name }}' has state '{{ stack_state }}'; expected + 'present' or 'absent'. + quiet: true + +# Which icons this stack ships, worked out from the repo checkout on the +# controller rather than from anything on the host — `is exists` and the +# `fileglob` lookup both evaluate locally. Computed here rather than in +# deploy.yml because both branches need it: deploy copies these files out, +# remove deletes them again. +# +# Two naming conventions, because Unraid has two icon consumers (see +# deploy.yml for the full explanation): +# +# icon.png the stack — Compose Manager's project row, plus the +# `net.unraid.docker.icon` label of whichever service +# interpolates ${STACK_ICON}. +# icon-.png one container — becomes ${STACK_ICON_}, so a +# multi-container stack can label each of its services +# with its own image instead of sharing the stack's. +# +# `` is the compose service name; the variable is it uppercased with +# `-` folded to `_` (icon-shelfarr-libation.png → STACK_ICON_SHELFARR_LIBATION). +- name: Set stack icon facts + ansible.builtin.set_fact: + # Empty when the stack ships no icon.png — an empty label is what the + # webgui already assumes, so it falls back to the placeholder as before. + stack_icon_dest: >- + {{ ((stack_local_dir ~ '/ansible/unraid/icon.png') is exists) + | ternary(unraid_icons_root ~ '/' ~ stack.name ~ '.png', '') }} + stack_service_icon_names: "{{ _service_icon_names }}" + stack_service_icons: >- + {{ dict(_service_icon_names + | map('upper') | map('replace', '-', '_') + | map('regex_replace', '^(.+)$', 'STACK_ICON_\1') + | zip(_service_icon_names + | map('regex_replace', '^(.+)$', + unraid_icons_root ~ '/' ~ stack.name ~ '-\1.png'))) }} + vars: + _service_icon_names: >- + {{ query('fileglob', stack_local_dir ~ '/ansible/unraid/icon-*.png') + | map('basename') + | map('regex_replace', '^icon-(.+)\.png$', '\1') + | map('lower') | list }} + +- name: Deploy stack + ansible.builtin.include_tasks: deploy.yml + when: stack_state == 'present' + +- name: Remove stack + ansible.builtin.include_tasks: remove.yml + when: stack_state == 'absent' diff --git a/build/config/ansible/roles/compose_stack/tasks/remove.yml b/build/config/ansible/roles/compose_stack/tasks/remove.yml new file mode 100644 index 0000000..0f73bab --- /dev/null +++ b/build/config/ansible/roles/compose_stack/tasks/remove.yml @@ -0,0 +1,106 @@ +--- +# Tears one stack down off the current Unraid host: `docker compose down`, +# then delete the project folder, then optionally drop its database. +# +# Included from main.yml when the stack's `state:` is `absent`. +# +# Ordering matters and is not obvious: `docker compose down` reads the compose +# file to know what it is removing, so the project folder has to survive until +# after that step. Delete the folder (or the stack's `src/` tree) first and +# there is nothing left to tell Docker what belonged to the project — the +# containers and networks have to be cleaned up by hand instead. +# +# What `down` takes with it, and what it deliberately doesn't: +# - removed: the containers, and any network the project itself created +# - kept: bind mounts. Appdata under /mnt/user/appdata/ survives, so +# a removal is reversible by flipping `state:` back to `present`. +# - kept: external networks (caddy-net, unraid_shared) — they belong to +# another stack or to Unraid, not to this project. +# - opt-in: named volumes (`remove_volumes: true`) and images +# (`remove_images: local` or `all`). +# +# Not touched at all: the stack's `homelab/` path in Vault. That has its +# own lifecycle and no reason to be destroyed by a redeployable teardown. + +- name: Check whether the stack's project directory is still present + ansible.builtin.stat: + path: "{{ stack_remote_dir }}/docker-compose.yml" + register: stack_compose_file + +- name: Tear down stack with Docker Compose + community.docker.docker_compose_v2: + project_src: "{{ stack_remote_dir }}" + state: absent + # Named volumes and images are destructive beyond "stop running this + # here", so they stay opt-in per stack rather than being implied by + # `state: absent`. + remove_volumes: "{{ stack.remove_volumes | default(false) }}" + remove_images: "{{ stack.remove_images | default(omit) }}" + remove_orphans: true + # Absent already — a re-run after a successful removal, or a stack that was + # never deployed to this host. Both are the desired end state, not an error. + when: stack_compose_file.stat.exists + +- name: Remove stack project directory + ansible.builtin.file: + path: "{{ stack_remote_dir }}" + state: absent + +# Unlike appdata, these are not state — they're copies of PNGs committed in +# the repo, put there purely so the Docker page had something to render. +# Nothing is lost by deleting them and a redeploy puts them back, so they +# aren't opt-in the way volumes and images are. +# +# Listed from the repo checkout (main.yml) rather than globbed on the host: +# `{{ stack.name }}-*.png` would also match the icons of any stack whose name +# starts with this one's. +- name: Remove stack icons + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: "{{ [unraid_icons_root ~ '/' ~ stack.name ~ '.png'] + + stack_service_icons.values() | list }}" + +- name: Drop application database + when: + - stack.db is defined + - stack.remove_database | default(false) + no_log: true + block: + # Only the Postgres superuser credentials are needed here. The stack's own + # Vault secrets are not read, so a teardown still works after its + # `homelab/` path has been deleted. + - name: Look up Postgres superuser credentials from Vault + ansible.builtin.set_fact: + pg_admin_secrets: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + stack.db.admin_vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + + # Database before role: Postgres refuses to drop a role that still owns + # objects, and the role owns this database. + - name: Drop application database + community.postgresql.postgresql_db: + name: "{{ stack.db.name }}" + login_host: "{{ stack.db.provision_host }}" + login_port: "{{ stack.db.provision_port }}" + login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}" + login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}" + state: absent + delegate_to: localhost + become: false + + - name: Drop application database role + community.postgresql.postgresql_user: + name: "{{ stack.db.user }}" + login_host: "{{ stack.db.provision_host }}" + login_port: "{{ stack.db.provision_port }}" + login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}" + login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}" + state: absent + delegate_to: localhost + become: false diff --git a/build/config/ansible/roles/compose_stack/templates/env.j2 b/build/config/ansible/roles/compose_stack/templates/env.j2 new file mode 100644 index 0000000..108e556 --- /dev/null +++ b/build/config/ansible/roles/compose_stack/templates/env.j2 @@ -0,0 +1,7 @@ +{# Non-secret defaults first, then Vault secrets — secrets win on key clashes. #} +{% for key, value in (env_defaults | default({})).items() %} +{{ key }}="{{ value }}" +{% endfor %} +{% for key, value in (vault_secrets | default({})).items() %} +{{ key }}="{{ value }}" +{% endfor %} diff --git a/build/config/ansible/roles/k3s_app/defaults/main.yml b/build/config/ansible/roles/k3s_app/defaults/main.yml new file mode 100644 index 0000000..36fccbd --- /dev/null +++ b/build/config/ansible/roles/k3s_app/defaults/main.yml @@ -0,0 +1,34 @@ +--- +# Ansible-side knobs for roles/k3s_app — the k3s equivalent of +# compose_stack (Unraid) and lxc_app (Proxmox): the generic role that +# deploys one *application* onto the homelab-utils cluster, as opposed to +# the k3s_* roles that deploy one *cluster service*. +# +# The distinction is worth stating, because the file layout looks similar. +# roles/k3s_metallb, k3s_monitoring, k3s_postgres and k3s_cert_manager each +# hard-code a single thing the cluster itself needs, with their config in +# defaults/ and group_vars. This role hard-codes nothing: it's driven by the +# `k3s_apps:` list in inventory/group_vars/k3s_cluster.yml and reads each +# app's config out of src//, the same src/ tree Unraid and Proxmox +# already deploy from. An app that moves between platforms gets a new +# platform vars file, not a rewrite — see CLAUDE.md → "How config is +# layered". +# +# Everything reaches the cluster the same way every other k3s role does: +# rendering a file into k3s's auto-deploying manifests directory and letting +# k3s's own controllers reconcile it. No helm binary, no kubeconfig, no +# kubernetes.core collection on the controller. + +# Where k3s watches for manifests to apply. Anything written here is applied +# by k3s's deploy controller, and — importantly for remove.yml — anything +# *deleted* from here has its resources torn down again, because the +# controller tracks what each file created via an Addon CR. +k3s_manifests_dir: /var/lib/rancher/k3s/server/manifests + +vault_kv_mount: kv + +# Which directory under src//ansible/ holds the Kubernetes deployment. +# "kubernetes" rather than "k3s": what's in there is plain Kubernetes plus +# k3s's HelmChart CR, and an app moved to a different distribution would +# keep most of it. +k3s_app_platform_dir: kubernetes diff --git a/build/config/ansible/roles/k3s_app/tasks/deploy.yml b/build/config/ansible/roles/k3s_app/tasks/deploy.yml new file mode 100644 index 0000000..6b0211f --- /dev/null +++ b/build/config/ansible/roles/k3s_app/tasks/deploy.yml @@ -0,0 +1,170 @@ +--- +# Deploys one app to the k3s cluster: layer its config, provision its +# database if it declares one, render its secrets into a Secret, then render +# whatever manifests the app ships into k3s's auto-deploying directory. +# +# The shape deliberately mirrors compose_stack/deploy.yml, because the +# separation it enforces is the same one: config that's committed, secrets +# that never are. On Unraid the split is "static docker-compose.yml +# referencing ${VAR}" + "rendered .env"; here it's "manifests carrying only +# non-secret config" + "a rendered Secret the manifests reference by name". +# In both cases the committed half is safe to read and the generated half +# never lands in git. +# +# Included from main.yml when the app's `state:` is `present`; the facts it +# relies on (app_local_dir) are set there. + +- name: Load portable app variables + ansible.builtin.include_vars: + file: "{{ app_local_dir }}/common/vars.yml" + name: app_common_vars + +- name: Load Kubernetes-specific app variables + ansible.builtin.include_vars: + file: "{{ app_local_dir }}/ansible/{{ k3s_app_platform_dir }}/vars.yml" + name: app_platform_vars + +# Portable values first, platform overrides on top — same precedence as +# compose_stack's env_defaults merge, and the same reason: an app's ports and +# database name don't change with the platform, its storage class and +# ingress class do. Vault secrets are *not* merged in here; they go to the +# Secret in a separate task below, so a manifest template can never +# accidentally interpolate one into a world-readable file. +- name: Merge app configuration + ansible.builtin.set_fact: + app_config: >- + {{ (app_common_vars.env_defaults | default({})) + | combine(app_platform_vars.env_defaults | default({})) }} + +- name: Look up app secrets from Vault + ansible.builtin.set_fact: + app_vault_secrets: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + app.vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + no_log: true + +# --- Database --------------------------------------------------------------- +# +# Identical to compose_stack's block, pointed at a different Postgres. That +# it *can* be identical is the whole point of roles/k3s_postgres publishing +# the CNPG primary on a MetalLB LoadBalancer: `community.postgresql` runs on +# the controller (delegate_to: localhost) and needs a real host:port, which +# an in-cluster ClusterIP Service isn't. See CLAUDE.md → "Key decisions" — +# this is the deploy path that entry says is missing. +# +# provision_host is the LoadBalancer address, not the -rw ClusterIP; the app +# itself still talks to the ClusterIP by DNS (see the app's vars.yml), so +# the LAN-facing address is only ever used by the controller at deploy time. +- name: Provision application database + when: app.db is defined + no_log: true + block: + - name: Look up Postgres superuser credentials from Vault + ansible.builtin.set_fact: + pg_admin_secrets: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + app.db.admin_vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + + - name: Ensure application database role exists + community.postgresql.postgresql_user: + name: "{{ app.db.user }}" + password: "{{ app_vault_secrets[app.db.password_vault_key] }}" + login_host: "{{ app.db.provision_host }}" + login_port: "{{ app.db.provision_port }}" + login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}" + login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}" + state: present + delegate_to: localhost + become: false + + # After the role, not before, and with owner: — unlike compose_stack, + # which creates the database first and then grants on it. The difference + # is that this database may be restored into from a pg_dump taken + # elsewhere (see docs/authentik-migration.md): a dump recreates objects + # with their original ownership, which only resolves if the owning role + # already exists and owns the database. Creating it owner-less and + # granting after works for an empty database and quietly leaves a + # restored one owned by postgres. + - name: Ensure application database exists + community.postgresql.postgresql_db: + name: "{{ app.db.name }}" + owner: "{{ app.db.user }}" + login_host: "{{ app.db.provision_host }}" + login_port: "{{ app.db.provision_port }}" + login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}" + login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}" + state: present + delegate_to: localhost + become: false + +# --- Manifests -------------------------------------------------------------- + +# Namespace + Secret in one file, so the namespace an app's manifests target +# is guaranteed to be created by something even if the app ships only a +# HelmChart CR (whose createNamespace: true fires too late for a Secret the +# chart's pods mount). Ordering between files doesn't otherwise matter here: +# k3s's deploy controller retries a manifest whose namespace or CRDs don't +# exist yet rather than failing once and giving up — the same property +# roles/k3s_postgres and roles/k3s_metallb already rely on. +# +# 0600 and no_log because this one carries every value from the app's Vault +# path. Note this is on-disk protection on the node only: the Secret's +# contents are then base64 in etcd like any Kubernetes Secret, which is the +# same trust boundary the rest of this cluster already assumes. +- name: Render the app namespace and secrets manifest + ansible.builtin.template: + src: app-secrets.yaml.j2 + dest: "{{ k3s_manifests_dir }}/{{ app.name }}-secrets.yaml" + owner: root + group: root + mode: "0600" + become: true + no_log: true + +# Whatever the app ships — a HelmChart CR, an Ingress, a PVC, a Certificate. +# Enumerated from the repo checkout on the controller (fileglob evaluates +# locally), so adding a manifest to an app means dropping a .yaml.j2 next to +# the others, with no role change. +# +# Prefixed with the app name on the node, because every app's manifests share +# one flat directory there. Name the files for their content and not for the +# app (helmchart.yaml.j2, not authentik.helmchart.yaml.j2) — the prefix is +# added here, and an app-named file stutters into authentik-authentik.yaml. +- name: Find the app's Kubernetes manifest templates + ansible.builtin.set_fact: + app_manifest_templates: >- + {{ query('fileglob', + app_local_dir ~ '/ansible/' ~ k3s_app_platform_dir ~ '/*.yaml.j2') + | sort }} + +- name: Fail fast if the app ships no manifests + ansible.builtin.assert: + that: app_manifest_templates | length > 0 + fail_msg: >- + App '{{ app.name }}' has no *.yaml.j2 under + src/{{ app.src }}/ansible/{{ k3s_app_platform_dir }}/ — nothing to + deploy. An app on this platform needs at least one manifest (normally + a HelmChart CR). + quiet: true + +- name: Render the app's Kubernetes manifests + ansible.builtin.template: + src: "{{ item }}" + dest: >- + {{ k3s_manifests_dir }}/{{ app.name }}-{{ + item | basename | regex_replace('\.j2$', '') }} + owner: root + group: root + mode: "0644" + loop: "{{ app_manifest_templates }}" + become: true diff --git a/build/config/ansible/roles/k3s_app/tasks/main.yml b/build/config/ansible/roles/k3s_app/tasks/main.yml new file mode 100644 index 0000000..353ba8b --- /dev/null +++ b/build/config/ansible/roles/k3s_app/tasks/main.yml @@ -0,0 +1,37 @@ +--- +# Converges one app (one loop iteration of `app` from playbooks/k3s.yml) to +# the state its k3s_apps: entry asks for. +# +# `state: present` (the default) deploys; `state: absent` tears down. Same +# explicit-removal rule as compose_stack, and for the same reason: Ansible +# keeps no record of what it deployed last run, so deleting an app from +# `k3s_apps:` only stops the loop visiting it — the workload keeps running, +# unmanaged, until something says `absent`. The entry stays as a tombstone. +# +# Unlike compose_stack, removal here does tear the workload down completely +# on the first pass, because k3s's deploy controller owns the resources a +# manifest created and garbage-collects them when the file goes away. What +# it does *not* touch, by the same "a default teardown should be +# reversible" reasoning compose_stack applies: the app's database, its +# PersistentVolumeClaims, and its Vault path. + +- name: Set app facts + ansible.builtin.set_fact: + app_local_dir: "{{ repo_root }}/src/{{ app.src }}" + app_state: "{{ app.state | default('present') }}" + +- name: Validate requested app state + ansible.builtin.assert: + that: app_state in ['present', 'absent'] + fail_msg: >- + App '{{ app.name }}' has state '{{ app_state }}'; expected 'present' + or 'absent'. + quiet: true + +- name: Deploy app + ansible.builtin.include_tasks: deploy.yml + when: app_state == 'present' + +- name: Remove app + ansible.builtin.include_tasks: remove.yml + when: app_state == 'absent' diff --git a/build/config/ansible/roles/k3s_app/tasks/remove.yml b/build/config/ansible/roles/k3s_app/tasks/remove.yml new file mode 100644 index 0000000..01f6f85 --- /dev/null +++ b/build/config/ansible/roles/k3s_app/tasks/remove.yml @@ -0,0 +1,60 @@ +--- +# Tears one app off the cluster: delete the manifests it was deployed from +# and let k3s's deploy controller garbage-collect what they created. +# +# This is the one place k3s is *less* work than Compose. `compose_stack` +# has to run `docker compose down` and therefore needs the compose file to +# still be on disk to know what it's tearing down (which is why removing an +# app there is a two-pass job — see CLAUDE.md → "Removing an app"). Here the +# deploy controller already tracks which resources each manifest file +# created, via the Addon CR it writes alongside them, so deleting the file +# is the teardown. src// can be deleted in the same commit. +# +# Deliberately *not* removed, same reasoning as compose_stack's opt-in +# flags — a default teardown should be reversible: +# +# - the app's database and role on the shared Postgres +# - its PersistentVolumeClaims (the app's manifests own those; if the +# chart's PVCs carry a Helm ownership annotation they go with the +# HelmChart CR, so check `kubectl -n get pvc` after) +# - its Vault path +# - the namespace, which the secrets manifest below creates but the +# controller will only remove if nothing else landed in it +# +# There's no `remove_database`/`remove_volumes` equivalent yet. Add one the +# day it's actually wanted rather than guessing at the shape now. + +- name: Find the app's Kubernetes manifest templates + ansible.builtin.set_fact: + app_manifest_templates: >- + {{ query('fileglob', + app_local_dir ~ '/ansible/' ~ k3s_app_platform_dir ~ '/*.yaml.j2') + | sort }} + +# The secrets manifest first, then the app's own. Order is cosmetic — the +# controller reconciles each file's removal independently — but removing the +# workload's namespace/Secret last would leave pods briefly running without +# the credentials they were started with, and losing them noisily in a log +# is worse than losing them quietly. +- name: Remove the app's Kubernetes manifests + ansible.builtin.file: + path: >- + {{ k3s_manifests_dir }}/{{ app.name }}-{{ + item | basename | regex_replace('\.j2$', '') }} + state: absent + loop: "{{ app_manifest_templates }}" + become: true + +- name: Remove the app namespace and secrets manifest + ansible.builtin.file: + path: "{{ k3s_manifests_dir }}/{{ app.name }}-secrets.yaml" + state: absent + become: true + +- name: Report what removal left behind + ansible.builtin.debug: + msg: >- + App '{{ app.name }}' manifests removed; k3s will garbage-collect the + resources they created. Its database, PVCs and Vault path were left + alone on purpose — remove those by hand if the teardown is meant to be + permanent. diff --git a/build/config/ansible/roles/k3s_app/templates/app-secrets.yaml.j2 b/build/config/ansible/roles/k3s_app/templates/app-secrets.yaml.j2 new file mode 100644 index 0000000..8926c09 --- /dev/null +++ b/build/config/ansible/roles/k3s_app/templates/app-secrets.yaml.j2 @@ -0,0 +1,35 @@ +{# + Managed by Ansible (roles/k3s_app) — do not edit on the node. + + The k3s counterpart of compose_stack's rendered .env: every key from the + app's Vault path, verbatim, in one Secret the app's manifests reference by + name. Keys are not renamed or filtered on the way through, which is what + keeps this role generic — an app decides what its environment looks like by + choosing its Vault keys, exactly as it does on Unraid, and the role stays + ignorant of any particular app's variables. + + The intended consumer is an envFrom/secretRef in the app's own manifest, + so the values become environment variables without ever being named in a + committed file. See src/authentik/ansible/kubernetes/ for the worked + example. + + Namespace lives here rather than in the app's manifests so it's guaranteed + to exist before anything mounts this Secret — a HelmChart CR's + createNamespace: true happens when the chart installs, which is after the + helm-controller job needs somewhere to put it. +-#} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ app_config.K8S_NAMESPACE }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ app.name }}-secrets + namespace: {{ app_config.K8S_NAMESPACE }} +type: Opaque +stringData: +{% for key, value in (app_vault_secrets | default({})) | dictsort %} + {{ key }}: {{ value | string | to_json }} +{% endfor %} diff --git a/build/config/ansible/roles/k3s_cert_manager/defaults/main.yml b/build/config/ansible/roles/k3s_cert_manager/defaults/main.yml new file mode 100644 index 0000000..d760e5f --- /dev/null +++ b/build/config/ansible/roles/k3s_cert_manager/defaults/main.yml @@ -0,0 +1,73 @@ +--- +# Ansible-side knobs for roles/k3s_cert_manager — cert-manager, which issues +# and renews the TLS certificates Ingresses on this cluster serve. Installed +# via k3s's own bundled helm-controller, same mechanism as roles/k3s_metallb, +# k3s_monitoring and k3s_postgres (see tasks/main.yml) — no helm binary or +# extra Ansible collection needed on the controller. +# +# This is the fourth cluster service, and the first one that exists purely +# for apps rather than for the cluster: nothing here needs a certificate, +# roles/k3s_app's tenants do. It's still a cluster service rather than an +# app, because a certificate issuer is shared infrastructure in the same way +# the shared Postgres is — one ClusterIssuer, every app's Ingress annotates +# itself against it. + +k3s_cert_manager_namespace: cert-manager + +k3s_cert_manager_chart_repo: https://charts.jetstack.io + +# Pinned, same reasoning as every other chart version here — a rebuild +# months from now should reproduce today's install, not whatever's newest. +# Bump deliberately; check the current release first at +# https://github.com/cert-manager/cert-manager/releases. +k3s_cert_manager_chart_version: "v1.21.1" + +# The name every app's Ingress annotates itself with +# (cert-manager.io/cluster-issuer: ). A ClusterIssuer rather than a +# per-namespace Issuer, so an app doesn't need its own copy of the ACME +# account and DNS credentials in its own namespace. +k3s_cert_manager_issuer_name: letsencrypt + +# Let's Encrypt's production directory. Swap for the staging URL +# (https://acme-staging-v02.api.letsencrypt.org/directory) while working out +# a solver configuration: production has a hard rate limit of 5 failed +# validations per account/hostname/hour, and burning it means waiting rather +# than retrying. Changing this changes which ACME account the issuer uses, so +# existing certificates are re-issued rather than renewed. +k3s_cert_manager_acme_server: https://acme-v02.api.letsencrypt.org/directory + +# The address Let's Encrypt sends expiry warnings to. No default — an ACME +# account is registered against it, so a wrong value is worth failing on +# rather than guessing. Set in inventory/group_vars/k3s_cluster.yml. +k3s_cert_manager_acme_email: "" + +# Where the ACME account's private key is kept. cert-manager generates it on +# first registration; it is *not* the certificate key, and losing it means +# re-registering rather than losing certificates. +k3s_cert_manager_acme_key_secret: letsencrypt-account-key + +# --- Solver ----------------------------------------------------------------- +# +# How cert-manager proves control of the domain. DNS-01 by default, not +# HTTP-01, because HTTP-01 requires Let's Encrypt to reach this cluster from +# the internet on port 80 — true only if the LAN is port-forwarded, and this +# cluster deliberately isn't. DNS-01 needs no inbound path at all, and is the +# only option that can issue wildcards. +# +# The provider block is left open rather than hard-coding Cloudflare, since +# it's the one part of this role that depends on who runs the DNS. Set +# k3s_cert_manager_solver in inventory/group_vars/k3s_cluster.yml to whatever +# your provider's stanza looks like in cert-manager's docs +# (https://cert-manager.io/docs/configuration/acme/dns01/); it's rendered +# into the ClusterIssuer's solvers list as-is. Any secret it references +# should name the Secret this role renders from Vault (below), so no token is +# ever committed. tasks/main.yml fails fast while this is empty. +k3s_cert_manager_solver: {} + +# The Vault path holding the DNS provider's API credentials. Every key at +# this path becomes a key in a Secret named after it in the cert-manager +# namespace, the same "Vault keys pass through verbatim" rule roles/k3s_app +# uses — so the solver above references whichever key name you stored. +# Leave empty for a solver that needs no credentials (e.g. HTTP-01). +k3s_cert_manager_vault_path: homelab/k3s-cert-manager +k3s_cert_manager_credentials_secret: cert-manager-dns-credentials diff --git a/build/config/ansible/roles/k3s_cert_manager/tasks/main.yml b/build/config/ansible/roles/k3s_cert_manager/tasks/main.yml new file mode 100644 index 0000000..378c622 --- /dev/null +++ b/build/config/ansible/roles/k3s_cert_manager/tasks/main.yml @@ -0,0 +1,75 @@ +--- +# Drops a HelmChart CR for cert-manager, plus a ClusterIssuer and the DNS +# credentials it solves with, into k3s's auto-deploying manifests directory; +# k3s's bundled helm-controller and deploy controller reconcile them — same +# mechanism and same two-manifest shape as roles/k3s_metallb (chart CR + +# plain config manifest referencing CRDs the chart hasn't installed yet). +# +# The ClusterIssuer is exactly that case: cert-manager.io/v1 doesn't exist +# until the chart has installed, so this manifest is unappliable at the +# moment it's written. That's fine and deliberate — k3s's deploy controller +# retries a manifest referencing not-yet-existing CRDs until they show up, +# rather than failing once and giving up. + +- name: Fail fast if cert-manager is not configured + ansible.builtin.assert: + that: + - k3s_cert_manager_acme_email | length > 0 + - k3s_cert_manager_solver | length > 0 + fail_msg: >- + k3s_cert_manager_acme_email and/or k3s_cert_manager_solver are unset — + set both in inventory/group_vars/k3s_cluster.yml before running + playbooks/k3s.yml. An ACME account is registered against the email, and + without a solver the ClusterIssuer would be created but never able to + prove domain control, leaving every Certificate pending indefinitely + rather than failing loudly. + quiet: true + run_once: true + +- name: Deploy the cert-manager HelmChart manifest + ansible.builtin.template: + src: cert-manager.helmchart.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/cert-manager.yaml + owner: root + group: root + mode: "0644" + become: true + +# Before the ClusterIssuer, because that's what references it. Ordering +# between files isn't actually enforced (see the header) — this is for the +# reader, not the controller. +- name: Look up the DNS provider credentials from Vault + when: k3s_cert_manager_vault_path | length > 0 + ansible.builtin.set_fact: + cert_manager_vault_secrets: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + k3s_cert_manager_vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + no_log: true + +# Separate file and mode from everything else here, same split +# roles/k3s_postgres makes: this is the only manifest carrying a credential, +# so it's the only one that isn't safe at 0644. +- name: Deploy the DNS provider credentials Secret + when: k3s_cert_manager_vault_path | length > 0 + ansible.builtin.template: + src: dns-credentials.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/cert-manager-dns-credentials.yaml + owner: root + group: root + mode: "0600" + become: true + no_log: true + +- name: Deploy the ClusterIssuer manifest + ansible.builtin.template: + src: cluster-issuer.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/cert-manager-cluster-issuer.yaml + owner: root + group: root + mode: "0644" + become: true diff --git a/build/config/ansible/roles/k3s_cert_manager/templates/cert-manager.helmchart.yaml.j2 b/build/config/ansible/roles/k3s_cert_manager/templates/cert-manager.helmchart.yaml.j2 new file mode 100644 index 0000000..a83ce1a --- /dev/null +++ b/build/config/ansible/roles/k3s_cert_manager/templates/cert-manager.helmchart.yaml.j2 @@ -0,0 +1,59 @@ +{# + Managed by Ansible (roles/k3s_cert_manager) — do not edit on the node. + + A HelmChart CR, same pattern as roles/k3s_metallb and roles/k3s_monitoring: + k3s's bundled helm-controller reconciles it, the same mechanism it uses to + install its own Traefik. +-#} +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: cert-manager + namespace: kube-system +spec: + chart: cert-manager + repo: {{ k3s_cert_manager_chart_repo }} + version: "{{ k3s_cert_manager_chart_version }}" + targetNamespace: {{ k3s_cert_manager_namespace }} + createNamespace: true + valuesContent: |- + # The chart installs its own CRDs. Without this they'd have to be applied + # separately before the chart, which the helm-controller gives no ordering + # hook for — and the ClusterIssuer this role also renders would then never + # become appliable. + crds: + enabled: true + + # Sized for a Raspberry Pi 4, same reasoning as k3s_monitoring_values and + # k3s_postgres_resources. cert-manager is idle almost all of the time — + # it wakes to renew a certificate every 60 days — so these are set for + # "doesn't get OOM-killed during a renewal", not for throughput. + resources: + requests: + cpu: 10m + memory: 64Mi + limits: + memory: 128Mi + webhook: + resources: + requests: + cpu: 10m + memory: 64Mi + limits: + memory: 128Mi + cainjector: + resources: + requests: + cpu: 10m + memory: 96Mi + limits: + memory: 192Mi + + # DNS-01 self-check queries the authoritative nameservers directly rather + # than going through the cluster's resolver. Without this, CoreDNS + # forwards to whatever the LAN's DNS is, and a split-horizon setup — + # where the internal view of the zone doesn't carry the _acme-challenge + # TXT record the public view does — makes cert-manager wait for a record + # it will never see, until the order times out. + dns01RecursiveNameservers: "1.1.1.1:53,9.9.9.9:53" + dns01RecursiveNameserversOnly: true diff --git a/build/config/ansible/roles/k3s_cert_manager/templates/cluster-issuer.yaml.j2 b/build/config/ansible/roles/k3s_cert_manager/templates/cluster-issuer.yaml.j2 new file mode 100644 index 0000000..d433719 --- /dev/null +++ b/build/config/ansible/roles/k3s_cert_manager/templates/cluster-issuer.yaml.j2 @@ -0,0 +1,30 @@ +{# + Managed by Ansible (roles/k3s_cert_manager) — do not edit on the node. + + Plain manifest, not a HelmChart — same pattern as + roles/k3s_metallb/templates/metallb-config.yaml.j2 and + roles/k3s_postgres/templates/postgres-cluster.yaml.j2: this references a + CRD (cert-manager.io) that only exists once the chart alongside it has + installed, and k3s's deploy controller retries rather than giving up. + + ClusterIssuer rather than Issuer so it's usable from every namespace — + an app's Ingress just annotates itself with + `cert-manager.io/cluster-issuer: {{ k3s_cert_manager_issuer_name }}` and + needs no ACME account or DNS credentials of its own. +-#} +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: {{ k3s_cert_manager_issuer_name }} +spec: + acme: + server: {{ k3s_cert_manager_acme_server }} + email: {{ k3s_cert_manager_acme_email }} + # Where cert-manager keeps the ACME *account* key it generates on first + # registration — not any certificate's key. In the cert-manager namespace + # because a ClusterIssuer's secrets always resolve there, regardless of + # which namespace the Certificate using it lives in. + privateKeySecretRef: + name: {{ k3s_cert_manager_acme_key_secret }} + solvers: +{{ [k3s_cert_manager_solver] | to_nice_yaml(indent=2) | indent(6, first=true) }} diff --git a/build/config/ansible/roles/k3s_cert_manager/templates/dns-credentials.yaml.j2 b/build/config/ansible/roles/k3s_cert_manager/templates/dns-credentials.yaml.j2 new file mode 100644 index 0000000..8713644 --- /dev/null +++ b/build/config/ansible/roles/k3s_cert_manager/templates/dns-credentials.yaml.j2 @@ -0,0 +1,23 @@ +{# + Managed by Ansible (roles/k3s_cert_manager) — do not edit on the node. + + The DNS provider API credentials the ACME DNS-01 solver authenticates with, + passed through from Vault verbatim — same rule roles/k3s_app's Secret + follows, so the key names in k3s_cert_manager_solver are whatever you chose + when populating the Vault path, and this role stays ignorant of which + provider is in use. + + In the cert-manager namespace because a ClusterIssuer resolves every Secret + it references there, never in the namespace of the Certificate being + issued. +-#} +apiVersion: v1 +kind: Secret +metadata: + name: {{ k3s_cert_manager_credentials_secret }} + namespace: {{ k3s_cert_manager_namespace }} +type: Opaque +stringData: +{% for key, value in (cert_manager_vault_secrets | default({})) | dictsort %} + {{ key }}: {{ value | string | to_json }} +{% endfor %} diff --git a/build/config/ansible/roles/k3s_maintenance/tasks/main.yml b/build/config/ansible/roles/k3s_maintenance/tasks/main.yml new file mode 100644 index 0000000..57fe07e --- /dev/null +++ b/build/config/ansible/roles/k3s_maintenance/tasks/main.yml @@ -0,0 +1,100 @@ +--- +# Rolling "reboot this node if unattended-upgrades left one pending" for one +# k3s node — one iteration of the serial: 1 loop in +# playbooks/k3s_maintenance.yml. roles/unattended_upgrades installs updates on +# its own schedule with Automatic-Reboot disabled, so a kernel/library update +# can sit applied-but-inactive on a node indefinitely; this is what actually +# reboots it — cordoned and drained first, so workloads move off before the +# node disappears rather than during. +# +# kubectl commands are delegated to the control-plane node and run as +# `k3s kubectl`, k3s's own bundled client — no separate kubectl install or +# local kubeconfig needed. Works the same whether the node having its turn +# right now *is* the control plane: delegating to itself, over SSH, before it +# reboots itself. + +- name: Sanity-check there is exactly one control-plane node + ansible.builtin.assert: + that: groups['k3s_control_plane'] | length == 1 + fail_msg: >- + k3s_maintenance delegates kubectl to groups['k3s_control_plane'][0] — + it assumes a single control-plane node. Update this role before adding + a second one for HA. + quiet: true + +- name: Check whether a reboot is required + ansible.builtin.stat: + path: /var/run/reboot-required + register: k3s_reboot_required + +- name: Node is up to date — nothing to do + ansible.builtin.debug: + msg: "{{ inventory_hostname }}: no reboot required, skipping." + when: not k3s_reboot_required.stat.exists + +- name: Reboot this node if unattended-upgrades left one pending + when: k3s_reboot_required.stat.exists + block: + # --force: this is a homelab, not a cluster with a policy against bare + # pods — better to evict them than have a stray one block every rebuild. + # --delete-emptydir-data: emptyDir contents are expected to be + # disposable; anything that isn't shouldn't be using emptyDir. + - name: Cordon and drain the node + ansible.builtin.command: + argv: + - k3s + - kubectl + - drain + - "{{ inventory_hostname }}" + - --ignore-daemonsets + - --delete-emptydir-data + - --force + - --timeout=120s + delegate_to: "{{ groups['k3s_control_plane'][0] }}" + become: false + changed_when: true + + - name: Reboot the node + ansible.builtin.reboot: + reboot_timeout: 300 + + # Polls rather than trusting the reboot handshake alone — the node can be + # reachable over SSH before k3s (and, if this is the control-plane node + # itself, the API server it just took down with it) has finished coming + # back up. + - name: Wait for the node to report Ready again + ansible.builtin.command: + argv: + - k3s + - kubectl + - wait + - --for=condition=Ready + - "node/{{ inventory_hostname }}" + - --timeout=20s + delegate_to: "{{ groups['k3s_control_plane'][0] }}" + become: false + register: k3s_node_ready + changed_when: false + failed_when: false + until: k3s_node_ready.rc == 0 + retries: 12 + delay: 15 + + - name: Fail if the node never came back Ready + ansible.builtin.fail: + msg: >- + {{ inventory_hostname }} rebooted but never reported Ready again — + it's left cordoned; check it by hand before re-running this against + the rest of the cluster. + when: k3s_node_ready.rc != 0 + + - name: Uncordon the node + ansible.builtin.command: + argv: [k3s, kubectl, uncordon, "{{ inventory_hostname }}"] + delegate_to: "{{ groups['k3s_control_plane'][0] }}" + become: false + changed_when: true + + - name: Node rebooted and rejoined the cluster + ansible.builtin.debug: + msg: "{{ inventory_hostname }}: rebooted, drained and uncordoned cleanly." diff --git a/build/config/ansible/roles/k3s_metallb/defaults/main.yml b/build/config/ansible/roles/k3s_metallb/defaults/main.yml new file mode 100644 index 0000000..45f4f83 --- /dev/null +++ b/build/config/ansible/roles/k3s_metallb/defaults/main.yml @@ -0,0 +1,31 @@ +--- +# Ansible-side knobs for roles/k3s_metallb — MetalLB, the LoadBalancer +# implementation for the homelab-utils cluster. Installed via k3s's own +# bundled helm-controller, same mechanism as roles/k3s_monitoring (see +# tasks/main.yml) — no helm binary or extra Ansible collection needed on the +# controller. +# +# MetalLB replaces k3s's bundled ServiceLB (Klipper), it doesn't sit +# alongside it — both would otherwise try to satisfy the same LoadBalancer +# Services and fight over IPs. That's why `k3s_extra_args` in +# inventory/group_vars/k3s_cluster.yml carries `--disable=servicelb`. + +k3s_metallb_namespace: metallb-system + +k3s_metallb_chart_repo: https://metallb.github.io/metallb + +# Pinned, same reasoning as k3s_monitoring_chart_version — a rebuild months +# from now should reproduce today's stack, not whatever's newest at the +# time. Bump deliberately; check the current release first at +# https://github.com/metallb/metallb/releases. +k3s_metallb_chart_version: "0.14.9" + +# The pool of IPs MetalLB hands out to LoadBalancer-type Services, as a +# "first-last" range or CIDR (e.g. "192.168.50.240-192.168.50.250"). Must be +# addresses on the cluster's LAN that nothing else — DHCP, static +# assignments, the hosts in inventory/hosts.yml — will ever claim: MetalLB +# doesn't coordinate with your router, it just hands out whatever's in this +# range. No default on purpose; set it in +# inventory/group_vars/k3s_cluster.yml before the first run. tasks/main.yml +# fails fast if it's still empty. +k3s_metallb_address_range: "" diff --git a/build/config/ansible/roles/k3s_metallb/tasks/main.yml b/build/config/ansible/roles/k3s_metallb/tasks/main.yml new file mode 100644 index 0000000..f79b717 --- /dev/null +++ b/build/config/ansible/roles/k3s_metallb/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Drops a HelmChart CR for MetalLB, plus its IPAddressPool/L2Advertisement +# config, into k3s's auto-deploying manifests directory; k3s's bundled +# helm-controller and deploy controller reconcile them — same mechanism as +# roles/k3s_monitoring (see its tasks/main.yml for why this needs no helm +# binary or kubeconfig on the controller). +# +# The config manifest references CRDs that only exist once the HelmChart +# above has actually installed the chart, so it's templated in the same +# pass rather than gated behind a "wait for CRDs" step: k3s's deploy +# controller retries a manifest referencing not-yet-existing CRDs until they +# show up, instead of failing once and giving up. + +- name: Fail fast if no MetalLB address range is configured + ansible.builtin.assert: + that: + - k3s_metallb_address_range | length > 0 + fail_msg: >- + k3s_metallb_address_range is empty — set it in + inventory/group_vars/k3s_cluster.yml before running playbooks/k3s.yml. + run_once: true + +- name: Deploy the MetalLB HelmChart manifest + ansible.builtin.template: + src: metallb.helmchart.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/metallb.yaml + owner: root + group: root + mode: "0644" + become: true + +- name: Deploy the MetalLB address pool config + ansible.builtin.template: + src: metallb-config.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/metallb-config.yaml + owner: root + group: root + mode: "0644" + become: true diff --git a/build/config/ansible/roles/k3s_metallb/templates/metallb-config.yaml.j2 b/build/config/ansible/roles/k3s_metallb/templates/metallb-config.yaml.j2 new file mode 100644 index 0000000..6cf85af --- /dev/null +++ b/build/config/ansible/roles/k3s_metallb/templates/metallb-config.yaml.j2 @@ -0,0 +1,26 @@ +# Managed by Ansible (roles/k3s_metallb) — do not edit on the node. +# +# Plain manifests, not a HelmChart — these are MetalLB's own CRs, not chart +# values. k3s's deploy controller applies anything dropped in this +# directory, not only HelmChart CRs, and retries until the CRDs the +# HelmChart alongside this file installs actually exist. +# +# L2 (ARP/NDP) mode, not BGP — the simplest fit for a flat home LAN with no +# BGP-speaking router. One pool, one advertisement, both named "default". +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: default + namespace: {{ k3s_metallb_namespace }} +spec: + addresses: + - {{ k3s_metallb_address_range }} +--- +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: default + namespace: {{ k3s_metallb_namespace }} +spec: + ipAddressPools: + - default diff --git a/build/config/ansible/roles/k3s_metallb/templates/metallb.helmchart.yaml.j2 b/build/config/ansible/roles/k3s_metallb/templates/metallb.helmchart.yaml.j2 new file mode 100644 index 0000000..630d2a0 --- /dev/null +++ b/build/config/ansible/roles/k3s_metallb/templates/metallb.helmchart.yaml.j2 @@ -0,0 +1,18 @@ +# Managed by Ansible (roles/k3s_metallb) — do not edit on the node. +# +# A HelmChart CR, same pattern as +# roles/k3s_monitoring/templates/kube-prometheus-stack.helmchart.yaml.j2 — +# k3s's bundled helm-controller reconciles it, the same mechanism it uses to +# install its own bundled Traefik and ServiceLB. No helm binary, kubeconfig, +# or extra Ansible collection needed on the controller. +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: metallb + namespace: kube-system +spec: + chart: metallb + repo: {{ k3s_metallb_chart_repo }} + version: "{{ k3s_metallb_chart_version }}" + targetNamespace: {{ k3s_metallb_namespace }} + createNamespace: true diff --git a/build/config/ansible/roles/k3s_monitoring/defaults/main.yml b/build/config/ansible/roles/k3s_monitoring/defaults/main.yml new file mode 100644 index 0000000..909dd99 --- /dev/null +++ b/build/config/ansible/roles/k3s_monitoring/defaults/main.yml @@ -0,0 +1,86 @@ +--- +# Ansible-side knobs for roles/k3s_monitoring — the kube-prometheus-stack +# Helm chart, deployed via k3s's own bundled helm-controller rather than a +# helm binary or the kubernetes.core collection (see tasks/main.yml). Lean by +# design: OpenLens's Metrics feature reads straight off the Prometheus API +# through the k8s API server proxy, so there's no need for Grafana or +# Alertmanager in-cluster — both are disabled below, mainly to save RAM on +# the Pis. + +k3s_monitoring_namespace: monitoring + +k3s_monitoring_chart_repo: https://prometheus-community.github.io/helm-charts + +# Pinned, same reasoning as k3s_version in inventory/group_vars/k3s_cluster.yml +# — a rebuild months from now should reproduce today's stack, not whatever's +# newest at the time. Bump deliberately; check the current release first at +# https://github.com/prometheus-community/helm-charts/releases (tags look +# like kube-prometheus-stack-X.Y.Z). +k3s_monitoring_chart_version: "88.3.0" + +# Handed to the chart verbatim as valuesContent (see templates/). Notes on +# the less obvious choices: +# - grafana/alertmanager: off. OpenLens supplies its own dashboards and +# this cluster doesn't page anyone, so both would just be extra Pi RAM. +# - kubeControllerManager/kubeScheduler/kubeProxy/kubeEtcd: off. k3s bundles +# the control plane inside one static binary instead of exposing these as +# separate systemd units on their usual ports, so kube-prometheus-stack's +# default scrape targets for them sit permanently "down" — a known false +# alarm on k3s, not a sign anything's actually broken. kubelet (and the +# cAdvisor/node metrics it serves) is the one k3s does expose normally, +# and it's also the one OpenLens's node/pod metrics actually need. +# - prometheus retention/storage: short retention, no PVC template (so it +# runs on emptyDir). This stack exists to answer "what are the nodes +# doing right now", not to keep months of history, so losing the TSDB on +# a pod restart is an acceptable trade for not standing up persistent +# storage across 4 SD cards. +# - every component's resources: sized for a Raspberry Pi 4, not a +# datacenter node. +k3s_monitoring_values: + grafana: + enabled: false + alertmanager: + enabled: false + kubeApiServer: + enabled: true + kubeControllerManager: + enabled: false + kubeScheduler: + enabled: false + kubeProxy: + enabled: false + kubeEtcd: + enabled: false + kubelet: + enabled: true + prometheusOperator: + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + memory: 128Mi + prometheus: + prometheusSpec: + retention: 3d + scrapeInterval: 30s + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 512Mi + kube-state-metrics: + resources: + requests: + cpu: 20m + memory: 32Mi + limits: + memory: 64Mi + prometheus-node-exporter: + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + memory: 32Mi diff --git a/build/config/ansible/roles/k3s_monitoring/tasks/main.yml b/build/config/ansible/roles/k3s_monitoring/tasks/main.yml new file mode 100644 index 0000000..fc05a3c --- /dev/null +++ b/build/config/ansible/roles/k3s_monitoring/tasks/main.yml @@ -0,0 +1,16 @@ +--- +# Drops a HelmChart CR into k3s's auto-deploying manifests directory on the +# control-plane node; k3s's bundled helm-controller reconciles it, the same +# mechanism it uses to install its own bundled Traefik and ServiceLB. See +# templates/kube-prometheus-stack.helmchart.yaml.j2 for why this needs no +# helm binary or extra Ansible collection, and README.md "Metrics +# (Prometheus)" for how to point OpenLens at the result. + +- name: Deploy the kube-prometheus-stack HelmChart manifest + ansible.builtin.template: + src: kube-prometheus-stack.helmchart.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/kube-prometheus-stack.yaml + owner: root + group: root + mode: "0644" + become: true diff --git a/build/config/ansible/roles/k3s_monitoring/templates/kube-prometheus-stack.helmchart.yaml.j2 b/build/config/ansible/roles/k3s_monitoring/templates/kube-prometheus-stack.helmchart.yaml.j2 new file mode 100644 index 0000000..25ed6f7 --- /dev/null +++ b/build/config/ansible/roles/k3s_monitoring/templates/kube-prometheus-stack.helmchart.yaml.j2 @@ -0,0 +1,24 @@ +--- +# Managed by Ansible (roles/k3s_monitoring) — do not edit on the node. +# +# A HelmChart CR, not a `helm install`: k3s ships its own helm-controller +# that watches /var/lib/rancher/k3s/server/manifests/ (this file's +# destination) the same way it watches for any other auto-deploying +# manifest, and reconciles HelmChart resources found there — the same +# mechanism k3s uses to install its own bundled Traefik and ServiceLB. That +# means no helm binary, kubeconfig, or extra Ansible collection is needed on +# the controller; re-running the playbook just re-templates this file and +# the controller reconciles the diff. +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: kube-prometheus-stack + namespace: kube-system +spec: + chart: kube-prometheus-stack + repo: {{ k3s_monitoring_chart_repo }} + version: "{{ k3s_monitoring_chart_version }}" + targetNamespace: {{ k3s_monitoring_namespace }} + createNamespace: true + valuesContent: | +{{ k3s_monitoring_values | to_nice_yaml(indent=2) | indent(4, first=true) }} diff --git a/build/config/ansible/roles/k3s_node/defaults/main.yml b/build/config/ansible/roles/k3s_node/defaults/main.yml new file mode 100644 index 0000000..ce38dd6 --- /dev/null +++ b/build/config/ansible/roles/k3s_node/defaults/main.yml @@ -0,0 +1,18 @@ +--- +# Ubuntu Server's boot partition. Same path on Raspberry Pi OS, so this +# doesn't need to change if the Pis are ever reimaged to that instead. +k3s_boot_cmdline_path: /boot/firmware/cmdline.txt + +# k3s (like any Kubernetes distro) needs the memory cgroup controller, which +# isn't always on by default on a Pi kernel. Ubuntu's Pi images usually ship +# these already — this task is a safety net for that assumption, not the +# primary mechanism, and a no-op on most runs. See tasks/prep.yml. +k3s_cgroup_params: + - cgroup_memory=1 + - cgroup_enable=memory + +# Safety-net defaults for inventory/group_vars/k3s_cluster.yml's extra-args +# vars (see tasks/server.yml, tasks/agent.yml) — empty so the role doesn't +# break if a var is left undefined there. +k3s_extra_args: [] +k3s_server_extra_args: [] diff --git a/build/config/ansible/roles/k3s_node/tasks/agent.yml b/build/config/ansible/roles/k3s_node/tasks/agent.yml new file mode 100644 index 0000000..b2c7014 --- /dev/null +++ b/build/config/ansible/roles/k3s_node/tasks/agent.yml @@ -0,0 +1,100 @@ +--- +# Joins this node as a k3s agent (worker). Only ever runs after server.yml has +# succeeded somewhere — playbooks/k3s.yml targets k3s_control_plane before +# k3s_workers — so K3S_URL below always points at an already-live API server. +# +# --node-name pins the k8s node object to the Ansible inventory_hostname — +# see the matching comment in server.yml, same reason. + +- name: Bootstrap the k3s agent + no_log: true # K3S_TOKEN passes through this block + block: + - name: Look up the k3s cluster secrets + ansible.builtin.set_fact: + k3s_secrets: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + k3s_vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + + - name: Compute the desired k3s agent exec line + ansible.builtin.set_fact: + k3s_agent_exec: "agent --node-name {{ inventory_hostname }} {{ k3s_extra_args | join(' ') }}" + + # Compared against what k3s was last installed with (see the copy task + # below) so a change to k3s_extra_args gets applied on the next run + # instead of silently sitting unused: the version check below has no way + # to notice an exec-line-only change. Missing file (first install) counts + # as changed. + - name: Read the exec line k3s was last installed with + ansible.builtin.slurp: + src: /etc/rancher/k3s/.ansible_install_exec + register: k3s_installed_exec_raw + failed_when: false + check_mode: false + + - name: Determine whether the exec line has changed + ansible.builtin.set_fact: + k3s_exec_changed: >- + {{ k3s_installed_exec_raw.content is not defined + or (k3s_installed_exec_raw.content | b64decode) != k3s_agent_exec }} + + - name: Check the installed k3s version + ansible.builtin.command: k3s --version + register: k3s_installed_version + changed_when: false + failed_when: false + check_mode: false + + # The installer is safe to re-run — it's a no-op if the requested version + # and exec line are already active — but skipping it when neither changed + # avoids restarting the agent (and briefly dropping the node's kubelet) + # on every playbook run. Restarting the k3s-agent process itself (as + # opposed to rebooting the node, which roles/k3s_maintenance handles + # separately) doesn't touch already-running pods — containerd keeps them + # up underneath it. playbooks/k3s.yml still runs this play with + # serial: 1, so at most one node's kubelet is ever bouncing at a time. + - name: Install/upgrade k3s agent + ansible.builtin.shell: curl -sfL https://get.k3s.io | sh - + environment: + INSTALL_K3S_VERSION: "{{ k3s_version }}" + INSTALL_K3S_EXEC: "{{ k3s_agent_exec }}" + # The one control-plane node, addressed by its inventory IP rather + # than delegating a lookup to it — simple and correct as long as this + # cluster stays single-server. Revisit if it ever gets HA control + # plane nodes. + K3S_URL: "https://{{ hostvars[groups['k3s_control_plane'][0]].ansible_host }}:6443" + K3S_TOKEN: "{{ k3s_secrets.K3S_TOKEN }}" + when: >- + k3s_installed_version.rc != 0 + or k3s_version not in k3s_installed_version.stdout + or k3s_exec_changed + +- name: Ensure the k3s-agent service is enabled and running + ansible.builtin.systemd_service: + name: k3s-agent + enabled: true + state: started + +# k3s creates /etc/rancher/k3s itself once it has something to put there +# (a config.yaml, the generated kubeconfig on the server) — an agent with no +# extra config doesn't necessarily end up with anything else prompting that, +# so this marker can't assume the directory already exists. +- name: Ensure /etc/rancher/k3s exists + ansible.builtin.file: + path: /etc/rancher/k3s + state: directory + owner: root + group: root + mode: "0755" + +- name: Record the exec line k3s was installed with + ansible.builtin.copy: + dest: /etc/rancher/k3s/.ansible_install_exec + content: "{{ k3s_agent_exec }}" + owner: root + group: root + mode: "0600" diff --git a/build/config/ansible/roles/k3s_node/tasks/main.yml b/build/config/ansible/roles/k3s_node/tasks/main.yml new file mode 100644 index 0000000..ecbd30d --- /dev/null +++ b/build/config/ansible/roles/k3s_node/tasks/main.yml @@ -0,0 +1,24 @@ +--- +# Converges one node of the cluster: the same prep either way, then branch on +# which k3s role it plays. k3s_node_role is set per group, not per host — see +# inventory/group_vars/k3s_control_plane.yml (server) and k3s_workers.yml +# (agent) — so adding a fifth Pi later is a hosts.yml edit, not a role change. + +- name: Validate k3s_node_role + ansible.builtin.assert: + that: k3s_node_role in ['server', 'agent'] + fail_msg: >- + k3s_node_role is '{{ k3s_node_role }}'; expected 'server' or 'agent'. + Check inventory/group_vars/k3s_control_plane.yml and k3s_workers.yml. + quiet: true + +- name: Prepare the node for k3s + ansible.builtin.include_tasks: prep.yml + +- name: Install the k3s server (control plane) + ansible.builtin.include_tasks: server.yml + when: k3s_node_role == 'server' + +- name: Install the k3s agent (worker) + ansible.builtin.include_tasks: agent.yml + when: k3s_node_role == 'agent' diff --git a/build/config/ansible/roles/k3s_node/tasks/prep.yml b/build/config/ansible/roles/k3s_node/tasks/prep.yml new file mode 100644 index 0000000..47913b2 --- /dev/null +++ b/build/config/ansible/roles/k3s_node/tasks/prep.yml @@ -0,0 +1,80 @@ +--- +# OS-level prerequisites, identical for a server and an agent node. Runs +# before either install so a from-scratch Pi (fresh Ubuntu Server image, SSH +# + the `ansible` user already set up) can go straight to a working cluster +# in one playbook run — the "easy to rebuild" part of the design. + +- name: Read the current kernel boot parameters + ansible.builtin.command: cat {{ k3s_boot_cmdline_path }} + register: k3s_cmdline_current + changed_when: false + check_mode: false + +- name: Work out which cgroup parameters are missing + ansible.builtin.set_fact: + k3s_cmdline_missing: >- + {{ k3s_cgroup_params | reject('in', k3s_cmdline_current.stdout) | list }} + +# cmdline.txt is one line, space-separated — rewritten whole rather than +# appended in place, since there's no line-based anchor to insert after. +# Existing file mode is left alone (no `mode:` here) rather than guessed at. +- name: Add missing cgroup parameters to the boot command line + ansible.builtin.copy: + dest: "{{ k3s_boot_cmdline_path }}" + content: >- + {{ (k3s_cmdline_current.stdout.split() + k3s_cmdline_missing) | join(' ') }} + owner: root + group: root + register: k3s_cmdline_updated + when: k3s_cmdline_missing | length > 0 + +- name: Reboot to apply updated boot parameters + ansible.builtin.reboot: + reboot_timeout: 300 + when: k3s_cmdline_updated is changed + +- name: Check active swap devices + ansible.builtin.command: swapon --summary + register: k3s_swap_active + changed_when: false + check_mode: false + +- name: Turn off active swap + ansible.builtin.command: swapoff -a + when: k3s_swap_active.stdout | trim | length > 0 + +- name: Comment out swap entries in fstab + ansible.builtin.replace: + path: /etc/fstab + regexp: '^([^#\n]*\sswap\s.*)$' + replace: '# \1' + +# Best-effort: Ubuntu's zram-backed swap ships under different unit names +# across releases, and most won't be present at all. A missing unit is not a +# failure here — only an already-active one that we failed to disable would +# leave swap coming back on the next boot, and swapoff -a above already +# handles the running instance for this boot. +# +# `failed_when: false` rather than `ignore_errors: true`: systemd_service +# raises "Could not find the requested service" as a hard module failure +# when the unit is absent, and that's reported here as a genuine task +# failure regardless of ignore_errors — failed_when overrides the result +# directly instead of trying to catch it after the fact. +- name: Disable Ubuntu's zram-backed swap, if present + ansible.builtin.systemd_service: + name: "{{ item }}" + enabled: false + state: stopped + loop: + - zram-config.service + - systemd-zram-setup@zram0.service + register: k3s_zram_disable + failed_when: false + changed_when: k3s_zram_disable is changed + +- name: Ensure curl is installed + ansible.builtin.apt: + name: curl + state: present + update_cache: true + cache_valid_time: 3600 diff --git a/build/config/ansible/roles/k3s_node/tasks/server.yml b/build/config/ansible/roles/k3s_node/tasks/server.yml new file mode 100644 index 0000000..3c838e0 --- /dev/null +++ b/build/config/ansible/roles/k3s_node/tasks/server.yml @@ -0,0 +1,103 @@ +--- +# Bootstraps this node as the (single) k3s control plane. playbooks/k3s.yml +# runs the k3s_control_plane play before k3s_workers, so by the time +# agent.yml runs anywhere, K3S_URL below already points at something live. +# +# The token comes from Vault as a fixed, pre-shared value rather than letting +# k3s generate one on first install — see docs/vault-secrets.md +# (homelab/k3s-homelab-utils). That's what makes a full rebuild (wipe both SD +# cards, reinstall) reproduce the same cluster identity instead of needing the +# new token hunted down and re-distributed by hand. +# +# --node-name pins the k8s node object to the Ansible inventory_hostname +# rather than whatever the OS hostname happens to be — roles/k3s_maintenance +# addresses nodes by inventory_hostname when draining/uncordoning, and that +# only works if the two names are guaranteed to match. + +- name: Bootstrap the k3s server + no_log: true # K3S_TOKEN passes through this block + block: + - name: Look up the k3s cluster secrets + ansible.builtin.set_fact: + k3s_secrets: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + k3s_vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + + - name: Compute the desired k3s server exec line + ansible.builtin.set_fact: + k3s_server_exec: >- + server --node-name {{ inventory_hostname }} --tls-san {{ k3s_api_tls_san }} + --write-kubeconfig-mode 644 {{ (k3s_extra_args + k3s_server_extra_args) | join(' ') }} + + # Compared against what k3s was last installed with (see the copy task + # below) so a change to k3s_extra_args/k3s_server_extra_args — or + # k3s_api_tls_san — gets applied on the next run instead of silently + # sitting unused: the version check below has no way to notice an + # exec-line-only change. Missing file (first install) counts as changed. + - name: Read the exec line k3s was last installed with + ansible.builtin.slurp: + src: /etc/rancher/k3s/.ansible_install_exec + register: k3s_installed_exec_raw + failed_when: false + check_mode: false + + - name: Determine whether the exec line has changed + ansible.builtin.set_fact: + k3s_exec_changed: >- + {{ k3s_installed_exec_raw.content is not defined + or (k3s_installed_exec_raw.content | b64decode) != k3s_server_exec }} + + - name: Check the installed k3s version + ansible.builtin.command: k3s --version + register: k3s_installed_version + changed_when: false + failed_when: false + check_mode: false + + # The installer is safe to re-run — it's a no-op if the requested version + # and exec line are already active — but skipping it when neither changed + # avoids restarting the service (and briefly dropping the API) on every + # playbook run. Restarting the k3s process itself (as opposed to + # rebooting the node, which roles/k3s_maintenance handles separately) is + # a brief control-plane/kubelet blip, not a pod outage — containerd keeps + # every already-running pod up underneath it. + - name: Install/upgrade k3s server + ansible.builtin.shell: curl -sfL https://get.k3s.io | sh - + environment: + INSTALL_K3S_VERSION: "{{ k3s_version }}" + INSTALL_K3S_EXEC: "{{ k3s_server_exec }}" + K3S_TOKEN: "{{ k3s_secrets.K3S_TOKEN }}" + when: >- + k3s_installed_version.rc != 0 + or k3s_version not in k3s_installed_version.stdout + or k3s_exec_changed + +- name: Ensure the k3s service is enabled and running + ansible.builtin.systemd_service: + name: k3s + enabled: true + state: started + +# k3s creates /etc/rancher/k3s itself once it has something to put there +# (the generated kubeconfig, on the server) — defensive rather than relied +# on, so this marker doesn't assume the directory already exists. +- name: Ensure /etc/rancher/k3s exists + ansible.builtin.file: + path: /etc/rancher/k3s + state: directory + owner: root + group: root + mode: "0755" + +- name: Record the exec line k3s was installed with + ansible.builtin.copy: + dest: /etc/rancher/k3s/.ansible_install_exec + content: "{{ k3s_server_exec }}" + owner: root + group: root + mode: "0600" diff --git a/build/config/ansible/roles/k3s_postgres/defaults/main.yml b/build/config/ansible/roles/k3s_postgres/defaults/main.yml new file mode 100644 index 0000000..2094fd7 --- /dev/null +++ b/build/config/ansible/roles/k3s_postgres/defaults/main.yml @@ -0,0 +1,71 @@ +--- +# Ansible-side knobs for roles/k3s_postgres — CloudNativePG (CNPG), the +# Postgres operator for the homelab-utils cluster's shared database. +# Installed via k3s's own bundled helm-controller, same mechanism as +# roles/k3s_metallb and roles/k3s_monitoring (see tasks/main.yml) — no helm +# binary or extra Ansible collection needed on the controller. +# +# Chosen over Bitnami's postgresql-ha (repmgr+pgpool, more moving parts per +# pod, and Bitnami's free chart/image catalog was restructured into a +# "legacy" repo in 2025 — a risky long-term dependency) and the Zalando +# operator (mature but Patroni-based, heavier, and its own config +# conventions fit less naturally into this repo's lean HelmChart-CR-only +# pattern). CNPG installs as one operator chart plus one plain Cluster CR — +# the same "HelmChart + plain config manifest" shape already used for +# MetalLB (chart + IPAddressPool). +# +# The app-portable half of this config (instance count, storage size/class) +# lives with the app it belongs to instead: see +# src/shared/postgres/ansible/kubernetes/vars.yml, loaded into `pg_config` +# by tasks/main.yml — same common/vars.yml + platform-vars.yml split +# compose_stack and lxc_app use for every other app. + +k3s_postgres_operator_namespace: cnpg-system + +k3s_postgres_chart_repo: https://cloudnative-pg.github.io/charts + +# Pinned, same reasoning as k3s_monitoring_chart_version/k3s_metallb_chart_version +# — a rebuild months from now should reproduce today's operator, not +# whatever's newest at the time. Bump deliberately; check the current +# release first at https://github.com/cloudnative-pg/charts/releases. +k3s_postgres_chart_version: "0.29.0" + +# Where the actual Postgres Cluster (not the operator) lives. +k3s_postgres_namespace: shared-postgres + +# Postgres major/minor pinned via the operand image, independent of the +# chart/operator version above. This is a separate physical instance from +# the Docker-based shared/postgres (pinned to 13 there — see +# src/shared/postgres/common/vars.yml), so there's no need to match; pin to +# a current stable major instead. Multi-arch (amd64/arm64) upstream, same +# as every other image this cluster runs. +k3s_postgres_image: ghcr.io/cloudnative-pg/postgresql:18.4 + +# Same Vault path the Unraid/Proxmox shared-postgres instances already use +# (see docs/vault-secrets.md) — a separate physical instance, but one +# superuser identity for the "shared postgres" concept everywhere it runs. +k3s_postgres_vault_path: homelab/shared/postgres + +# The LAN address the primary is published on, via the LoadBalancer Service +# declared in templates/postgres-cluster.yaml.j2. Must be inside +# k3s_metallb_address_range — MetalLB only assigns from its own pools, and a +# request for anything outside them leaves the Service pending forever rather +# than failing loudly. +# +# Empty here on purpose, same reasoning as k3s_metallb_address_range's own +# default: the address is a property of the LAN, not of this role, so it's set +# in inventory/group_vars/k3s_cluster.yml next to the pool it has to fall +# inside. Unlike the pool, leaving it empty is survivable — the Service is +# still created and MetalLB assigns the next free address; you just don't know +# which one until you look, which is fine for in-cluster clients and no good +# for anything that has to write the address down. +k3s_postgres_loadbalancer_ip: "" + +# Resource requests/limits per instance — sized for a Raspberry Pi 4, not a +# datacenter node, same reasoning as k3s_monitoring_values. +k3s_postgres_resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 512Mi diff --git a/build/config/ansible/roles/k3s_postgres/tasks/main.yml b/build/config/ansible/roles/k3s_postgres/tasks/main.yml new file mode 100644 index 0000000..867300a --- /dev/null +++ b/build/config/ansible/roles/k3s_postgres/tasks/main.yml @@ -0,0 +1,91 @@ +--- +# Deploys the shared Postgres cluster onto k3s: install CloudNativePG (the +# operator) via a HelmChart CR, then define the actual cluster as a plain +# Cluster CR — same two-manifest pattern as roles/k3s_metallb (chart CR + +# plain config manifest), for the same reason: k3s's deploy controller +# retries a manifest referencing CRDs the HelmChart above hasn't installed +# yet instead of failing outright, so the Cluster CR doesn't need to wait on +# the operator chart finishing first. +# +# Config is layered the same way compose_stack/lxc_app layer an app's +# common/vars.yml + platform vars.yml — src/shared/postgres/ is the same +# shared service already used on Unraid/Proxmox, just deployed a third way +# here. Secrets come from the same Vault path those platforms already use +# (homelab/shared/postgres) — this is a separate physical instance, but +# reuses the path rather than inventing a k3s-specific one, same "one path +# per app-concept" convention as every other stack. + +- name: Set shared-postgres source facts + ansible.builtin.set_fact: + pg_local_dir: "{{ repo_root }}/src/shared/postgres" + +- name: Load portable shared-postgres variables + ansible.builtin.include_vars: + file: "{{ pg_local_dir }}/common/vars.yml" + name: pg_common_vars + +- name: Load Kubernetes-specific shared-postgres variables + ansible.builtin.include_vars: + file: "{{ pg_local_dir }}/ansible/kubernetes/vars.yml" + name: pg_platform_vars + +- name: Merge shared-postgres configuration + ansible.builtin.set_fact: + pg_config: >- + {{ (pg_common_vars.env_defaults | default({})) + | combine(pg_platform_vars.env_defaults | default({})) }} + +- name: Look up shared-postgres superuser credentials from Vault + ansible.builtin.set_fact: + pg_vault_secrets: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + k3s_postgres_vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + no_log: true + +- name: Deploy the CloudNativePG operator HelmChart manifest + ansible.builtin.template: + src: cnpg-operator.helmchart.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/cnpg-operator.yaml + owner: root + group: root + mode: "0644" + become: true + +# Namespace + Cluster CR, no secrets — safe to render even though the +# Cluster CR references the Secret below, for the same "later file, retried +# reconcile" reasoning as the Secret referencing this file's Namespace. +- name: Deploy the shared-postgres Cluster manifest + ansible.builtin.template: + src: postgres-cluster.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/shared-postgres-cluster.yaml + owner: root + group: root + mode: "0644" + become: true + +# Separate file (and mode) from the Cluster manifest above purely because +# this one carries the Vault-sourced password — everything else here is safe +# at 0644, this one isn't. +# +# The dest filename deliberately doesn't track the Secret's own name, which +# gained a `-vault` suffix (see the template's header for why). k3s's deploy +# controller records which resources each manifest *file* created and prunes +# the ones that disappear from it, so renaming the Secret inside this same +# file makes the old `shared-postgres-superuser` object get garbage-collected +# on the next run. Renaming the file too would orphan it instead: Ansible +# doesn't remove files it no longer writes, so the old manifest would sit +# there keeping the stale Secret alive. +- name: Deploy the shared-postgres superuser Secret + ansible.builtin.template: + src: postgres-superuser-secret.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/shared-postgres-superuser-secret.yaml + owner: root + group: root + mode: "0600" + become: true + no_log: true diff --git a/build/config/ansible/roles/k3s_postgres/templates/cnpg-operator.helmchart.yaml.j2 b/build/config/ansible/roles/k3s_postgres/templates/cnpg-operator.helmchart.yaml.j2 new file mode 100644 index 0000000..053a2ae --- /dev/null +++ b/build/config/ansible/roles/k3s_postgres/templates/cnpg-operator.helmchart.yaml.j2 @@ -0,0 +1,21 @@ +# Managed by Ansible (roles/k3s_postgres) — do not edit on the node. +# +# A HelmChart CR, same pattern as +# roles/k3s_metallb/templates/metallb.helmchart.yaml.j2 and +# roles/k3s_monitoring — k3s's bundled helm-controller reconciles it, the +# same mechanism it uses to install its own bundled Traefik and ServiceLB. +# No helm binary, kubeconfig, or extra Ansible collection needed on the +# controller. This installs the CloudNativePG operator and its CRDs only — +# the actual Postgres cluster is templates/postgres-cluster.yaml.j2, a plain +# manifest applied once the CRD it references exists (see that file). +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: cloudnative-pg + namespace: kube-system +spec: + chart: cloudnative-pg + repo: {{ k3s_postgres_chart_repo }} + version: "{{ k3s_postgres_chart_version }}" + targetNamespace: {{ k3s_postgres_operator_namespace }} + createNamespace: true diff --git a/build/config/ansible/roles/k3s_postgres/templates/postgres-cluster.yaml.j2 b/build/config/ansible/roles/k3s_postgres/templates/postgres-cluster.yaml.j2 new file mode 100644 index 0000000..2f82be7 --- /dev/null +++ b/build/config/ansible/roles/k3s_postgres/templates/postgres-cluster.yaml.j2 @@ -0,0 +1,104 @@ +# Managed by Ansible (roles/k3s_postgres) — do not edit on the node. +# +# Plain manifests, not a HelmChart — same pattern as +# roles/k3s_metallb/templates/metallb-config.yaml.j2: the Cluster CR below +# references a CRD (postgresql.cnpg.io) that only exists once the +# cloudnative-pg HelmChart (cnpg-operator.yaml, same manifests directory) has +# actually installed the operator, so it's applied here rather than gated +# behind a "wait for CRDs" step — k3s's deploy controller retries a manifest +# referencing not-yet-existing CRDs until they show up, instead of failing +# once and giving up. templates/postgres-superuser-secret.yaml.j2 relies on +# the Namespace created here the same way, for the same reason. +apiVersion: v1 +kind: Namespace +metadata: + name: {{ k3s_postgres_namespace }} +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: shared-postgres + namespace: {{ k3s_postgres_namespace }} +spec: + # 1 primary + 1 replica (src/shared/postgres/ansible/kubernetes/vars.yml). + # CNPG's own default pod anti-affinity (preferred, topology key hostname) + # spreads them across distinct nodes; the nodeAffinity below narrows which + # nodes those can be. + instances: {{ pg_config.POSTGRES_INSTANCES }} + imageName: {{ k3s_postgres_image }} + + # Password-based superuser login, from the Secret Ansible renders + # alongside this file. Off by default in CNPG — without this, the operator + # ignores the secret's content and sets the postgres user's password to + # NULL, disabling remote login as it. + # + # The name must stay clear of `-superuser` (i.e. + # `shared-postgres-superuser`), which is what CNPG calls the secret it + # generates for itself when this stanza is absent. Claiming that name for + # our own object stops password changes reaching the database — see + # postgres-superuser-secret.yaml.j2 for the full symptom. + enableSuperuserAccess: true + superuserSecret: + name: shared-postgres-superuser-vault + + # A LoadBalancer Service for the primary, on top of the ClusterIP -rw/-ro/-r + # Services CNPG creates for every Cluster. Declared here as a CNPG *managed + # service* rather than as a Service manifest of our own: selectorType: rw + # means the operator maintains the selector, so the LB follows a failover to + # the other instance the same way the built-in -rw Service does. A + # hand-written Service would need its selector re-pointed by hand after + # every promotion. + # + # This is what makes the cluster reachable from the Ansible controller at a + # fixed host:port, which is the thing per-app database provisioning needs — + # the community.postgresql tasks compose_stack/lxc_app use can't talk to a + # ClusterIP, and `kubectl port-forward` isn't a stable address. It does put + # the superuser on the LAN: acceptable here for the same reason the Unraid + # instance's published port is, and the LAN is the boundary either way. + managed: + services: + additional: + - selectorType: rw + serviceTemplate: + metadata: + name: shared-postgres-lb +{% if k3s_postgres_loadbalancer_ip %} + # Pinned rather than left to MetalLB's next-free pick, because + # this address ends up in config elsewhere (inventory, an app's + # vars.yml) and shouldn't move when the Service is recreated. + annotations: + metallb.universe.tf/loadBalancerIPs: "{{ k3s_postgres_loadbalancer_ip }}" +{% endif %} + spec: + type: LoadBalancer + # No ports: — the operator fills in Postgres's own (5432) from + # the same template it builds the -rw Service from; naming them + # here would only risk disagreeing with it. + + # POSTGRES_PORT (common/vars.yml) is 5432, the same port CNPG always + # listens on inside the pod/Service — nothing to override here. + storage: + size: {{ pg_config.POSTGRES_STORAGE_SIZE }} + storageClass: {{ pg_config.POSTGRES_STORAGE_CLASS }} + + resources: +{{ k3s_postgres_resources | to_nice_yaml(indent=2) | indent(4, first=true) }} + + # Workers only — k3s-ctrl-01 stays free of app pods, same boundary the + # cluster already keeps for monitoring/MetalLB. + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: DoesNotExist + + # No bootstrap: stanza — CNPG's default initdb bootstrap creates a + # `postgres` superuser (above) plus a default `app` database owned by an + # auto-generated `app` role/secret. That default app database is unused + # today: no app is deployed onto k3s yet, and real per-app database + # provisioning here is a deferred design problem (see CLAUDE.md → "Key + # decisions") — the community.postgresql approach compose_stack/lxc_app + # use needs a controller-reachable host:port, which this cluster's + # in-cluster -rw Service isn't without further work. diff --git a/build/config/ansible/roles/k3s_postgres/templates/postgres-superuser-secret.yaml.j2 b/build/config/ansible/roles/k3s_postgres/templates/postgres-superuser-secret.yaml.j2 new file mode 100644 index 0000000..38c3aed --- /dev/null +++ b/build/config/ansible/roles/k3s_postgres/templates/postgres-superuser-secret.yaml.j2 @@ -0,0 +1,49 @@ +# Managed by Ansible (roles/k3s_postgres) — do not edit on the node. +# +# Superuser credentials for the shared-postgres Cluster +# (postgres-cluster.yaml.j2), sourced from the same Vault path +# (homelab/shared/postgres) the Unraid/Proxmox shared-postgres instances +# already use — see docs/vault-secrets.md. CNPG requires +# `type: kubernetes.io/basic-auth` with username/password keys, and +# `enableSuperuserAccess: true` on the Cluster (set there) for this secret to +# actually be usable for password login rather than just ignored. +# +# The `-vault` suffix is load-bearing, and removing it reintroduces a bug +# that is very hard to read off the symptoms. CNPG generates its own +# superuser secret named `-superuser` when the Cluster doesn't name +# one — for a Cluster called `shared-postgres` that is exactly +# `shared-postgres-superuser`, the name this file used to claim. Writing our +# own object at the operator's reserved name meant the operator treated it as +# one it had already authored and reconciled, so a password change in Vault +# propagated as far as this Secret and stopped there: the role in the +# database kept whatever was set at bootstrap. Every check short of querying +# pg_authid agrees the change landed, while `psql` from the LAN keeps +# returning "password authentication failed for user postgres". Any name +# outside the `-*` pattern the operator owns avoids it. +# +# Namespace comes from postgres-cluster.yaml.j2, a separate file in the same +# manifests directory — see that file's header for why applying this before +# the Namespace exists isn't a problem. +# +# The username is layered the same way every other value in this repo is — +# Vault on top, common/vars.yml underneath — rather than required from Vault. +# It isn't a secret (only the password is), and the Vault path is allowed to +# carry just POSTGRES_SUPERUSER_PASSWORD; setting POSTGRES_SUPERUSER there +# still wins, which is what docs/vault-secrets.md means by "Vault is +# authoritative since it's paired with the password". +apiVersion: v1 +kind: Secret +metadata: + name: shared-postgres-superuser-vault + namespace: {{ k3s_postgres_namespace }} +type: kubernetes.io/basic-auth +# to_json, not bare interpolation: it emits a double-quoted YAML scalar with +# escaping handled, so a password containing `#`, `:`, a leading `*`/`&`, or +# leading/trailing whitespace survives intact. Unquoted, YAML silently +# reinterprets those (`#` starts a comment, an all-digit password becomes an +# int) and the Secret ends up holding something other than what Vault has — +# which fails authentication while every value still looks right in playbook +# output. Alphanumeric-only passwords hide the problem rather than fix it. +stringData: + username: {{ (pg_vault_secrets.POSTGRES_SUPERUSER | default(pg_config.POSTGRES_SUPERUSER)) | to_json }} + password: {{ pg_vault_secrets.POSTGRES_SUPERUSER_PASSWORD | to_json }} diff --git a/build/config/ansible/roles/k3s_traefik/defaults/main.yml b/build/config/ansible/roles/k3s_traefik/defaults/main.yml new file mode 100644 index 0000000..af3c0ae --- /dev/null +++ b/build/config/ansible/roles/k3s_traefik/defaults/main.yml @@ -0,0 +1,115 @@ +--- +# Ansible-side knobs for roles/k3s_traefik — the ingress controller k3s +# bundles. The odd one out among the k3s_* cluster services: every other one +# installs a chart this repo chose (roles/k3s_metallb, k3s_monitoring, +# k3s_postgres, k3s_cert_manager), whereas Traefik is already installed by +# k3s itself before Ansible ever connects. So this role installs nothing — +# it only adjusts what's there, via a HelmChartConfig (see +# templates/traefik.helmchartconfig.yaml.j2 for why that, and not an edit to +# k3s's own traefik.yaml). +# +# Scope today is the dashboard, which a stock k3s serves 404s for. That's +# not a broken install: Traefik builds the dashboard regardless +# (`api.dashboard` defaults on), but the Traefik chart stopped creating the +# router that reaches it in v28, so there is nothing routing /dashboard/ on +# any entrypoint until something puts that router back. + +# Where k3s installs Traefik, and so where the HelmChartConfig has to live — +# a HelmChartConfig only applies to the HelmChart of the same name in the +# same namespace. +k3s_traefik_namespace: kube-system + +# --- The dashboard, reachable by port-forward -------------------------------- +# +# Re-enables the chart's own dashboard IngressRoute, which binds to Traefik's +# internal `traefik` entrypoint (port 9000). That entrypoint is not published +# on Traefik's Service, so this exposes nothing to the LAN; it makes exactly +# one thing work: +# +# kubectl -n kube-system port-forward deploy/traefik 9000:9000 +# # then http://127.0.0.1:9000/dashboard/ — the trailing slash is required +# +# Left on even when the dashboard is also exposed on a hostname below, and +# that's deliberate rather than redundant. Once the exposed route is behind +# Authentik, the dashboard depends on Authentik being up, which depends on +# CNPG being up, which depends on the cluster being healthy — precisely the +# things you would open the dashboard to diagnose. Port-forward talks to the +# pod and traverses none of it, so it stays as the break-glass path. Don't +# "tidy this up" once the hostname works. +k3s_traefik_dashboard_enabled: true + +# --- The dashboard, exposed on a hostname ------------------------------------ +# +# Empty (the default) means the port-forward above is the only way in, and +# tasks/main.yml renders no IngressRoute at all — it removes one it previously +# rendered, so clearing this is a real teardown rather than a no-op. +# +# Set it to a hostname (e.g. traefik.turtlesystems.uk) to publish the +# dashboard on the `websecure` entrypoint with a cert-manager certificate and +# the forward-auth middleware below in front of it. Doing so requires +# k3s_traefik_dashboard_auth_address to be set as well — the role refuses to +# run otherwise, because the failure mode of getting this wrong is a +# read-only view of every route, service and middleware in the cluster +# published unauthenticated to the LAN. +# +# The hostname needs a DNS record pointing at Traefik's MetalLB address +# (`kubectl -n kube-system get svc traefik`), same as any app's Ingress. +k3s_traefik_dashboard_host: "" + +# The ClusterIssuer roles/k3s_cert_manager creates. Must match +# k3s_cert_manager_issuer_name — stated rather than referenced because that +# lives in the other role's defaults, which aren't in scope here. Same +# convention as CERT_ISSUER in src/authentik/ansible/kubernetes/vars.yml. +# +# An explicit Certificate rather than an Ingress annotation, unlike every app +# on this cluster: cert-manager watches Ingress resources, and the dashboard +# can't be one. It's served by Traefik's internal `api@internal` service, +# which has no Kubernetes Service for an Ingress to point at — hence an +# IngressRoute, which cert-manager doesn't watch. +k3s_traefik_dashboard_cert_issuer: letsencrypt +k3s_traefik_dashboard_tls_secret: traefik-dashboard-tls + +# --- Authentication in front of the exposed dashboard ------------------------ +# +# A Traefik forwardAuth middleware pointing at Authentik, which now runs on +# this cluster (roles/k3s_app, src/authentik). No default, and required as +# soon as k3s_traefik_dashboard_host is set: a default here would be a guess +# at another app's service name, and a wrong guess that happens to be +# unreachable is a 500, not an open dashboard — but one that happens to +# resolve to the wrong thing is worse. Set it explicitly, next to the host, +# in inventory/group_vars/k3s_cluster.yml. +# +# For Authentik's embedded outpost, in-cluster: +# +# http://authentik-server.authentik.svc.cluster.local/outpost.goauthentik.io/auth/traefik +# +# The provider in Authentik must be a Proxy Provider in **forward auth +# (domain level)** mode, not single-application. Single-application mode +# additionally requires /outpost.goauthentik.io/ to be routed to Authentik on +# *this* host, which from an IngressRoute in kube-system means referencing a +# Service in the authentik namespace — a cross-namespace reference Traefik +# rejects unless started with providers.kubernetescrd.allowCrossNamespace, +# which it isn't. Domain-level mode handles the redirect on Authentik's own +# hostname instead and needs no such route, so it's the mode that fits. +k3s_traefik_dashboard_auth_address: "" + +k3s_traefik_dashboard_auth_middleware: dashboard-auth + +# What Authentik sets on the way back through, forwarded to the dashboard. +# The dashboard itself reads none of them — it has no notion of a user — so +# this is really about the headers existing for anything else that reuses +# this middleware later, and about matching Authentik's documented list +# rather than inventing a shorter one. +k3s_traefik_dashboard_auth_response_headers: + - X-authentik-username + - X-authentik-groups + - X-authentik-entitlements + - X-authentik-email + - X-authentik-name + - X-authentik-uid + - X-authentik-jwt + - X-authentik-meta-jwks + - X-authentik-meta-outpost + - X-authentik-meta-provider + - X-authentik-meta-app + - X-authentik-meta-version diff --git a/build/config/ansible/roles/k3s_traefik/tasks/main.yml b/build/config/ansible/roles/k3s_traefik/tasks/main.yml new file mode 100644 index 0000000..8fd2686 --- /dev/null +++ b/build/config/ansible/roles/k3s_traefik/tasks/main.yml @@ -0,0 +1,59 @@ +--- +# Adjusts the Traefik k3s installs for itself, rather than installing +# anything: a HelmChartConfig merged over k3s's own Traefik HelmChart, plus — +# only when the dashboard is being published on a hostname — the +# Certificate/Middleware/IngressRoute trio that puts it behind Authentik. +# See defaults/main.yml and templates/traefik.helmchartconfig.yaml.j2. +# +# Same delivery mechanism as every other k3s_* role: template a file into +# /var/lib/rancher/k3s/server/manifests/ and let k3s's bundled helm-controller +# and deploy controller reconcile it. No helm binary, kubeconfig or extra +# collection on the controller. + +- name: Fail fast if the dashboard would be exposed without authentication + when: k3s_traefik_dashboard_host | length > 0 + ansible.builtin.assert: + that: + - k3s_traefik_dashboard_auth_address | length > 0 + fail_msg: >- + k3s_traefik_dashboard_host is set but k3s_traefik_dashboard_auth_address + is empty — set both in inventory/group_vars/k3s_cluster.yml, or neither. + Publishing the dashboard without the forward-auth middleware would put a + read-only view of every router, service and middleware on this cluster + on the LAN unauthenticated, so this refuses rather than defaulting to an + address that might not be Authentik. + quiet: true + run_once: true + +- name: Deploy the Traefik HelmChartConfig + ansible.builtin.template: + src: traefik.helmchartconfig.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/traefik-config.yaml + owner: root + group: root + mode: "0644" + become: true + +- name: Deploy the exposed dashboard route + when: k3s_traefik_dashboard_host | length > 0 + ansible.builtin.template: + src: dashboard-ingressroute.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/traefik-dashboard.yaml + owner: root + group: root + mode: "0644" + become: true + +# Clearing k3s_traefik_dashboard_host is a teardown, not just a stop-managing: +# k3s's deploy controller garbage-collects the objects a manifest created when +# the manifest is removed, so deleting this file withdraws the public route, +# its middleware and its certificate. Same reasoning as roles/k3s_app's +# removal path — with the difference that this one needs no `state: absent` +# tombstone, because the hostname is a single value rather than a list entry +# that could be silently dropped. +- name: Withdraw the exposed dashboard route when no host is configured + when: k3s_traefik_dashboard_host | length == 0 + ansible.builtin.file: + path: /var/lib/rancher/k3s/server/manifests/traefik-dashboard.yaml + state: absent + become: true diff --git a/build/config/ansible/roles/k3s_traefik/templates/dashboard-ingressroute.yaml.j2 b/build/config/ansible/roles/k3s_traefik/templates/dashboard-ingressroute.yaml.j2 new file mode 100644 index 0000000..e869918 --- /dev/null +++ b/build/config/ansible/roles/k3s_traefik/templates/dashboard-ingressroute.yaml.j2 @@ -0,0 +1,83 @@ +{# + Managed by Ansible (roles/k3s_traefik) — do not edit on the node. + + Plain manifests, not a HelmChart — same pattern as + roles/k3s_metallb/templates/metallb-config.yaml.j2 and + roles/k3s_cert_manager/templates/cluster-issuer.yaml.j2: CRs belonging to + charts installed elsewhere, dropped in the auto-deploying directory and + retried by k3s's deploy controller until the CRDs they need exist. + + Rendered only when k3s_traefik_dashboard_host is set; tasks/main.yml + deletes this file when it isn't, and k3s's deploy controller + garbage-collects what the file created. + + traefik.io/v1alpha1, not traefik.containo.us/v1alpha1 — the group changed + with Traefik v3, which is what current k3s bundles. +-#} +{# Explicit Certificate rather than the cert-manager.io/cluster-issuer + annotation every app's Ingress uses: cert-manager watches Ingress + resources, and this route can't be one (see below). DNS-01, so this + issues without the hostname resolving anywhere yet. -#} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: traefik-dashboard + namespace: {{ k3s_traefik_namespace }} +spec: + secretName: {{ k3s_traefik_dashboard_tls_secret }} + issuerRef: + name: {{ k3s_traefik_dashboard_cert_issuer }} + kind: ClusterIssuer + dnsNames: + - {{ k3s_traefik_dashboard_host }} +--- +{# Authentik, in forward-auth (domain level) mode — see defaults/main.yml for + why domain level and not single-application. Fails closed: if Authentik is + down this returns an error rather than passing the request through, which + is the correct direction and the reason the port-forward path stays. -#} +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: {{ k3s_traefik_dashboard_auth_middleware }} + namespace: {{ k3s_traefik_namespace }} +spec: + forwardAuth: + address: {{ k3s_traefik_dashboard_auth_address }} + # Traefik strips X-Forwarded-* from client requests by default; Authentik + # needs them to know which host and scheme the user actually asked for, + # and builds its redirect back out of them. Safe here because the only + # thing that can set them is Traefik itself — nothing reaches this + # middleware without passing through the entrypoint first. + trustForwardHeader: true + authResponseHeaders: +{% for header in k3s_traefik_dashboard_auth_response_headers %} + - {{ header }} +{% endfor %} +--- +{# An IngressRoute rather than an Ingress, and not by preference: the + dashboard is served by api@internal, a Traefik-internal service with no + Kubernetes Service behind it, so there is nothing for an Ingress backend + to name. That single fact is why the Certificate above is explicit and why + the middleware is attached here rather than by annotation. -#} +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: traefik-dashboard + namespace: {{ k3s_traefik_namespace }} +spec: + entryPoints: + - websecure + routes: + # Both prefixes, because the dashboard is a static bundle under + # /dashboard/ that reads its data from /api — serving the first without + # the second gets you a page that loads and then stays empty. + - kind: Rule + match: Host(`{{ k3s_traefik_dashboard_host }}`) && (PathPrefix(`/dashboard`) || PathPrefix(`/api`)) + middlewares: + - name: {{ k3s_traefik_dashboard_auth_middleware }} + namespace: {{ k3s_traefik_namespace }} + services: + - kind: TraefikService + name: api@internal + tls: + secretName: {{ k3s_traefik_dashboard_tls_secret }} diff --git a/build/config/ansible/roles/k3s_traefik/templates/traefik.helmchartconfig.yaml.j2 b/build/config/ansible/roles/k3s_traefik/templates/traefik.helmchartconfig.yaml.j2 new file mode 100644 index 0000000..f70814e --- /dev/null +++ b/build/config/ansible/roles/k3s_traefik/templates/traefik.helmchartconfig.yaml.j2 @@ -0,0 +1,34 @@ +{# + Managed by Ansible (roles/k3s_traefik) — do not edit on the node. + + A HelmChartConfig, not a HelmChart, because this is the one chart on the + cluster this repo doesn't install: k3s installs Traefik itself, rewriting + /var/lib/rancher/k3s/server/manifests/traefik.yaml on every server start + and overwriting whatever is in it. A HelmChartConfig carrying the same + name and namespace as that HelmChart is the supported way in — k3s's + helm-controller merges its valuesContent over the chart's values on the + next reconcile. Editing traefik.yaml directly works until the next + restart, and then silently doesn't. + + Hence also the filename this renders to (traefik-config.yaml): anything + called traefik.yaml here belongs to k3s and would be reclaimed. +-#} +apiVersion: helm.cattle.io/v1 +kind: HelmChartConfig +metadata: + name: traefik + namespace: {{ k3s_traefik_namespace }} +spec: + valuesContent: |- + ingressRoute: + dashboard: + # Traefik builds the dashboard either way — `api.dashboard` is on by + # default. What the chart stopped shipping in v28 is the router that + # reaches it, which is why a stock k3s answers 404 to + # :9000/dashboard/ while :9000/ping happily returns 200. This puts + # that router back, on Traefik's internal `traefik` entrypoint + # (9000), which isn't published on the Service — so it is reachable + # by `kubectl port-forward` and nothing else. See defaults/main.yml + # for why that stays true even once the dashboard is also exposed on + # a hostname. + enabled: {{ k3s_traefik_dashboard_enabled | bool | lower }} diff --git a/build/config/ansible/roles/lxc_app/defaults/main.yml b/build/config/ansible/roles/lxc_app/defaults/main.yml new file mode 100644 index 0000000..a4f0e89 --- /dev/null +++ b/build/config/ansible/roles/lxc_app/defaults/main.yml @@ -0,0 +1,2 @@ +--- +vault_kv_mount: kv diff --git a/build/config/ansible/roles/lxc_app/tasks/main.yml b/build/config/ansible/roles/lxc_app/tasks/main.yml new file mode 100644 index 0000000..b1007e0 --- /dev/null +++ b/build/config/ansible/roles/lxc_app/tasks/main.yml @@ -0,0 +1,124 @@ +--- +# Installs one app (one loop iteration of `app` from playbooks/proxmox.yml) +# natively into a Proxmox LXC: merge portable + Proxmox config, fetch secrets +# from Vault, provision its database if it declares one, run the app's own +# install steps, then manage its systemd unit. +# +# The Proxmox counterpart to `compose_stack`. Everything up to "Run +# app-specific install steps" is deliberately the same shape as that role — +# same vars merge, same Vault lookup, same DB provisioning — so config lives +# in one place regardless of which platform an app lands on. What differs is +# the deployment primitive: a systemd service built from packages/binaries +# rather than `docker compose up -d`. +# +# UNPROVEN: `src/shared/postgres/` and `src/forgejo/` both ship an +# `ansible/proxmox/install.yml` now, but neither has been run end to end +# against a real LXC. The structure mirrors proven code; the details are not +# yet exercised. + +- name: Set app facts + ansible.builtin.set_fact: + app_local_dir: "{{ repo_root }}/src/{{ app.src }}" + +- name: Load portable app variables + ansible.builtin.include_vars: + file: "{{ app_local_dir }}/common/vars.yml" + name: app_common_vars + +- name: Load Proxmox-specific app variables + ansible.builtin.include_vars: + file: "{{ app_local_dir }}/ansible/proxmox/vars.yml" + name: app_platform_vars + +- name: Merge app configuration + ansible.builtin.set_fact: + # Portable values first, Proxmox-specific overrides on top. Available to + # the app's install.yml, alongside `vault_secrets` below. + app_config: >- + {{ (app_common_vars.env_defaults | default({})) + | combine(app_platform_vars.env_defaults | default({})) }} + +- name: Look up app secrets from Vault + ansible.builtin.set_fact: + vault_secrets: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + app.vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + no_log: true + +- name: Provision application database + when: app.db is defined + no_log: true + block: + - name: Look up Postgres superuser credentials from Vault + ansible.builtin.set_fact: + pg_admin_secrets: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + app.db.admin_vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + + - name: Ensure application database role exists + community.postgresql.postgresql_user: + name: "{{ app.db.user }}" + password: "{{ vault_secrets[app.db.password_vault_key] }}" + login_host: "{{ app.db.provision_host }}" + login_port: "{{ app.db.provision_port }}" + login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}" + login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}" + state: present + delegate_to: localhost + become: false + + # Role first, then the database with `owner:` — the shape roles/k3s_app + # already uses, and not the `postgresql_user` + `priv: ALL` this role was + # written with. Two independent reasons it had to change: + # + # 1. `priv` was deprecated in community.postgresql 3.x and REMOVED in + # 4.0.0, so the old call is a hard "Unsupported parameters" failure + # on any current collection. requirements.yml asks for >=3.0.0, + # which installs 4.x. + # 2. On this instance it would have been wrong even if it still worked. + # `priv: ALL` grants database-level privileges (CONNECT, CREATE, + # TEMPORARY); since Postgres 15 the `public` schema no longer grants + # CREATE to PUBLIC, so a role holding all of those still cannot + # create a table. The Proxmox cluster is 17 (the Unraid one is 13, + # which is why the same code never failed that way there). Making + # the app role own the database covers it on both: on 15+ `public` + # is owned by `pg_database_owner`, which resolves to whoever owns + # the database. + # + # Ownership also makes a pg_dump restore land correctly, for the reason + # spelled out at the same tasks in roles/k3s_app. + - name: Ensure application database exists + community.postgresql.postgresql_db: + name: "{{ app.db.name }}" + owner: "{{ app.db.user }}" + login_host: "{{ app.db.provision_host }}" + login_port: "{{ app.db.provision_port }}" + login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}" + login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}" + state: present + delegate_to: localhost + become: false + +# Everything genuinely app-specific — fetching the binary or package, creating +# the service user, laying out data directories, rendering the app's own +# config file from `app_config` + `vault_secrets`, and installing a systemd +# unit — lives with the app, not here. See src//ansible/proxmox/. +- name: Run app-specific install steps + ansible.builtin.include_tasks: "{{ app_local_dir }}/ansible/proxmox/install.yml" + +- name: Enable and start the service + ansible.builtin.systemd_service: + name: "{{ app.service_name | default(app.name) }}" + enabled: true + state: started + daemon_reload: true diff --git a/build/config/ansible/roles/pve_backup/defaults/main.yml b/build/config/ansible/roles/pve_backup/defaults/main.yml new file mode 100644 index 0000000..ba8e34e --- /dev/null +++ b/build/config/ansible/roles/pve_backup/defaults/main.yml @@ -0,0 +1,61 @@ +--- +# Backup storage on the NAS, and the vzdump job that writes to it. Override +# per-node in inventory/host_vars/.yml. + +# Name the storage appears under in Proxmox. Also what `pve_backup_job` +# targets. +pve_backup_storage: nas-backup + +# `nfs` or `pbs`. NFS is the plain option and needs nothing on the NAS beyond +# an export; PBS gets deduplication and incremental backups but needs a Proxmox +# Backup Server to exist first. See tasks/main.yml for what each consumes. +pve_backup_storage_type: nfs + +# NFS (pve_backup_storage_type: nfs) +pve_backup_nfs_server: "" +pve_backup_nfs_export: "" +# 4.2 rather than 3: better locking and no separate portmapper. +pve_backup_nfs_options: vers=4.2 + +# PBS (pve_backup_storage_type: pbs) +pve_backup_pbs_server: "" +pve_backup_pbs_datastore: "" +pve_backup_pbs_username: "" +pve_backup_pbs_password: "" +pve_backup_pbs_fingerprint: "" + +# How many archives the storage keeps. Proxmox prunes archives, not their +# contents — anything inside a guest (such as Forgejo's nightly pg_dump output) +# is pruned by that guest, on its own schedule. +pve_backup_prune: + keep-daily: 7 + keep-weekly: 4 + keep-monthly: 6 + +# systemd calendar event. Guests that dump a database into their own filesystem +# first must finish before this — see src/forgejo/ansible/proxmox/README.md. +pve_backup_schedule: "02:00" + +# Stable marker used to find this job again on later runs. The Proxmox API +# assigns job IDs itself, so matching on the comment is what makes this +# idempotent rather than creating a duplicate job every deploy. +pve_backup_comment: managed-by-homelab-iac + +# Which guests to back up. `true` covers everything on the node, including +# guests added later without an edit here; set it false and list VMIDs in +# pve_backup_vmids to be selective. +pve_backup_all: true +pve_backup_vmids: [] + +# snapshot: no downtime, and the guest filesystem is captured as if it had +# crashed at that instant. Fine for Forgejo — git repositories tolerate it, and +# the database is a separate consistent dump rather than live files. +pve_backup_mode: snapshot +pve_backup_compress: zstd +pve_backup_enabled: true + +# Optional: address Proxmox mails backup reports to. Left empty means the node +# default applies. PVE 8 moved most of this to its notification-target system; +# `mailto` still works, but if you want anything richer, configure a +# notification target in the UI rather than extending this role. +pve_backup_mailto: "" diff --git a/build/config/ansible/roles/pve_backup/tasks/main.yml b/build/config/ansible/roles/pve_backup/tasks/main.yml new file mode 100644 index 0000000..b9890ea --- /dev/null +++ b/build/config/ansible/roles/pve_backup/tasks/main.yml @@ -0,0 +1,161 @@ +--- +# Configures backups on a Proxmox node: a storage on the NAS to write archives +# to, and a scheduled vzdump job that fills it. +# +# This is the off-box copy for every Proxmox guest. Guests deliberately keep +# all their state on their own rootfs rather than bind-mounting anything in, +# because vzdump excludes bind mounts — see src/forgejo/terraform/README.md. +# Where an app's data lives outside the container (the shared Postgres), the +# app dumps it into its own filesystem on a timer that runs before the window +# below, so one archive is one restore point. +# +# Driven through `pvesm`/`pvesh` rather than modules: as of writing, +# `community.proxmox` has no storage or backup-job module. Swap them in if that +# changes — the shape here is create-if-absent, update-if-drifted, which is +# what a module would do anyway. +# +# UNVERIFIED — this has not been run against a real node. `pvesm add` and +# `pvesh create /cluster/backup` accept slightly different options across PVE +# versions; check yours with `pvesm help add`, `pvesh help create +# /cluster/backup`, and a `--check` run before trusting it. + +- name: Check that the storage backend is configured + ansible.builtin.assert: + that: + - pve_backup_storage_type in ['nfs', 'pbs'] + - pve_backup_storage_type != 'nfs' or (pve_backup_nfs_server | length > 0 + and pve_backup_nfs_export | length > 0) + - pve_backup_storage_type != 'pbs' or (pve_backup_pbs_server | length > 0 + and pve_backup_pbs_datastore | length > 0) + - pve_backup_all or (pve_backup_vmids | length > 0) + fail_msg: >- + pve_backup is missing required settings for + pve_backup_storage_type={{ pve_backup_storage_type }}. Set them in + inventory/host_vars/{{ inventory_hostname }}.yml. + quiet: true + +- name: Build the prune-backups option string + ansible.builtin.set_fact: + pve_backup_prune_string: >- + {% for key, value in pve_backup_prune.items() %}{{ key }}={{ value }}{% if not loop.last %},{% endif %}{% endfor %} + +# --- Storage ------------------------------------------------------------- + +- name: Read the configured storages + ansible.builtin.command: + argv: [pvesh, get, /storage, --output-format, json] + register: pve_storages + changed_when: false + check_mode: false + +- name: Add the backup storage (NFS) + ansible.builtin.command: + argv: + - pvesm + - add + - nfs + - "{{ pve_backup_storage }}" + - --server + - "{{ pve_backup_nfs_server }}" + - --export + - "{{ pve_backup_nfs_export }}" + - --options + - "{{ pve_backup_nfs_options }}" + - --content + - backup + - --prune-backups + - "{{ pve_backup_prune_string }}" + when: + - pve_backup_storage_type == 'nfs' + - pve_backup_storage not in (pve_storages.stdout | from_json | map(attribute='storage') | list) + +- name: Add the backup storage (PBS) + # The password goes on a command line, which is visible in the node's process + # list for the moment the command runs. `pvesm` has no stdin form, so the + # alternative is configuring PBS by hand — acceptable for a one-time create + # that only fires when the storage is absent. + ansible.builtin.command: + argv: + - pvesm + - add + - pbs + - "{{ pve_backup_storage }}" + - --server + - "{{ pve_backup_pbs_server }}" + - --datastore + - "{{ pve_backup_pbs_datastore }}" + - --username + - "{{ pve_backup_pbs_username }}" + - --password + - "{{ pve_backup_pbs_password }}" + - --fingerprint + - "{{ pve_backup_pbs_fingerprint }}" + - --content + - backup + - --prune-backups + - "{{ pve_backup_prune_string }}" + when: + - pve_backup_storage_type == 'pbs' + - pve_backup_storage not in (pve_storages.stdout | from_json | map(attribute='storage') | list) + no_log: true + +# --- Backup job ---------------------------------------------------------- + +- name: Read the configured backup jobs + ansible.builtin.command: + argv: [pvesh, get, /cluster/backup, --output-format, json] + register: pve_backup_jobs + changed_when: false + check_mode: false + +# Proxmox assigns job IDs itself, so there is nothing stable to key on except +# a comment we set. Without this the role would add a duplicate job on every +# run. +- name: Find a job this role already created + ansible.builtin.set_fact: + pve_backup_existing: >- + {{ (pve_backup_jobs.stdout | from_json) + | selectattr('comment', 'defined') + | selectattr('comment', 'equalto', pve_backup_comment) + | list | first | default({}) }} + +- name: Build the vzdump job arguments + ansible.builtin.set_fact: + pve_backup_job_args: >- + {{ ['--schedule', pve_backup_schedule, + '--storage', pve_backup_storage, + '--mode', pve_backup_mode, + '--compress', pve_backup_compress, + '--comment', pve_backup_comment, + '--enabled', ('1' if pve_backup_enabled else '0')] + + (['--all', '1'] if pve_backup_all + else ['--all', '0', '--vmid', pve_backup_vmids | join(',')]) + + (['--mailto', pve_backup_mailto] if pve_backup_mailto | length > 0 else []) }} + +- name: Create the vzdump job + ansible.builtin.command: + argv: "{{ ['pvesh', 'create', '/cluster/backup'] + pve_backup_job_args }}" + when: pve_backup_existing | length == 0 + +# Compared field by field rather than blindly re-applying, so the role reports +# a change only when there is one. The VMID list is compared as the +# comma-joined string Proxmox stores, which makes it order-sensitive — reorder +# pve_backup_vmids and you get one no-op update. +- name: Work out whether the existing job still matches + ansible.builtin.set_fact: + pve_backup_drifted: >- + {{ pve_backup_existing.schedule | default('') != pve_backup_schedule + or pve_backup_existing.storage | default('') != pve_backup_storage + or pve_backup_existing.mode | default('') != pve_backup_mode + or (pve_backup_existing.enabled | default(1) | int == 1) != pve_backup_enabled + or (pve_backup_existing.all | default(0) | int == 1) != pve_backup_all + or (not pve_backup_all + and pve_backup_existing.vmid | default('') != pve_backup_vmids | join(',')) }} + when: pve_backup_existing | length > 0 + +- name: Update the vzdump job if its settings drifted + ansible.builtin.command: + argv: "{{ ['pvesh', 'set', '/cluster/backup/' ~ pve_backup_existing.id] + pve_backup_job_args }}" + when: + - pve_backup_existing | length > 0 + - pve_backup_drifted | bool diff --git a/build/config/ansible/roles/unattended_upgrades/defaults/main.yml b/build/config/ansible/roles/unattended_upgrades/defaults/main.yml new file mode 100644 index 0000000..0a37495 --- /dev/null +++ b/build/config/ansible/roles/unattended_upgrades/defaults/main.yml @@ -0,0 +1,19 @@ +--- +# Origins unattended-upgrades installs from — Ubuntu's own stock list +# (security, plus the ESM ones, which are inert without Ubuntu Pro attached +# rather than an error). Override per-host/group for less than this. +unattended_upgrades_origins: + - "${distro_id}:${distro_codename}" + - "${distro_id}:${distro_codename}-security" + - "${distro_id}ESMApps:${distro_codename}-apps-security" + - "${distro_id}ESM:${distro_codename}-infra-security" + +# Never reboot automatically. playbooks/k3s_maintenance.yml (role +# k3s_maintenance) does that instead — draining the node first, one at a +# time — which is the entire reason this role exists rather than just +# `apt install unattended-upgrades` with Ubuntu's own defaults (which reboot +# at 02:00 unprompted, k3s node or not). +unattended_upgrades_automatic_reboot: false + +unattended_upgrades_remove_unused_deps: true +unattended_upgrades_remove_unused_kernel_packages: true diff --git a/build/config/ansible/roles/unattended_upgrades/handlers/main.yml b/build/config/ansible/roles/unattended_upgrades/handlers/main.yml new file mode 100644 index 0000000..07ebbff --- /dev/null +++ b/build/config/ansible/roles/unattended_upgrades/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: restart unattended-upgrades + ansible.builtin.systemd_service: + name: unattended-upgrades + state: restarted diff --git a/build/config/ansible/roles/unattended_upgrades/tasks/main.yml b/build/config/ansible/roles/unattended_upgrades/tasks/main.yml new file mode 100644 index 0000000..e93a553 --- /dev/null +++ b/build/config/ansible/roles/unattended_upgrades/tasks/main.yml @@ -0,0 +1,43 @@ +--- +# Debian/Ubuntu-generic — not k3s-specific, kept independent of roles/k3s_node +# so it can be pointed at any apt-based host later (a Proxmox guest, say) +# without dragging k3s along. Currently only applied to k3s_cluster, from +# playbooks/k3s.yml. +# +# Installs updates hands-off, but leaves rebooting to +# playbooks/k3s_maintenance.yml — see unattended_upgrades_automatic_reboot in +# defaults/main.yml for why. + +- name: Install unattended-upgrades + ansible.builtin.apt: + name: + - unattended-upgrades + - update-notifier-common + state: present + update_cache: true + cache_valid_time: 3600 + +- name: Configure unattended-upgrades + ansible.builtin.template: + src: 50unattended-upgrades.j2 + dest: /etc/apt/apt.conf.d/50unattended-upgrades + owner: root + group: root + mode: "0644" + notify: restart unattended-upgrades + +- name: Enable periodic apt updates and unattended-upgrades + ansible.builtin.copy: + dest: /etc/apt/apt.conf.d/20auto-upgrades + content: | + APT::Periodic::Update-Package-Lists "1"; + APT::Periodic::Unattended-Upgrade "1"; + owner: root + group: root + mode: "0644" + +- name: Ensure unattended-upgrades is enabled and running + ansible.builtin.systemd_service: + name: unattended-upgrades + enabled: true + state: started diff --git a/build/config/ansible/roles/unattended_upgrades/templates/50unattended-upgrades.j2 b/build/config/ansible/roles/unattended_upgrades/templates/50unattended-upgrades.j2 new file mode 100644 index 0000000..01b5944 --- /dev/null +++ b/build/config/ansible/roles/unattended_upgrades/templates/50unattended-upgrades.j2 @@ -0,0 +1,17 @@ +// Managed by Ansible (roles/unattended_upgrades) — edits here are overwritten +// on the next run. + +Unattended-Upgrade::Allowed-Origins { +{% for origin in unattended_upgrades_origins %} + "{{ origin }}"; +{% endfor %} +}; + +Unattended-Upgrade::Remove-Unused-Dependencies "{{ unattended_upgrades_remove_unused_deps | lower }}"; +Unattended-Upgrade::Remove-Unused-Kernel-Packages "{{ unattended_upgrades_remove_unused_kernel_packages | lower }}"; + +// The whole point of this role: never reboot on its own. A reboot-required +// package (kernel, libc, ...) is left applied-but-inactive until +// playbooks/k3s_maintenance.yml (role k3s_maintenance) drains this node and +// reboots it deliberately, one node at a time. +Unattended-Upgrade::Automatic-Reboot "{{ unattended_upgrades_automatic_reboot | lower }}"; diff --git a/build/config/terraform/.terraform.lock.hcl b/build/config/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..4ae0b84 --- /dev/null +++ b/build/config/terraform/.terraform.lock.hcl @@ -0,0 +1,23 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/bpg/proxmox" { + version = "0.111.1" + hashes = [ + "h1:ML2D3UUZTM99yrll/EBXj7wBYMb8xmQgomqFNybEoxY=", + "zh:18fb7c31a08dde6bffa1a4d4a211e604d6d17eec7092fd59331b3db3c6f3742c", + "zh:1cd60761538289d4dd2a1086b3ae62a7b0bdd4b1a2f824e9a44e243413168dba", + "zh:2eb76f6fc8299b6820ff678c8252332cc3366e226b5ae2e61748fd2449c1ed92", + "zh:45e6f7ebd0bf48911d37060359a4f359b5743b3092e985295733990e406d0416", + "zh:4aa8ba912eae37975d2e983394d173e595ca34fc76b5bf220b37d0e99d76e98c", + "zh:58e0789923103a77d502a0a9fc3eb920625e8eb935ec2d4ac0d006aebd1d186c", + "zh:6df8aa85fb8865915537e946c19b02538ad188018a629759c213c6f03730f642", + "zh:6ed47bc00d0913a1d0880618fa1376115e9edab6b4a658c081061a7f0e4ca360", + "zh:c5b10ff4f33df7e4c29e8f1127d49845b561b37b57517e844fb0954d7923d65e", + "zh:d016510e14b738499f0db9d9b3aafe82fc6877fb4ab4e9f831fb68a8d70a1385", + "zh:d941f394069bbf24351b363da1c64383f487067aaee0a84f9b96476d4912e212", + "zh:ddf271dbc2632ae8ffa8de3972f243ee47d260cb2ac90aa784f2746d98e21a0f", + "zh:ed0caa3501c42f611b7e9622c9b1df69fd85dc25a3cd88d3076381829688cd62", + "zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597", + ] +} diff --git a/build/config/terraform/README.md b/build/config/terraform/README.md new file mode 100644 index 0000000..e05550a --- /dev/null +++ b/build/config/terraform/README.md @@ -0,0 +1,66 @@ +# build/config/terraform + +Centralized Terraform config for Proxmox — provider, backend, and the module +calls that say what infrastructure exists. The counterpart to +`build/config/ansible`, which holds inventory and the shared roles. + +## Division of labour + +Terraform's job stops at the guest. It creates the LXC (template, cores, +memory, disk, IP) and nothing more; Ansible then installs the app into it via +the `lxc_app` role. That split is why Ansible is the common tool across both +platforms and Terraform is Proxmox-only: + +| | Unraid | Proxmox | +|---|---|---| +| Provision the host | n/a — it already exists | Terraform (this directory) | +| Install the app | Ansible + `compose_stack` (Docker Compose) | Ansible + `lxc_app` (native + systemd) | +| App config | `src//common/vars.yml` + `src//ansible/unraid/vars.yml` | `src//common/vars.yml` + `src//ansible/proxmox/vars.yml` | + +Per-app LXC specs live in `src//terraform/` as modules, called from +`main.tf` here. + +## State + +`backend "pg"` — specifically against the **CloudNativePG cluster on the k3s +Pis**, reached on its MetalLB address (`k3s_postgres_loadbalancer_ip`, +192.168.50.81). *Not* the shared Postgres this configuration itself +provisions on Proxmox, which would be circular: Terraform would need the +database to exist in order to create the container the database runs in. +State goes somewhere Terraform has no hand in building, which breaks the +cycle outright. Local state would make Terraform workstation-only; CI needs +to see the same state. + +The connection string holds a password, so it is passed at init rather than +committed: + +```sh +terraform init \ + -backend-config="conn_str=postgres://terraform:$PG_PASSWORD@192.168.50.81:5432/terraform_state" +``` + +The bootstrap dependency this creates is on **k3s, not Unraid**: the cluster +and its CNPG instance must be up, with a `terraform_state` database and a +`terraform` role created on it, before `terraform init` works. That is the +one cross-platform dependency the rest of the repo avoids, and it is +deliberate — the alternative is a local-state-then-`init -migrate-state` +dance that has to be got right exactly once. Ordering is in +`docs/postgres-proxmox.md`. + +## Credentials + +Proxmox API token and the Postgres connection string both come from Vault +(`homelab/ci/proxmox`, `homelab/ci/terraform`), exported as environment +variables before running — the same pattern `group_vars/all.yml` uses for +`VAULT_ADDR` and friends. Nothing authenticating to anything is committed. + +```sh +export TF_VAR_proxmox_endpoint='https://turtle-proxmox-01.home.turtlesystems.co.uk:8006/' +export TF_VAR_proxmox_api_token='root@pam!terraform=' +``` + +## Status + +Not yet applied against anything. The provider version is intentionally +unpinned until the first `terraform init` — pin what it resolves and commit +`.terraform.lock.hcl` (which is not gitignored). diff --git a/build/config/terraform/main.tf b/build/config/terraform/main.tf new file mode 100644 index 0000000..c37c86e --- /dev/null +++ b/build/config/terraform/main.tf @@ -0,0 +1,102 @@ +# Per-app LXC modules are called from here, so there is one place that knows +# what infrastructure exists. Each module lives with its app in +# src//terraform/ and outputs the guest's address, which the Ansible +# side then picks up from the Proxmox API (inventory/proxmox.yml) rather than +# by wiring outputs into inventory by hand. + +# The shared Postgres, and the first module here that is infrastructure other +# modules depend on rather than an app in its own right. Declared first for +# readability only — Terraform orders by dependency, not by position, and +# nothing else in this file references it. The ordering that does matter is +# on the Ansible side, where a guest's `db:` provisioning needs this cluster +# already answering; see the runbook in docs/postgres-proxmox.md. +module "postgres" { + source = "../../../src/shared/postgres/terraform" + + node_name = var.proxmox_node + template_file_id = var.lxc_template_file_id + ssh_public_keys = var.ssh_public_keys + + hostname = "postgres" + vm_id = 161 + ip_address = "192.168.50.54/24" + gateway = "192.168.50.254" + + cores = 2 + memory = 2048 + + # Sized for every database this host will ever hold, not for the OS. The + # one-volume rule below means growth happens here rather than by adding a + # second disk: `terraform apply` turns an increase into a `pct resize` of + # the rootfs, which on ZFS is a refquota change — online, no data move, no + # filesystem grow step. It is one-way, though; shrinking is a replacement + # and `prevent_destroy` blocks it, so overshoot rather than creep upward. + disk_size = 256 + + # The ZFS pool, so the guest's volume can be replicated at all, and the + # second node to replicate it to. Both are the whole point of this module + # differing from the one below — see src/shared/postgres/terraform/main.tf. + datastore_id = "AppData" + replication_target_node = var.proxmox_replication_node +} + +module "forgejo" { + source = "../../../src/forgejo/terraform" + + # The second node, not `var.proxmox_node` — a deliberate split of the two + # guests across the cluster rather than an oversight. Written as a literal + # here for the same reason `hostname`/`vm_id`/`ip_address` are: it states + # where this one guest goes. `var.proxmox_replication_node` happens to hold + # the same string today, but it means "where Postgres replicates to", and + # borrowing it would tie Forgejo's placement to a decision about the + # database. + # + # Safe to change only while this container does not exist. Once it does, the + # provider treats `node_name` as a replacement — and `prevent_destroy` in + # the module turns that into a failed plan, which is the intended outcome: + # moving a live Forgejo between nodes is a Proxmox migration, not a + # `terraform apply`. + node_name = "turtle-proxmox-02" + template_file_id = var.lxc_template_file_id + ssh_public_keys = var.ssh_public_keys + + hostname = "forgejo" + vm_id = 160 + ip_address = "192.168.50.52/24" + gateway = "192.168.50.254" + + cores = 2 + memory = 2048 + + # Overrides the module default only in the sense of restating it; both are + # 256 GiB. Growing this is one-way, same as the Postgres guest above: + # `terraform apply` turns an increase into an online `pct resize` of the + # rootfs, but shrinking is a replacement and `prevent_destroy` blocks it. + # Sized generously because *everything* Forgejo owns is on this volume — + # repos, LFS, attachments, indexers, and the local database dumps. + disk_size = 256 + + # The ZFS pool, which exists on both nodes, rather than the module's + # `local-lvm` default — this cluster has no storage by that name. No + # replication job to go with it, unlike the Postgres guest above: Forgejo's + # off-box copy is the vzdump archive `pve_backup` writes to the NAS, which + # is a complete filesystem restore point precisely because nothing here is + # bind-mounted. + datastore_id = "AppData" +} + +# Consumed by the Proxmox nodes' host_vars, which can pin the vzdump job to +# explicit VMIDs. Kept as an output rather than restated there so the two +# can't drift. +output "backup_vmids" { + description = "VMIDs of the guests this configuration creates, for the vzdump job." + value = [module.postgres.vm_id, module.forgejo.vm_id] +} + +# The address every app's `db.provision_host` and `DB_HOST` has to agree with. +# An output rather than something to look up in the Proxmox UI, because it is +# copied into several files by hand and this is the one authoritative copy. +output "postgres_address" { + description = "LAN address of the shared Postgres LXC." + value = module.postgres.ip_address +} diff --git a/build/config/terraform/providers.tf b/build/config/terraform/providers.tf new file mode 100644 index 0000000..2e3c732 --- /dev/null +++ b/build/config/terraform/providers.tf @@ -0,0 +1,42 @@ +terraform { + required_version = ">= 1.6" + + required_providers { + proxmox = { + source = "bpg/proxmox" + # Deliberately unpinned until the first real `terraform init` — pin to + # whatever it resolves, and commit .terraform.lock.hcl (which is not + # gitignored) so everyone and CI get the same provider. + # version = "~> 0.0" + } + } + + # State in Postgres rather than a local file: CI and your workstation need + # to see the same state, and a local file makes Terraform workstation-only. + # + # Specifically the CloudNativePG cluster on k3s, reached at + # k3s_postgres_loadbalancer_ip — *not* the shared Postgres this + # configuration itself provisions on Proxmox. That would be circular: + # Terraform would need the database to exist in order to create the + # container the database runs in. Putting state on a cluster this + # configuration has no hand in building breaks the cycle outright, which is + # why it is worth the cross-platform dependency the rest of the repo + # otherwise avoids — `terraform apply` now needs the Pis up. + # + # Left empty on purpose — the connection string contains a password, so it + # is supplied at init time instead of being committed: + # + # terraform init \ + # -backend-config="conn_str=postgres://terraform:$PG_PASSWORD@192.168.50.81:5432/terraform_state" + # + # Bootstrap ordering: the k3s cluster and its CNPG instance must exist (and + # a `terraform_state` database be created on it) before `terraform init` + # succeeds. See docs/postgres-proxmox.md. + backend "pg" {} +} + +provider "proxmox" { + endpoint = var.proxmox_endpoint + api_token = var.proxmox_api_token + insecure = var.proxmox_insecure +} diff --git a/build/config/terraform/variables.tf b/build/config/terraform/variables.tf new file mode 100644 index 0000000..9b68051 --- /dev/null +++ b/build/config/terraform/variables.tf @@ -0,0 +1,78 @@ +variable "proxmox_endpoint" { + description = "Proxmox VE API endpoint, e.g. https://turtle-proxmox-01.home.turtlesystems.co.uk:8006/" + type = string +} + +variable "proxmox_api_token" { + description = <<-EOT + Proxmox API token in `USER@REALM!TOKENID=SECRET` form. Sourced from Vault + (homelab/ci/proxmox), never committed — pass via the TF_VAR_proxmox_api_token + environment variable. + EOT + type = string + sensitive = true +} + +variable "proxmox_insecure" { + description = "Skip TLS verification — true while Proxmox has a self-signed cert." + type = bool + default = true +} + +variable "proxmox_node" { + description = <<-EOT + Proxmox node guests are created on. Must be the node's **short** name as + the cluster knows it (`pvecm nodes`), never an FQDN, even though the API + endpoint is one. + + pveproxy compares this against its own short hostname to decide whether a + request is local. An FQDN never matches, so it proxies the request onward + — to itself — over a connection it verifies against the cluster CA, which + a self-signed cert fails. The result is an HTTP 596 carrying an OpenSSL + `certificate verify failed`, which reads like a TLS misconfiguration on + this side and is not: `proxmox_insecure` governs the provider's own + connection and has no bearing on Proxmox's internal one. + EOT + type = string + default = "turtle-proxmox-02" +} + +variable "proxmox_replication_node" { + description = <<-EOT + The other node in the cluster, which guests on a ZFS pool replicate to. + Only the shared Postgres module uses this today; it lives here rather than + in that module's own defaults because it names a node of this cluster, the + same as `proxmox_node` does. + + Null disables replication. Set it explicitly even on a single-node + cluster — a null default is indistinguishable from an oversight, and the + failure mode is a database with no second copy that looks entirely + healthy. + EOT + type = string + default = "turtle-proxmox-01" +} + +variable "lxc_template_file_id" { + description = <<-EOT + Container template every LXC module is built from, e.g. + `local:vztmpl/debian-12-standard_12.7-1_amd64.tar.zst`. Must already be + downloaded on the node — `pveam list local` shows what is there, `pveam + available`/`pveam download local ` fetches one. Not defaulted: the + exact filename moves with each point release, so a stale default would fail + at apply time rather than here. + EOT + type = string +} + +variable "ssh_public_keys" { + description = <<-EOT + Public keys installed for root in every container, so Ansible can reach + them. The matching private key is in Vault at `homelab/ci/ssh` — see "SSH + access" in the top-level README.md. Public keys aren't secret, but they are + site-specific, so pass rather than commit: + + export TF_VAR_ssh_public_keys='["'"$(cat ~/.ssh/unraid_ansible.pub)"'"]' + EOT + type = list(string) +} diff --git a/docs/authentik-migration.md b/docs/authentik-migration.md new file mode 100644 index 0000000..69a4311 --- /dev/null +++ b/docs/authentik-migration.md @@ -0,0 +1,420 @@ +# Migrating Authentik from Unraid to k3s + +A one-time, hand-run cutover: move the Authentik database off Unraid onto the +cluster's CloudNativePG instance, bring Authentik up on k3s from +`src/authentik/`, verify it, then change DNS. + +This is deliberately not automated. It runs once, most of its steps are +irreversible in the wrong order, and the verification between them is a +judgement call rather than a task result — the sort of thing a playbook makes +harder rather than easier. The repo half *is* automated: everything from +"deploy Authentik" onwards is `playbooks/k3s.yml`, and re-running it later is +an ordinary converge. + +Read the whole thing before starting. The dump is taken with Authentik +stopped, so there's a service outage from step 5 to step 11 — budget an hour +and do it when nobody needs to log in. + +--- + +## What moves, and what doesn't + +| | Where it lives now | How it gets across | +|---|---|---| +| Users, groups, applications, providers, flows, tokens, certificates | Postgres | `pg_dump` \| `psql` (step 7) | +| Uploaded icons and flow backgrounds | `/media` on disk | `scp` + `kubectl cp` (step 9) | +| Session cookies | signed by `AUTHENTIK_SECRET_KEY` | carry the key over (step 2) | +| Cache / task queue | in-process | nothing — recent Authentik has no Redis, and the chart has no Redis dependency | + +Authentik stores its certificates (including the signing keys OIDC/SAML +providers use) **in the database**, not in a `certs/` directory, so a +successful database restore carries them. That is what lets integrations like +Shelfarr's OIDC (`src/arr/`) keep working without being re-registered. + +The one piece with no automatic path is anything you mounted into the Unraid +container by hand — custom templates, a `certs/` directory you populated +yourself. Step 1 is where you find out whether you have any. + +--- + +## 1. Gather the facts + +Everything below needs these. Run on the Unraid box (web terminal, or SSH). + +```sh +# Which containers make up the current deployment, and what image/tag. +docker ps -a --format '{{.Names}}\t{{.Image}}\t{{.Status}}' | grep -i authentik +``` + +Write down the **image tag** — that is the Authentik version, and step 3 +depends on it. + +```sh +# Every mount, so you can see where /media lives and spot anything unexpected. +docker inspect authentik-server \ + --format '{{range .Mounts}}{{.Source}} -> {{.Destination}}{{println}}{{end}}' + +# The database connection it currently uses, and the secret key. +docker inspect authentik-server \ + --format '{{range .Config.Env}}{{println .}}{{end}}' | grep -E 'POSTGRES|SECRET_KEY' +``` + +That last command tells you the thing this document can't assume: **which +Postgres the Unraid Authentik talks to**. Two possibilities, and they change +step 7: + +- `AUTHENTIK_POSTGRESQL__HOST=shared-postgres` (or `192.168.50.1`) — it uses + the shared instance this repo already manages, `src/shared/postgres/`. The + superuser password is in Vault at `homelab/shared/postgres`. +- anything else (`authentik-postgresql`, `postgres`, a container name from + Authentik's own compose bundle) — the Unraid UI deployment brought its own + Postgres container. You'll need *its* credentials, which are in that + container's own environment: + + ```sh + docker inspect \ + --format '{{range .Config.Env}}{{println .}}{{end}}' | grep POSTGRES + ``` + + and it is probably not published on a host port, which step 7 handles. + +Also note `AUTHENTIK_SECRET_KEY` from the output above — step 2 needs it +verbatim. + +--- + +## 2. Populate Vault + +```sh +vault kv put kv/homelab/authentik \ + AUTHENTIK_SECRET_KEY='' \ + AUTHENTIK_POSTGRESQL__PASSWORD='' +``` + +Two different rules for those two values, and it's worth being clear about +why: + +- **`AUTHENTIK_SECRET_KEY` must be the existing one.** It signs session + cookies. A fresh value doesn't break anything permanently, but it + invalidates every active session at the moment DNS flips — so instead of a + silent cutover, everyone gets logged out and the migration announces + itself. Copy it across. +- **`AUTHENTIK_POSTGRESQL__PASSWORD` can be new.** You are creating a new + role on a different Postgres; nothing in the dump references it. Generate a + fresh one. + +If you're also using cert-manager's DNS-01 solver (step 4), populate its +credentials now too: + +```sh +vault kv put kv/homelab/k3s-cert-manager \ + CLOUDFLARE_API_TOKEN='' +``` + +--- + +## 3. Confirm the repo's values + +Three values are guesses until you check them. All are flagged in-file. + +**`src/authentik/common/vars.yml`** + +- `AUTHENTIK_VERSION` — **set this to the image tag from step 1**, not to + whatever is newest. Authentik runs Django migrations on startup against + whatever schema it finds, and they are one-way: a newer Authentik pointed + at an older dump upgrades the schema silently, and if the cutover then has + to be rolled back, the Unraid instance can no longer read its own database. + Migrate first, upgrade second, in a separate commit. +- `AUTHENTIK_HOST` / `AUTHENTIK_URL` — the hostname you'll be moving. Must + match what's there today, or every OIDC/SAML redirect URI registered in + Authentik breaks. + +**`build/config/ansible/inventory/group_vars/k3s_cluster.yml`** + +- `k3s_cert_manager_solver` — empty by default, and + `roles/k3s_cert_manager` refuses to run until it's set. The commented + Cloudflare example in that file is the common case; for anything else take + the stanza from + [cert-manager's DNS-01 docs](https://cert-manager.io/docs/configuration/acme/dns01/). + +While you're testing, point `k3s_cert_manager_acme_server` at Let's Encrypt +staging. Production allows five failed validations per hostname per hour, and +exhausting it means waiting rather than retrying. + +--- + +## 4. Deploy cert-manager and confirm the issuer works + +```sh +cd build/config/ansible +ansible-playbook playbooks/k3s.yml --tags cert-manager +``` + +Then check it actually registered — a `ClusterIssuer` is created whether or +not its solver is valid, so "the resource exists" proves nothing: + +```powershell +$env:KUBECONFIG = "$PWD\local\k3s\homelab-utils.kubeconfig" +kubectl get clusterissuer letsencrypt -o jsonpath='{.status.conditions[*].message}' +``` + +You want `The ACME account was registered with the ACME server`. Anything +else, fix it here — not after Authentik is deployed and waiting on a +certificate. + +Worth knowing: with DNS-01, the certificate can be issued **before** DNS +points at the cluster. The challenge is a TXT record, not a request to your +web server, so cert-manager never needs to be reachable. That's why TLS is +sorted out in this step and not after the flip. + +--- + +## 5. Freeze the Unraid instance + +Stop writes before dumping. Skipping this gives you a dump that's missing +whatever happened during it — usually invisible, occasionally a user account +that no longer exists. + +```sh +docker stop authentik-server authentik-worker +``` + +Leave its Postgres running — step 7 reads from it. **The outage starts +here.** + +--- + +## 6. Create the role and database on CNPG + +The restore needs the `authentik` role to exist first so the dump's objects +land with the right ownership. + +```powershell +$env:KUBECONFIG = "$PWD\local\k3s\homelab-utils.kubeconfig" +kubectl -n shared-postgres exec -i shared-postgres-1 -- psql -v ON_ERROR_STOP=1 <<'SQL' +CREATE ROLE authentik LOGIN PASSWORD 'the-password-from-step-2'; +CREATE DATABASE authentik OWNER authentik; +SQL +``` + +`kubectl exec` into a CNPG pod lands you as the `postgres` superuser over the +local socket, so bare `psql` needs no credentials. + +You could instead let `roles/k3s_app` create these — its tasks are idempotent +and will run in step 8 regardless — but doing it by hand here keeps the +database creation and the restore adjacent, and avoids deploying Authentik +against an empty database first (which would have it run initial migrations +and build a schema the restore then has to fight). + +--- + +## 7. Dump and restore + +Run the whole thing **inside the CNPG pod**, piping `pg_dump` straight into +`psql`. Three reasons this is better than dumping to a file on Windows: + +- `pg_dump` must be at least the version of the server being dumped, and the + CNPG pod ships Postgres 18 client tools — dumping the Unraid Postgres 13 + from there is the supported direction. Your workstation probably has no + `pg_dump` at all. +- PowerShell re-encodes bytes passing through `>` and `|`, which silently + corrupts a dump file. Nothing on the Windows filesystem means nothing to + corrupt. +- No intermediate file holding the entire user database in cleartext. + +### If Authentik used the shared Postgres (published on `192.168.50.1:5432`) + +```powershell +$env:KUBECONFIG = "$PWD\local\k3s\homelab-utils.kubeconfig" +$srcPw = Read-Host 'Unraid postgres superuser password' + +kubectl -n shared-postgres exec -i shared-postgres-1 -- ` + env "PGPASSWORD=$srcPw" bash -c @' +set -euo pipefail +{ echo "SET ROLE authentik;" + pg_dump -h 192.168.50.1 -p 5432 -U postgres -d authentik \ + --no-owner --no-privileges +} | psql -v ON_ERROR_STOP=1 -d authentik +'@ +``` + +`SET ROLE authentik` before the dump stream, combined with `--no-owner`, is +what gets ownership right: the session creates every object as `authentik`, +so the application role owns its own schema. Without it, everything lands +owned by `postgres` and Authentik can read but not migrate. + +`ON_ERROR_STOP=1` matters — without it `psql` reports success having skipped +every statement that failed. + +### If it used its own Postgres container (not published) + +Publish it temporarily, then use the command above with the container's own +credentials and port: + +```sh +# on Unraid +docker run -d --rm --name pgbridge -p 55432:5432 \ + --network container: alpine/socat \ + TCP-LISTEN:5432,fork TCP:127.0.0.1:5432 +``` + +then `-h 192.168.50.1 -p 55432 -U ` in the `pg_dump` above, and +`docker stop pgbridge` afterwards. Simpler alternative if you'd rather not: +`docker exec pg_dump ... > /mnt/user/appdata/authentik.sql` on +Unraid, `scp` it to your workstation, and `kubectl exec -i ... -- psql -d +authentik < authentik.sql` — but a plain-SQL file through PowerShell +redirection is exactly the encoding hazard noted above, so run that last +command from WSL. + +### Check the restore + +```powershell +kubectl -n shared-postgres exec -i shared-postgres-1 -- ` + psql -d authentik -c '\dt' -c 'SELECT count(*) FROM authentik_core_user;' +``` + +You should see a long table list and a user count matching what the old +instance had. An empty table list means the restore did nothing. + +--- + +## 8. Deploy Authentik on k3s + +```sh +cd build/config/ansible +ansible-playbook playbooks/k3s.yml --tags apps +``` + +A green run means the manifests landed and the database was provisioned — not +that the workload came up. Watch it actually start: + +```powershell +kubectl -n authentik get pods -w +``` + +The first start takes several minutes on a Pi: pulling the image, then the +worker checking migrations. Because the deployed version matches the dumped +version (step 3), it should find nothing to migrate. + +If pods stay `Pending`, check the PVC — `local-path` uses +`WaitForFirstConsumer`, so a `Pending` PVC alongside a `Pending` pod is +normal only until the scheduler picks a node: + +```powershell +kubectl -n authentik describe pvc authentik-media +kubectl -n authentik logs -l app.kubernetes.io/component=worker --tail=50 +``` + +--- + +## 9. Copy `/media` across + +Two hops rather than a pipe, again to keep binary data out of a PowerShell +pipeline: + +```powershell +# from the media path you noted in step 1 +scp -r root@192.168.50.1:/mnt/user/appdata/authentik/media ./local/authentik-media + +$pod = kubectl -n authentik get pod -l app.kubernetes.io/component=server ` + -o jsonpath='{.items[0].metadata.name}' +kubectl -n authentik cp ./local/authentik-media/. "${pod}:/media" +``` + +`local/` is already gitignored, so the staging copy won't be committed. + +Skip this entirely if you never uploaded custom icons or backgrounds — +Authentik falls back to its built-in assets, and the directory is empty on a +stock install. + +--- + +## 10. Verify before touching DNS + +The hostname still resolves to Unraid, so test by overriding resolution +rather than changing it. `--resolve` sends the request to Traefik with the +right SNI and `Host` header: + +```powershell +curl.exe -sv --resolve auth.turtlesystems.uk:443:192.168.50.80 ` + https://auth.turtlesystems.uk/-/health/ready/ +``` + +Check, in order: + +1. The TLS certificate is the Let's Encrypt one, not Traefik's self-signed + default (`kubectl -n authentik get certificate` → `READY True`). If you + used the staging server in step 3, it will be untrusted — expected; switch + to production and re-run `--tags cert-manager` once satisfied. +2. `/-/health/ready/` returns 204. +3. Add a hosts-file entry pointing the name at `192.168.50.80` and log in + through a browser with a **real user account**. This is the check that + matters — it exercises the restored password hashes, the flows, and the + session cookie signed with the carried-over secret key. +4. Open an existing application from the Authentik dashboard and confirm the + OIDC/SAML round-trip still works (Shelfarr is the one this repo knows + about — see `src/arr/`). + +Remove the hosts-file entry afterwards. + +--- + +## 11. Flip DNS + +Point `auth.turtlesystems.uk` at `192.168.50.80` (Traefik's MetalLB address). + +Lower the record's TTL a few hours beforehand if you can — it shortens the +window in which a rollback is still invisible to clients. + +--- + +## Rollback + +Viable until users start making changes in the new instance; after that, +rolling back loses whatever they did. + +1. Point DNS back at Unraid. +2. `docker start authentik-server authentik-worker`. + +The old database was only ever read, never written, so it's exactly as it was +at step 5. Nothing to restore. + +Do **not** run both instances at once against their separate databases — they +diverge immediately, and there is no way to merge them afterwards. That is +the same "never declare an app on two platforms" rule from CLAUDE.md, and it +is why `src/authentik/ansible/unraid/` is a README rather than a compose file. + +--- + +## After it has settled + +Once you're confident, in this order: + +1. **Delete the Unraid containers.** A UI action, not `state: absent` — this + repo never deployed them, so it has nothing to tear down. Keep the appdata + directory a while longer. +2. **Drop the old database**, if it was on the shared Postgres: + ```sh + docker exec -it shared-postgres psql -U postgres -c 'DROP DATABASE authentik;' + ``` + Take a copy first if you want one; nothing else does. +3. **Upgrade Authentik**, now as an ordinary change: bump + `AUTHENTIK_VERSION` in `src/authentik/common/vars.yml`, read the release + notes for anything between your version and the target, and + `ansible-playbook playbooks/k3s.yml --tags apps -e only_apps=authentik`. + Do this as its own commit, so a bad upgrade is one `git revert` away from + the version you know worked. + +## Backups, afterwards + +Worth stating plainly: the cluster's CNPG has **no backups configured** +(CLAUDE.md → "Key decisions" — HA replication only). Before this migration, +Authentik's database sat on Unraid, inside whatever covers `/mnt/user/appdata`. +After it, the only redundancy is a streaming replica on another Pi, which +protects against a dead SD card and not against a bad migration, a dropped +table, or a mistake in this document. + +The gap this leaves is real and this cutover widens it. CNPG's answer is a +`ScheduledBackup` to object storage or an NFS volume on the NAS; that's not +built here yet, and it's the obvious next piece of work after Authentik is +settled. diff --git a/docs/forgejo-proxmox.md b/docs/forgejo-proxmox.md new file mode 100644 index 0000000..2a65bab --- /dev/null +++ b/docs/forgejo-proxmox.md @@ -0,0 +1,287 @@ +# Bringing up Forgejo on Proxmox + +A run-once bootstrap, same shape as `postgres-proxmox.md` and deliberately +downstream of it — Forgejo's database lives on the shared Postgres that +runbook creates, so none of this works until that one is finished. + +The end state: an LXC on the `AppData` pool of **turtle-proxmox-02** (VMID +160, 192.168.50.52), running Forgejo natively under systemd, with its database +on the shared Postgres LXC at 192.168.50.54 and a nightly `pg_dump` landing on +its own disk in time for the node's 02:00 vzdump. + +## Why turtle-proxmox-02 + +Nothing forces it. Postgres is on `-01`, so putting Forgejo on `-02` splits +the two guests across the cluster: a node going down takes one of them with +it rather than both. Note what that does *not* buy — Forgejo with its database +unreachable is not a working forge, so this is about not losing both +filesystems at once, not about staying up. + +It is set as a literal `node_name` on the `forgejo` module in +`build/config/terraform/main.tf`, not from `var.proxmox_node`. **Change it +only before the container exists**: the provider treats `node_name` as a +replacement, and `prevent_destroy` in `src/forgejo/terraform/main.tf` turns +that into a failed plan. Moving a live Forgejo is a Proxmox migration +(`pct migrate`) followed by editing the literal to match, not a +`terraform apply`. + +## 1. Secrets + +`homelab/forgejo` needs five keys. Four of them may already exist from the +Unraid stack; **`LFS_JWT_SECRET` almost certainly does not** — the Compose +deployment let Forgejo generate it, and the native install renders `app.ini` +in full with `INSTALL_LOCK = true`, so there is no first boot for Forgejo to +invent one on. + +```sh +vault kv get kv/homelab/forgejo +``` + +If `LFS_JWT_SECRET` is missing, generate one and patch it in without +disturbing the others: + +```sh +vault kv patch kv/homelab/forgejo \ + LFS_JWT_SECRET="$(docker run --rm codeberg.org/forgejo/forgejo:10 forgejo generate secret)" +``` + +Full key table in `vault-secrets.md`. `install.yml` asserts on all five before +it touches the container, so a missing one fails on the first task with a +message naming it rather than three tasks later on an undefined variable. + +## 2. The container template + +The template must be present on **turtle-proxmox-02**, not just on `-01` where +Postgres was built. If `StorageOne` is not shared across the cluster this is a +separate download: + +```sh +ssh root@turtle-proxmox-02 'pveam list StorageOne' +# if the ubuntu-26.04 template is absent: +ssh root@turtle-proxmox-02 'pveam update && pveam download StorageOne ubuntu-26.04-standard_26.04-1_amd64.tar.zst' +``` + +Whatever is there has to match `lxc_template_file_id` exactly — the filename +moves with each point release, which is why that variable has no default. + +## 3. Terraform + +State is on the CNPG cluster, so the Pis have to be up (see +`postgres-proxmox.md` § "Why this order"). + +```sh +cd build/config/terraform +terraform init -backend-config=... # as per postgres-proxmox.md +terraform plan -out=deploy.plan -target=module.forgejo +terraform apply deploy.plan +``` + +Read the plan before applying. It should be **one resource to add** and +nothing to change or destroy; anything touching `module.postgres` — in +particular a *replacement* — means stop and work out why, because that guest +is live. + +Note `deploy.plan` embeds the state and every input variable, `sensitive` ones +included, so it holds the Proxmox API token in the clear. `.gitignore` covers +`*.plan`; delete it once applied rather than leaving it in the tree. + +Confirm the guest exists and Ansible can see it: + +```sh +ssh root@turtle-proxmox-02 'pct list | grep 160' +cd ../ansible && ansible-inventory --list --yaml proxmox_guests | grep -A2 forgejo +``` + +If it is absent from the inventory but present in `pct list`, the `terraform` +tag is missing — that filter is what tells this repo's guests from hand-made +ones, and a guest without it is silently never deployed to. + +## 4. Install Forgejo into it + +```sh +cd build/config/ansible +ansible-playbook playbooks/proxmox.yml -e only_stacks=forgejo +``` + +This provisions the `forgejo` database and role on 192.168.50.54, then +installs the binary, `app.ini`, the systemd unit and the dump timer. The +database step runs from **your controller**, not from inside the container +(`delegate_to: localhost`), so 192.168.50.54:5432 has to be reachable from +wherever you run this. + +Check: + +```sh +ssh root@192.168.50.52 'systemctl is-active forgejo && systemctl list-timers forgejo-dbdump.timer' +curl -sI http://192.168.50.52:3000/ | head -1 +``` + +## 4b. The first admin user + +A fresh install has **no way in**: `INSTALL_LOCK = true` skips the setup +wizard, which is where the first admin would normally be created, and +`DISABLE_REGISTRATION = true` means you cannot self-register either (both in +`src/forgejo/ansible/proxmox/templates/app.ini.j2`). Nothing in `install.yml` +creates a user, so the account has to be made from the CLI inside the +container: + +```sh +ssh root@192.168.50.52 + +sudo -u git forgejo admin user create --admin --username --email @turtlesystems.co.uk --password '' --config /etc/forgejo/app.ini --work-path /var/lib/forgejo +``` + +Three things that are easy to get wrong: + +- **`sudo -u git`, not root.** `RUN_USER` is `git`; running the CLI as root + leaves root-owned files under `/var/lib/forgejo` that Forgejo then cannot + write. `app.ini` is `0640 root:git`, so the `git` user can read it. +- **`--config` is not optional.** Forgejo looks for `custom/conf/app.ini` + under the work path by default; this install puts it at + `/etc/forgejo/app.ini` (`FORGEJO_CONFIG_DIR` in `vars.yml`). Without the + flag the CLI reads a config that isn't there and never reaches the + database. +- Recent versions default `--must-change-password` to true, so expect a + forced change on first web login. Pass `--must-change-password=false` to + skip it. Confirm the flags for the pinned `FORGEJO_RELEASE` with + `forgejo admin user create --help` rather than assuming. + +Then log in at `http://192.168.50.52:3000/user/login` — by IP, because +`ROOT_URL` is `https://git.turtlesystems.uk` and nothing resolves there until +the cutover in step 7. The login form works over the IP; some links and +redirects Forgejo renders will point at the not-yet-live hostname. + +**Skip this step if you are restoring a database in step 6.** Users live in +the database, so a dump brings its own admin back and an account created here +is overwritten by the restore. Create one only if this is genuinely an empty +forge — or after the restore, if the dump turns out to have no usable admin. + +## 5. Node backups + +VMID 160 is on `-02`, so it is `-02`'s vzdump job that covers it. +`pve_backup_all: true` in `group_vars/proxmox_nodes.yml` means no edit is +needed — but the job has to actually exist on that node: + +```sh +ansible-playbook playbooks/pve_host.yml --limit turtle-proxmox-02 +ssh root@turtle-proxmox-02 'cat /etc/pve/jobs.cfg' +``` + +The 01:30 dump and the 02:00 vzdump are a pair. Move one, move the other — +see `src/forgejo/ansible/proxmox/README.md`. + +## 6. Repository data + +**Ansible does not move any of this.** A fresh install is a working, empty +forge; the repositories are a separate restore into +`/var/lib/forgejo/data/forgejo-repositories`. + +The original instructions here were an `rsync` off nas2, which no longer +exists as a machine to read from — whatever repository data survives has to +come from wherever it went when that host was retired. After restoring: + +```sh +ssh root@192.168.50.52 'chown -R git:git /var/lib/forgejo/data && systemctl restart forgejo' +``` + +Restore the database the same way, from whatever dump you have, before +starting Forgejo against it — an empty database with populated repositories +gives back every file and no issues, pull requests, users or permissions. + +## 7. Cutover + +Last, once the above is verified: + +- Point `git.turtlesystems.uk` at this container. +- **SSH clone URLs move from port 2222 to 22.** The Unraid stack published + 2222 to dodge the host's own sshd; this container has its own IP and uses + its own sshd, with Forgejo managing the `git` user's `authorized_keys`. + Every existing remote needs editing — there is no redirect for this. +- The Forgejo Actions runner registration is tied to the instance. Re-register + it against the new host, or CI stops running (`.forgejo/workflows/`). + +## 8. Sign-in through Authentik (optional) + +Forgejo supports OIDC, and `app.ini` is already set up for it — but only the +*policy* half. The provider itself cannot be configured from `app.ini`: +Forgejo keeps authentication sources in its **database**. That makes this the +one part of Forgejo's configuration Ansible does not own, and the reason it is +a runbook step rather than a task in `install.yml`. It is run once and then +carried forward by the nightly `pg_dump`, the same as every other row in that +database. + +**Do this after step 7, not before.** The redirect URI has to match `ROOT_URL` +(`https://git.turtlesystems.uk`), so DNS and TLS must already be live. +Configuring it against `http://192.168.50.52:3000` means doing it twice. + +### On the Authentik side + +Create an OAuth2/OIDC **Provider** plus an **Application** for it, as for any +other app on `auth.turtlesystems.uk`. The redirect URI is: + +``` +https://git.turtlesystems.uk/user/oauth2/authentik/callback +``` + +The last path segment is the *name of the auth source in Forgejo*, not a fixed +string — it has to match the `--name` below. Store the generated client secret +in Vault (`vault-secrets.md` → `OIDC_CLIENT_SECRET`). + +### On the Forgejo side + +```sh +ssh root@192.168.50.52 + +sudo -u git forgejo admin auth add-oauth --name authentik --provider openidConnect --key --secret --auto-discover-url https://auth.turtlesystems.uk/application/o/forgejo/.well-known/openid-configuration --config /etc/forgejo/app.ini --work-path /var/lib/forgejo +``` + +Same `sudo -u git` and `--config` requirements as step 4b, for the same +reasons. Verify the flag names against the pinned `FORGEJO_RELEASE` with +`forgejo admin auth add-oauth --help` before running — and list what exists +afterwards with `forgejo admin auth list`, which is also how you find the `id` +for `update-oauth` if the secret is ever rotated. + +No restart is needed: the source is a database row, not a file Forgejo reads +at boot. + +### What the app.ini side already does + +Set in `src/forgejo/ansible/proxmox/templates/app.ini.j2`, so a redeploy keeps +them — the comments there carry the detail: + +| Setting | Effect | +|---|---| +| `DISABLE_REGISTRATION = false` | It blocks OIDC auto-registration too, not just the local signup form. Left `true`, Authentik logins authenticate and are then refused an account. | +| `ALLOW_ONLY_EXTERNAL_REGISTRATION = true` | Restores "no self-service signup" without blocking Authentik. | +| `ENABLE_AUTO_REGISTRATION = true` | Creates the Forgejo account on first successful SSO login. | +| `ACCOUNT_LINKING = auto` | Attaches the OIDC identity to an existing local account with the same email. | + +`ACCOUNT_LINKING` is the one that matters on the first login: if the +break-glass admin from step 4b uses the same email address as your Authentik +account, signing in through Authentik lands you *in that admin account*. With +linking off you would arrive as a second, unprivileged user and have to +promote it from the CLI. + +### Keep the local admin + +Do not delete the step 4b account or convert it to SSO-only. The login path is +now Forgejo → Authentik → CloudNativePG → a healthy k3s cluster, and any of +those failing takes SSO down with it — including the cases you would most want +to log into the forge to investigate. This is the same reasoning +`roles/k3s_traefik` uses for keeping the port-forward-only dashboard route +alive alongside the Authentik-published one. + +### Git over HTTPS + +Users who arrive through Authentik have no Forgejo password, so HTTPS clones +need a personal access token. SSH keys are unaffected — and since step 7 put +git-over-SSH on port 22 with Forgejo managing the `git` user's +`authorized_keys`, SSH is the smoother default to point people at. + +## Rolling back + +There is no `state: absent` for `lxc_app` — removal would need a per-app +`uninstall.yml` and Forgejo has none. To back out: stop the service +(`systemctl stop forgejo`), point DNS back at whatever was serving before, and +leave the container in place. Destroying it needs the `prevent_destroy` block +removed by hand first, which is deliberate. diff --git a/docs/postgres-proxmox.md b/docs/postgres-proxmox.md new file mode 100644 index 0000000..6394bce --- /dev/null +++ b/docs/postgres-proxmox.md @@ -0,0 +1,237 @@ +# Bringing up the shared Postgres on Proxmox + +A run-once bootstrap, in the order the dependencies actually demand. Each step +exists because the one after it cannot start without it; if you already have a +piece, skip it and check the assertion at the end of the section. + +The end state: a Debian LXC on the `AppData` ZFS pool of `turtle-proxmox-01`, +running Postgres 17, replicated every five minutes to `turtle-proxmox-02`, and +answering on 192.168.50.54:5432 for every app that declares a `db:`. + +## Why this order + +Two chains have to be satisfied before `terraform apply` will run at all: + +- **State.** Terraform stores state in Postgres. Storing it in the database + this configuration provisions would be circular, so it goes on the + CloudNativePG cluster on the k3s Pis instead — a cluster Terraform has no + hand in building. That cluster therefore has to be up first. +- **Inventory.** `playbooks/proxmox.yml` targets `proxmox_guests`, a group + that only exists because of the API-backed dynamic inventory. Ansible cannot + install into a container Terraform has just made until it can see it. + +## 1. The k3s cluster and its CNPG instance + +Needed only for Terraform state. If the cluster is already up, confirm the +LoadBalancer answers and move on. + +```sh +cd build/config/ansible +ansible-playbook playbooks/k3s.yml --tags services +``` + +Then, against `k3s_postgres_loadbalancer_ip` (192.168.50.81), create the state +database and its role. This is the only hand-run SQL in the whole flow — +everything else provisions itself: + +```sql +CREATE ROLE terraform LOGIN PASSWORD ''; +CREATE DATABASE terraform_state OWNER terraform; +``` + +Store the resulting connection string at `homelab/ci/terraform` in Vault (see +`vault-secrets.md`), then check it: + +```sh +psql "postgres://terraform:$PG_PASSWORD@192.168.50.81:5432/terraform_state" -c '\conninfo' +``` + +## 2. Secrets + +`homelab/shared/postgres` is the same path the Unraid and k3s instances +already use — one superuser identity for the "shared postgres" concept +wherever it runs. If it is populated, nothing to do: + +```sh +vault kv get kv/homelab/shared/postgres +``` + +`homelab/ci/proxmox` needs a Proxmox API token, and `homelab/ci/ssh` the key +pair whose public half Terraform installs into the container. + +## 3. The container template + +On the node, once: + +```sh +pveam available | grep debian-12 +pveam download local debian-12-standard_12.7-1_amd64.tar.zst +``` + +## 4. Terraform + +```sh +cd build/config/terraform +terraform init -backend-config="conn_str=$PG_CONN_STR" +``` + +Pin the provider to whatever this resolves and commit the resulting +`.terraform.lock.hcl`. Then: + +```sh +export TF_VAR_proxmox_endpoint='https://turtle-proxmox-01.home.turtlesystems.co.uk:8006/' +export TF_VAR_proxmox_api_token='...' # homelab/ci/proxmox +export TF_VAR_lxc_template_file_id='local:vztmpl/debian-12-standard_12.7-1_amd64.tar.zst' +export TF_VAR_ssh_public_keys='["'"$(cat ~/.ssh/unraid_ansible.pub)"'"]' + +terraform apply -target=module.postgres +``` + +`-target` on purpose: the root module also defines the Forgejo LXC, and +bringing that up is a separate decision with its own migration story +(`inventory/host_vars/forgejo.yml`). Drop the flag once you want both. + +If the apply 403s on `changing feature flags (except nesting) is only allowed +for root@pam`, the container already exists without nesting enabled and the +`ansible@pam` token cannot add it. Set it as root on the node, then re-plan — +it will read clean, and Postgres's systemd unit needs it: + +```sh +ssh root@turtle-proxmox-01.home.turtlesystems.co.uk 'pct set 161 --features nesting=1' +ssh root@turtle-proxmox-01.home.turtlesystems.co.uk 'pct reboot 161' +``` + +Check the replication job exists — this is the step most easily missed, +because a container with no job looks identical in the storage view: + +```sh +ssh root@turtle-proxmox-01 'pvesr status' +``` + +## 5. Install Postgres into it + +```sh +cd build/config/ansible +ansible-inventory -i inventory/proxmox.yml --graph # the guest should appear +ansible-playbook playbooks/proxmox.yml -e only_stacks=postgres +``` + +Then from the controller, confirming both that it listens on the LAN and that +the Vault password took: + +```sh +psql "postgres://postgres:$PGPASSWORD@192.168.50.54:5432/postgres" -c 'SELECT version();' +``` + +## 6. Node backups + +Replication covers a dead node, not a dropped table. `roles/pve_backup` writes +vzdump archives to the NAS: + +```sh +ansible-playbook playbooks/pve_host.yml +``` + +## Growing the disk + +Everything lives on the rootfs — that is what makes a replication snapshot +atomic (`src/shared/postgres/terraform/README.md`), so there is no second +volume to add when space runs short. Raise `disk_size` on the `postgres` +module in `build/config/terraform/main.tf` and apply. + +Check the pool has the room first. Terraform will not: ZFS lets an apply +overcommit the pool happily, and the failure surfaces later as a write inside +the guest hitting ENOSPC. + +```sh +ssh root@turtle-proxmox-01 'zpool list AppData; zfs list -o name,used,avail,refquota -r AppData' +ssh root@turtle-proxmox-02 'zpool list AppData' # the target needs the room too +``` + +Then: + +```sh +cd build/config/terraform +terraform plan # expect an in-place update to disk.size, NOT a replacement +terraform apply +``` + +A **replacement** in that plan means something other than the size changed — +`prevent_destroy` will refuse it, which is the point. Never work around it +here; replacing this container destroys every database on it. + +The resize itself is a `pct resize` of the rootfs, which on ZFS-backed storage +is a refquota change rather than a partition operation. It applies to a +running guest, ships no data, and needs nothing done inside the container +afterwards — no `resize2fs`, no Postgres restart. Confirm from inside: + +```sh +ssh root@192.168.50.54 'df -h /' +``` + +Two things the larger number does *not* change: replication traffic, which is +a function of what actually gets written rather than of the quota, and vzdump +archive size, which covers used data only. Both track the databases, not the +headroom. + +Shrinking is not available — it is a replacement, and `prevent_destroy` blocks +it — so overshoot rather than raising this every few months. + +## Failing over + +Replication makes the far copy a volume, not a running guest, so failover is +deliberate. Planned, with both nodes up, is an ordinary migration — fast, +because replication means only the delta has to ship: + +```sh +ssh root@turtle-proxmox-01 'pct migrate 161 turtle-proxmox-02 --restart' +``` + +Unplanned, with the source node gone, means telling Proxmox to run the guest +from the replicated volume, and it loses every transaction committed since the +last successful send — up to `replication_schedule`, five minutes. Check what +you are about to accept first: + +```sh +ssh root@turtle-proxmox-02 'pvesr status' # look at "Last Sync" +``` + +Automating this is what a HA resource would add, and it is deliberately not +configured: automatic failover on a two-node cluster with no third vote is a +good way to get both nodes deciding they are the survivor. + +## Afterwards: repointing clients + +The address is written down in several files that cannot discover it. The +authoritative copy is `ip_address` on the `postgres` module in +`build/config/terraform/main.tf`, published as the `postgres_address` output. + +| Where | What | +|---|---| +| `inventory/host_vars/.yml` | `db.provision_host` on each app that declares one | +| `src//ansible/proxmox/vars.yml` | that app's own `DB_HOST` | +| `src//ansible/unraid/vars.yml` | ditto, for anything still on Unraid — no `unraid_shared` network alias reaches this host | + +Apps on the k3s cluster keep using the in-cluster CloudNativePG instance and +are unaffected. + +## Migrating an existing database onto it + +Per database, not `pg_dumpall` — the roles are recreated by the `db:` block on +each app's own entry, so only the data needs moving: + +```sh +pg_dump -h -U postgres -Fc forgejo > forgejo.dump +# create the role and database by deploying the app once with its `db:` block, +# then: +pg_restore -h 192.168.50.54 -U postgres -d forgejo --no-owner --role=forgejo forgejo.dump +``` + +`--no-owner --role=` rather than a straight restore: the dump carries +ownership from the old cluster, and the role provisioning in `lxc_app` / +`compose_stack` has already created the owner here with a Vault-managed +password. Restoring ownership from the dump would fight it. + +Stop the application before dumping. A dump of a live database is consistent +as of its start, so anything written during it is silently absent from the +restore. diff --git a/docs/vault-secrets.md b/docs/vault-secrets.md new file mode 100644 index 0000000..25ad4e0 --- /dev/null +++ b/docs/vault-secrets.md @@ -0,0 +1,551 @@ +# Vault secret layout + +KV v2 mount: `kv` (matches `vault_kv_mount` in +`build/config/ansible/inventory/group_vars/all.yml`). +One path per stack, fetched whole via `community.hashi_vault.vault_kv2_get`. + +Paths are prefixed `homelab/`, not `unraid/`: an app may run as a Compose +stack on Unraid or natively in a Proxmox LXC, and its secrets are the same +either way. If you populated the old `unraid/*` paths before this rename, +move them and delete the originals: + +```sh +for p in shared/postgres forgejo ci/ssh; do + vault kv get -format=json "kv/unraid/$p" \ + | jq -r '.data.data | to_entries | map("\(.key)=\(.value)") | .[]' \ + | xargs vault kv put "kv/homelab/$p" + vault kv metadata delete "kv/unraid/$p" +done +``` + +## Prerequisites + +- [Vault CLI](https://developer.hashicorp.com/vault/docs/install) installed + on whatever machine you're running these commands from (your workstation + during bootstrap — not the Unraid box, and not the same thing as the + Ansible controller needing `hvac`, though in practice it's the same WSL + shell for both). +- `VAULT_ADDR` set to your Vault instance: + + ```sh + export VAULT_ADDR='https://vault.example.internal:8200' + ``` + + If Vault is using a self-signed cert, either trust it properly or, for a + homelab-only instance, set `export VAULT_SKIP_VERIFY=true` — don't do this + against anything reachable outside your LAN. + +## Login + +Log in interactively once per shell session (or whenever the token expires); +the resulting token is cached to `~/.vault-token` and picked up automatically +by later `vault` commands and by `VAULT_TOKEN` if you export it. + +Token auth (simplest, fine for a single-operator homelab): + +```sh +vault login +# prompts for the token +``` + +If you're using a different auth backend, log in with that method instead — +adjust to whatever your Vault instance actually has enabled: + +```sh +vault login -method=userpass username= +vault login -method=oidc # opens a browser +vault login -method=ldap username= +``` + +Confirm the login worked and check the token's TTL: + +```sh +vault token lookup +``` + +## One-time setup: enable the KV v2 mount + +Only needed once, the first time this Vault instance is used for this repo +(skip if `kv/` already exists — check with `vault secrets list`): + +```sh +vault secrets enable -path=kv -version=2 kv +``` + +## `homelab/shared/postgres` + +| Key | Purpose | +|---|---| +| `POSTGRES_SUPERUSER` | Superuser name (matches `vars.yml` default `postgres`, but Vault is authoritative since it's paired with the password below) | +| `POSTGRES_SUPERUSER_PASSWORD` | Superuser password | + +```sh +vault kv put kv/homelab/shared/postgres \ + POSTGRES_SUPERUSER=postgres \ + POSTGRES_SUPERUSER_PASSWORD='' +``` + +Also seeds the k3s cluster's shared Postgres (`roles/k3s_postgres`, +CloudNativePG) — a separate physical instance from the Unraid/Proxmox ones, +but the same superuser identity, same path. See README.md → "Postgres +(CloudNativePG)". + +## `homelab/forgejo` + +| Key | Purpose | +|---|---| +| `DB_PASSWORD` | Password for the `forgejo` Postgres role (Ansible creates the role with this password) | +| `SECRET_KEY` | Forgejo `security.SECRET_KEY` | +| `INTERNAL_TOKEN` | Forgejo `security.INTERNAL_TOKEN` | +| `JWT_SECRET` | Forgejo `oauth2.JWT_SECRET` | +| `LFS_JWT_SECRET` | Forgejo `server.LFS_JWT_SECRET` — **Proxmox only**, see below | +| `OIDC_CLIENT_SECRET` | Authentik OAuth2 provider secret — **optional**, see below | + +Generate the Forgejo secrets once and store them, rather than letting Forgejo +auto-generate on first boot — that keeps a from-scratch redeploy (fresh data +volume) reproducible instead of silently rotating tokens: + +```sh +docker run --rm codeberg.org/forgejo/forgejo:10 forgejo generate secret +``` + +Run that four times (once per secret) and write them all together with the +DB password: + +```sh +vault kv put kv/homelab/forgejo \ + DB_PASSWORD='' \ + SECRET_KEY='' \ + INTERNAL_TOKEN='' \ + JWT_SECRET='' \ + LFS_JWT_SECRET='' +``` + +`LFS_JWT_SECRET` is only consumed by the Proxmox native install, which renders +`app.ini` in full and sets `INSTALL_LOCK` — nothing is left for Forgejo to +generate on first boot, and a value it invented for itself would be overwritten +on the next deploy anyway. The Unraid Compose stack doesn't reference it; the +key being present just means one extra unused line in the rendered `.env`. If +you only run the Unraid stack today, add it before the Proxmox cutover — +`src/forgejo/ansible/proxmox/install.yml` asserts on it rather than failing +three tasks later with an undefined-variable error: + +```sh +vault kv patch kv/homelab/forgejo LFS_JWT_SECRET='' +``` + +### `OIDC_CLIENT_SECRET` + +Only needed if Forgejo signs users in through Authentik (step 8 of +`docs/forgejo-proxmox.md`). Unlike every other key on this path it is **not +read by Ansible** — nothing templates it, and `install.yml` does not assert on +it, so its absence is not an error. Forgejo keeps authentication sources in +its database, not in `app.ini`, so the value is consumed once by a +`forgejo admin auth add-oauth` run and lives in the database from then on. + +It is kept here anyway, rather than only in Authentik, for the same reason as +the rest: a rebuild that restores from a `pg_dump` gets the auth source back +with the dump, but a rebuild from *nothing* has to re-run that command, and +this is where it looks for the value. Generated by Authentik when you create +the provider, not by you: + +```sh +vault kv patch kv/homelab/forgejo OIDC_CLIENT_SECRET='' +``` + +The matching client ID is not a secret and is not stored here — it is an +argument to that one command, recorded in the runbook step alongside it. + +## `homelab/arr` + +| Key | Purpose | +|---|---| +| `SECRET_KEY_BASE` | Signs sessions and cookies | +| `ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY` | Encrypts every credential Shelfarr stores — indexer API keys, download client passwords, OIDC secrets | +| `ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY` | As above, for values Shelfarr needs to query on | +| `ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT` | As above | +| `SHELFARR_SETTING_OIDC_CLIENT_SECRET` | Client secret of the Authentik provider for Shelfarr | +| `PROWLARR_API_KEY` | Prowlarr's API key, for anything integrating with it | + +One path for the whole stack, not one per container — `compose_stack` fetches +`vault_path` once and renders every key into a single `.env` that both +services read, so Prowlarr's key belongs here rather than under a path of its +own. + +Shelfarr's entrypoint generates the first four on first run if they're unset, into +`/rails/storage/.secret_key_base` and `/rails/storage/.encryption_keys` — i.e. +into the appdata volume. Set them explicitly for the same reason Forgejo's +three secrets are fixed above: a from-scratch redeploy on a fresh volume would +otherwise generate new ones, and everything encrypted with the old values +becomes unreadable. + +```sh +vault kv put kv/homelab/arr \ + SECRET_KEY_BASE="$(openssl rand -hex 64)" \ + ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY="$(openssl rand -base64 32)" \ + ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY="$(openssl rand -base64 32)" \ + ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT="$(openssl rand -base64 32)" +``` + +Set them **before the first start**, and set the three encryption keys as a +group — the entrypoint only falls back to its generated file when the primary +key is absent, so a partial set silently mixes provided and generated values. +Changing them after Shelfarr has stored anything means re-entering every +credential through Admin → Settings. + +> **Do not add `RAILS_MASTER_KEY` here.** Despite the name it is not a +> generatable secret: it decrypts the `config/credentials.yml.enc` compiled +> into the upstream image, so only upstream's own key works. Supplying a value +> of your own aborts startup with +> `ActiveSupport::MessageEncryptor::InvalidMessage` during `db:prepare`. + +The OIDC client secret is the one key here that isn't generated — copy it off +the Authentik provider: + +```sh +vault kv patch kv/homelab/arr \ + SHELFARR_SETTING_OIDC_CLIENT_SECRET='' +``` + +`SHELFARR_SETTING_` is how upstream maps environment variables onto the +Admin → Settings store; the non-secret half of the OIDC config lives in +`src/arr/common/vars.yml`. Both halves have to be named in +`docker-compose.yml`'s `environment:` — `compose_stack` renders every Vault +key and every `env_defaults` entry into `.env`, but Compose reads `.env` only +to interpolate `${...}` in the compose file. A var that nothing references +never reaches the container, and Shelfarr just carries on with OIDC off. + +Prowlarr's API key is generated the same way as Shelfarr's secrets, and for +the same reason — left unset, Prowlarr writes one of its own into +`/config/config.xml` on first start, where the only way to find it is the UI: + +```sh +vault kv patch kv/homelab/arr PROWLARR_API_KEY="$(openssl rand -hex 16)" +``` + +Prowlarr binds config-file settings to environment as `PROWLARR__` plus the +section and key, so this reaches it as `PROWLARR__AUTH__APIKEY`. Releases +before the auth section was split read the same value as `PROWLARR__APIKEY`; +an unrecognised variable is ignored rather than fatal, so the symptom of the +wrong form is a key in the UI that doesn't match Vault, not a container that +won't start. + +## `homelab/k3s-homelab-utils` + +| Key | Purpose | +|---|---| +| `K3S_TOKEN` | Cluster join token — shared by the server and every agent | + +Not app config — this is what makes `playbooks/k3s.yml` reproducible. +`k3s server` normally invents a token on first install and writes it to +`/var/lib/rancher/k3s/server/node-token`; every agent then has to be told that +value. Pinning it in Vault instead means a full rebuild (wipe both SD cards, +reinstall) reproduces the same cluster identity, because both the control +plane and every worker fetch the *same* value independently rather than one +generating it and handing it to the other. + +Generate it once, before the first run of `playbooks/k3s.yml`: + +```sh +vault kv put kv/homelab/k3s-homelab-utils \ + K3S_TOKEN="$(openssl rand -hex 32)" +``` + +Changing this after the cluster is up means re-running the playbook against +every node — the server re-issues its cert bundle around the new token and +every agent needs to reconnect with it, which is disruptive enough that +there's no automatic rotation path here, only a manual one. + +## `homelab/authentik` + +| Key | Purpose | +|---|---| +| `AUTHENTIK_SECRET_KEY` | Signs session cookies | +| `AUTHENTIK_POSTGRESQL__PASSWORD` | Password for the `authentik` role on the cluster's shared Postgres | + +The first app on the k3s cluster, deployed by `roles/k3s_app` from the +`k3s_apps:` list in `inventory/group_vars/k3s_cluster.yml`. Both keys are +named for the environment variables Authentik itself reads, because +`roles/k3s_app` passes Vault keys through to the app's Kubernetes Secret +verbatim — so one value at `AUTHENTIK_POSTGRESQL__PASSWORD` serves both the +database provisioning step and the running app, with nothing restated. + +```sh +vault kv put kv/homelab/authentik \ + AUTHENTIK_SECRET_KEY='' \ + AUTHENTIK_POSTGRESQL__PASSWORD="$(openssl rand -base64 32)" +``` + +`AUTHENTIK_SECRET_KEY` is the exception to "generate a fresh secret" — it +must be copied from the existing Unraid deployment, not invented. It signs +session cookies, so a new value logs every user out at the moment DNS moves. +See `docs/authentik-migration.md`, which is the only reason this path has a +"copy the old value" step at all; a from-scratch install would generate both. + +## `homelab/k3s-cert-manager` + +| Key | Purpose | +|---|---| +| `CLOUDFLARE_API_TOKEN` | DNS provider API token for cert-manager's ACME DNS-01 solver | + +Consumed by `roles/k3s_cert_manager`, which renders every key at this path +into a Secret in the `cert-manager` namespace — so the key name above is a +convention, not a requirement: store whatever your DNS provider needs and +reference that name from `k3s_cert_manager_solver` in +`inventory/group_vars/k3s_cluster.yml`. + +```sh +vault kv put kv/homelab/k3s-cert-manager \ + CLOUDFLARE_API_TOKEN='' +``` + +DNS-01 rather than HTTP-01 because HTTP-01 needs Let's Encrypt to reach the +cluster from the internet on port 80, and this LAN deliberately isn't +reachable. Scope the token to the single zone it manages — cert-manager only +ever needs to write `_acme-challenge` TXT records. + +## `homelab/ci/ssh` + +Not consumed by `compose_stack` — this is the SSH private key Ansible itself +connects to the Unraid hosts with. It lives in Vault rather than as a +Forgejo Actions secret so there's one place secrets come from, not two; see +README.md "SSH access" for generating the key pair and installing the public +half on each host. + +| Key | Purpose | +|---|---| +| `PRIVATE_KEY` | OpenSSH private key `ansible_user` (root) authenticates with | + +```sh +# `@path` reads the value from a file, which handles the embedded newlines +# in a private key correctly — don't try to inline it as a normal KEY=value. +# Use $HOME, not `~`: this is a plain command argument, not a shell +# assignment, so bash won't tilde-expand a `~` sitting after the `@` — it'd +# get passed through literally and vault would look for a file named `~`. +vault kv put kv/homelab/ci/ssh PRIVATE_KEY=@"$HOME/.ssh/unraid_ansible" +``` + +## `homelab/ci/ssh-k3s` + +The SSH private key Ansible connects to the 4 k3s Pis with. A separate key +pair from `homelab/ci/ssh` above, deliberately — the two host groups don't +share a trust boundary, so a leaked key for one shouldn't also be a working +key for the other. Same shape as the Unraid key otherwise: + +| Key | Purpose | +|---|---| +| `PRIVATE_KEY` | OpenSSH private key the `ansible` user (per `inventory/hosts.yml` → `k3s_cluster`) authenticates with | + +```sh +ssh-keygen -t ed25519 -f ~/.ssh/k3s_ansible -C "ansible@homelab-utils" -N "" +``` + +Create the `ansible` user on each of the 4 Pis with NOPASSWD sudo and install +the public half as its `authorized_keys` — see README.md "SSH access". Then: + +```sh +vault kv put kv/homelab/ci/ssh-k3s PRIVATE_KEY=@"$HOME/.ssh/k3s_ansible" +``` + +Unlike the Unraid key above, nothing needs loading into `ssh-agent` to use +this: `playbooks/k3s_ssh_key.yml` reads this path and writes the key to +`local/k3s/homelab-utils.key` on the controller, and both `playbooks/k3s.yml` +and `playbooks/k3s_maintenance.yml` import it as their first play. A manual +run needs only `VAULT_ADDR`/`VAULT_TOKEN` in the environment (the "Login" +section above — no AppRole involvement yet, both k3s playbooks being +manual-only for now). See README.md "SSH access" for the fallback flags if +Vault is unreachable. + +## `homelab/ci/proxmox` + +Proxmox VE API token, used by two things: the `community.proxmox` dynamic +inventory plugin (`build/config/ansible/inventory/proxmox.yml`), and the +Terraform provider that creates LXCs. A token rather than a password — same +reasoning as the SSH key, a password means an interactive prompt or a +plaintext secret on disk. + +Create the user and its token on either node (ACLs are cluster-wide), then +grant it a role. This is more involved than it looks, and the failure modes +give errors that don't obviously point at permissions: + +```sh +pveum user add ansible@pam +pveum user token add ansible@pam terraform --privsep 0 + +pveum role add TerraformProv -privs "Datastore.Allocate \ + Datastore.AllocateSpace Datastore.AllocateTemplate Datastore.Audit \ + Pool.Allocate SDN.Use Sys.Audit Sys.Console Sys.Modify \ + VM.Allocate VM.Audit VM.Clone VM.Console VM.Migrate VM.PowerMgmt \ + VM.Replicate \ + VM.Config.CPU VM.Config.Disk VM.Config.HWType VM.Config.Memory \ + VM.Config.Network VM.Config.Options" + +pveum acl modify / --users 'ansible@pam' --roles TerraformProv +``` + +Three things that otherwise cost an afternoon: + +- **`--privsep 0` is load-bearing.** A token created with privilege separation + on (the UI default) carries its *own* ACL and ignores what the user was + granted, so `pveum acl modify --users` looks like it did nothing. Either + turn it off as above, or grant the token as well with + `--tokens 'ansible@pam!terraform'`. With a purpose-made user rather than + `root@pam` there is nothing gained by maintaining both layers — least + privilege is already enforced on the user. +- **Privilege names are version-sensitive.** `VM.Monitor` appears in most + guides for this and is rejected outright by current PVE (it is a QEMU + monitor privilege, meaningless for LXC). `SDN.Use` is required from PVE 8.2 + on, where attaching a container to `vmbr0` is gated on + `/sdn/zones/localnetwork/vmbr0`. Check against your own node rather than + against any list, this one included: `pvesh get /access/roles/Administrator` + prints the complete valid set. +- **`VM.Replicate` is there for the shared Postgres**, whose module creates a + `pvesr` job (`replication_target_node` — see + `src/shared/postgres/terraform/`). A module without replication won't need + it. + +Verify the grant landed before reaching for Terraform — this answers the +token's effective permissions, not the user's: + +```sh +pveum user permissions ansible@pam --token terraform +``` + +Every `terraform apply` failure here is an HTTP 403 naming the exact missing +privilege, so the recovery loop is `pveum role modify TerraformProv --privs +"" --append` and re-run. Note `--append`: without it, `--privs` replaces +the whole list rather than adding to it. + +Then: + +| Key | Purpose | +|---|---| +| `PROXMOX_URL` | API endpoint, e.g. `https://turtle-proxmox-01.home.turtlesystems.co.uk:8006/` | +| `PROXMOX_USER` | Token owner, e.g. `ansible@pam` | +| `PROXMOX_TOKEN_ID` | Token ID (the part after `!`) | +| `PROXMOX_TOKEN_SECRET` | Token secret — shown once at creation | + +```sh +vault kv put kv/homelab/ci/proxmox \ + PROXMOX_URL='https://turtle-proxmox-01.home.turtlesystems.co.uk:8006/' \ + PROXMOX_USER='ansible@pam' \ + PROXMOX_TOKEN_ID='terraform' \ + PROXMOX_TOKEN_SECRET='' +``` + +Ansible's inventory plugin reads these from the environment, so export them +before running anything that touches `proxmox_guests`. Terraform wants the +token in one combined string instead: + +```sh +export TF_VAR_proxmox_api_token="${PROXMOX_USER}!${PROXMOX_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}" +``` + +## `homelab/ci/terraform` + +Connection string for the Postgres database Terraform keeps its state in +(`backend "pg"` — see `build/config/terraform/README.md`). Separate from the +superuser credentials above: Terraform gets its own role, scoped to its own +database. + +| Key | Purpose | +|---|---| +| `PG_CONN_STR` | `postgres://terraform:@192.168.50.81:5432/terraform_state` | + +The host is the **CloudNativePG cluster on k3s** +(`k3s_postgres_loadbalancer_ip`), not either shared Postgres this repo +deploys — Terraform cannot keep its state in a database it provisions itself. +Create the role and database once, against that cluster, then store the +string: + +```sh +vault kv put kv/homelab/ci/terraform \ + PG_CONN_STR='postgres://terraform:@192.168.50.81:5432/terraform_state' +``` + +## Verifying what was written + +```sh +vault kv get kv/homelab/shared/postgres +vault kv get kv/homelab/forgejo +``` + +`vault kv put` (used above) replaces the whole secret at that path — to add +or change a single key without touching the others, use `vault kv patch` +instead: + +```sh +vault kv patch kv/homelab/forgejo DB_PASSWORD='' +``` + +## Adding a new stack's secrets + +```sh +vault kv put kv/homelab/ KEY1=value1 KEY2=value2 +``` + +Then reference the same path as `vault_path` in the stack's entry under +`build/config/ansible/inventory/host_vars/.yml`. + +## AppRole for CI (Forgejo Actions) + +The bootstrap commands above use your own human token. The Forgejo Actions +runner (`.forgejo/workflows/deploy.yml`) authenticates as an AppRole instead, +scoped to read-only on the `kv/homelab/*` paths. One-time setup: + +```sh +# 1. Policy: read-only on every stack's secrets +vault policy write unraid-deploy - <<'EOF' +path "kv/data/homelab/*" { + capabilities = ["read"] +} +EOF + +# 2. Enable AppRole auth (skip if already enabled — check `vault auth list`) +vault auth enable approle + +# 3. Create the role, bound to that policy +vault write auth/approle/role/unraid-deploy \ + token_policies="unraid-deploy" \ + token_ttl=15m \ + token_max_ttl=1h + +# 4. Fetch the role ID (stable, not secret on its own) +vault read auth/approle/role/unraid-deploy/role-id + +# 5. Generate a secret ID (treat as a secret — shown once) +vault write -f auth/approle/role/unraid-deploy/secret-id +``` + +Store the `role_id` and `secret_id` from steps 4–5 as `VAULT_ROLE_ID` and +`VAULT_SECRET_ID` in the Forgejo repo's Actions secrets (alongside +`VAULT_ADDR`) — that's what `deploy.yml` and +`build/config/ansible/inventory/group_vars/all.yml` +(`vault_auth_method: approle`) expect. + +These never live in a file in this repo, in either mode: + +- **CI:** Forgejo repo → Settings → Actions → Secrets. `deploy.yml` reads + them from there via `${{ secrets.VAULT_ROLE_ID }}` / + `${{ secrets.VAULT_SECRET_ID }}` and exports them as job env vars. +- **Manual runs** (bootstrap, or any ad hoc `ansible-playbook` invocation): + export them as shell env vars first — `group_vars/all.yml` picks them up + via `lookup('env', ...)`: + + ```sh + export VAULT_AUTH_METHOD=approle + export VAULT_ROLE_ID='' + export VAULT_SECRET_ID='' + ``` + + (Bootstrap can also just use your own human token instead — leave + `VAULT_AUTH_METHOD` unset, it defaults to `token`, and `vault login` from + the "Login" section above is enough.) + +Secret IDs can be regenerated (step 5) and old ones revoked without touching +the role itself if one ever leaks: + +```sh +vault write auth/approle/role/unraid-deploy/secret-id-accessor/destroy \ + secret_id_accessor='' +``` diff --git a/src/arr/ansible/proxmox/README.md b/src/arr/ansible/proxmox/README.md new file mode 100644 index 0000000..4bfc58c --- /dev/null +++ b/src/arr/ansible/proxmox/README.md @@ -0,0 +1,22 @@ +# ansible/proxmox/ — arr + +Not built. Shelfarr runs as a Compose stack on nas1 (`../unraid/`), and there +is no reason to move it — this folder exists because every app keeps the same +shape whether or not both platforms are in use. + +A native install would be a poor fit here in a way most apps aren't. Shelfarr +is a Rails app shipping two containers' worth of runtime (Ruby, a bundled +Solid Queue worker, and a .NET companion for the Libation sidecar), with no +released binary or package — upstream distributes container images only. Where +`forgejo` drops a single Go binary and a systemd unit into an LXC, +reproducing this natively means building the Rails app from source on the +guest. + +If it ever needs to move, the realistic option is Docker inside a privileged +LXC rather than the `lxc_app` role, which would be a new deployment primitive +for this repo — decide that deliberately rather than by extending the role. + +What wouldn't change: `../../common/vars.yml` already holds the portable +config (version, published port), and the Vault path stays `homelab/arr`. +Only paths and PUID/PGID are Unraid-specific, and those live in +`../unraid/vars.yml`. diff --git a/src/arr/ansible/unraid/.env.example b/src/arr/ansible/unraid/.env.example new file mode 100644 index 0000000..501d278 --- /dev/null +++ b/src/arr/ansible/unraid/.env.example @@ -0,0 +1,101 @@ +# Reference only — real values are rendered by Ansible from vars.yml (non- +# secret) and Vault path homelab/arr (secret). Do not fill this in and +# deploy it directly. + +# Non-secret (see ../../common/vars.yml) +SHELFARR_VERSION=2026.08.05.1 +SHELFARR_HTTP_PORT=5056 + +# Non-secret (see vars.yml) — check the media paths against your own shares +SHELFARR_DATA_PATH=/mnt/user/appdata/arr/data +SHELFARR_AUDIOBOOKS_PATH=/mnt/user/Media/AudioBooks +SHELFARR_EBOOKS_PATH=/mnt/user/Media/Books +SHELFARR_DOWNLOADS_PATH=/mnt/user/downloads +PUID=99 +PGID=100 +CHOWN_ON_START=auto + +# Docker network to join, and it must already exist on the target host — +# Compose declares it `external`, so it will not create it. Defined in +# Unraid's Docker settings, not by any stack in this repo. +SHELFARR_NETWORK=caddy-net + +# Non-secret, supplied by the compose_stack role rather than either vars.yml — +# the path it copies ansible/unraid/icon.png to on the host, for the +# net.unraid.docker.icon label. Empty when the stack ships no icon.png, which +# just leaves Unraid's question-mark placeholder. Override it in vars.yml to +# point at a hosted PNG instead. +STACK_ICON=/mnt/user/appdata/icons/arr.png + +# Secret — Vault key: SECRET_KEY_BASE +# Signs sessions and cookies. Generate with: +# openssl rand -hex 64 +SECRET_KEY_BASE= + +# Secret — Vault keys: ACTIVE_RECORD_ENCRYPTION_* +# Encrypt every credential Shelfarr stores (indexer API keys, download client +# passwords). Generate each with: +# openssl rand -base64 32 +# Set these before the first start, and set all three together. Changing them +# later makes everything already encrypted with the old values unreadable. +ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY= +ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY= +ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT= + +# Non-secret (see ../../common/vars.yml) — OIDC against Authentik. +# Upstream maps environment onto the settings store by SHELFARR_SETTING_ plus +# the uppercased setting key. There is no redirect-URI variable: the callback +# is fixed at /auth/oidc/callback and is registered on the +# Authentik provider. +SHELFARR_SETTING_OIDC_ENABLED=true +SHELFARR_SETTING_OIDC_PROVIDER_NAME=Authentik +SHELFARR_SETTING_OIDC_ISSUER=https://auth.turtlesystems.uk/application/o/shelfarr/ +SHELFARR_SETTING_OIDC_CLIENT_ID= +SHELFARR_SETTING_OIDC_SCOPES=openid profile email +SHELFARR_SETTING_OIDC_AUTO_CREATE_USERS=true +SHELFARR_SETTING_OIDC_DEFAULT_ROLE=user +SHELFARR_SETTING_OIDC_LINK_EXISTING_USERS=true +SHELFARR_SETTING_OIDC_AUTO_REDIRECT=false + +# Secret — Vault key: SHELFARR_SETTING_OIDC_CLIENT_SECRET +# The client secret of the Authentik provider for Shelfarr. +SHELFARR_SETTING_OIDC_CLIENT_SECRET= + +# Do NOT set RAILS_MASTER_KEY. It decrypts the credentials file built into the +# image, so only upstream's key works — a self-generated one aborts startup +# with ActiveSupport::MessageEncryptor::InvalidMessage. + +# Only if the Libation companion is enabled — see docker-compose.yml +# LIBATION_CONFIG_PATH=/mnt/user/appdata/arr/libation/config +# LIBATION_BOOKS_PATH=/mnt/user/appdata/arr/libation/books +# LIBATION_CONTROL_PATH=/mnt/user/appdata/arr/libation/control + +# Only if running behind a reverse proxy at a sub-path, e.g. /arr +# RAILS_RELATIVE_URL_ROOT=/ + +# --- Prowlarr --------------------------------------------------------------- + +# Non-secret (see ../../common/vars.yml). LinuxServer tags are the upstream +# version plus their build suffix (1.37.0.5076-ls117); pin to a full one read +# off the registry rather than leaving this at latest. +PROWLARR_VERSION=latest +PROWLARR_HTTP_PORT=9696 + +# Non-secret, supplied by the compose_stack role — the path it copies +# icon-prowlarr.png to, for the prowlarr container's net.unraid.docker.icon +# label. Per-service, so it's separate from STACK_ICON above, which is the +# stack's own icon.png and labels shelfarr. +STACK_ICON_PROWLARR=/mnt/user/appdata/icons/arr-prowlarr.png + +# Non-secret (see vars.yml) +PROWLARR_CONFIG_PATH=/mnt/user/appdata/arr/prowlarr +TZ=Europe/London + +# Secret — Vault key: PROWLARR_API_KEY +# Bound to Prowlarr's config.xml via PROWLARR__AUTH__APIKEY, so the key is +# known before first boot and anything integrating with Prowlarr can be +# configured from the same Vault path. Generate with: +# openssl rand -hex 16 +# Leave it out and Prowlarr generates its own on first start — then the value +# only exists in the UI and in /config/config.xml. +PROWLARR_API_KEY= diff --git a/src/arr/ansible/unraid/docker-compose.yml b/src/arr/ansible/unraid/docker-compose.yml new file mode 100644 index 0000000..eab6d4f --- /dev/null +++ b/src/arr/ansible/unraid/docker-compose.yml @@ -0,0 +1,252 @@ +services: + shelfarr: + image: ghcr.io/pedro-revez-silva/shelfarr:${SHELFARR_VERSION} + container_name: arr + restart: unless-stopped + # Unraid's Docker page normally reads icon, WebUI link and console shell + # from the dockerMan template that created the container. A Compose stack + # has no template, so 6.10+ falls back to these labels; without them the + # container renders as a question mark with no WebUI or Console entry. + # + # STACK_ICON comes from the compose_stack role (a path under + # /mnt/user/appdata/icons, or a URL if vars.yml overrides it). `[IP]` is + # substituted by the webgui with the host's address; the port has to be + # the published one, hence SHELFARR_HTTP_PORT rather than the container's + # 80. If you drop the `ports:` mapping below and reach Shelfarr only + # through Caddy, replace the whole value with the proxied URL — the + # webgui has no way to know about the reverse proxy. + labels: + net.unraid.docker.icon: ${STACK_ICON} + net.unraid.docker.webui: "http://[IP]:${SHELFARR_HTTP_PORT}/" + net.unraid.docker.shell: bash + networks: + - proxy + environment: + # Runs the Solid Queue background worker inside the Puma process rather + # than as a second container. Upstream's own compose does this; Shelfarr + # is a single-user-scale app and a separate worker buys nothing here. + SOLID_QUEUE_IN_PUMA: "1" + PUID: ${PUID} + PGID: ${PGID} + CHOWN_ON_START: ${CHOWN_ON_START} + # Deliberately NOT RAILS_MASTER_KEY. That variable decrypts the + # config/credentials.yml.enc baked into the image at build time, so only + # upstream's own key works — setting it to a generated value aborts boot + # with ActiveSupport::MessageEncryptor::InvalidMessage. + # + # These four are the ones the entrypoint actually treats as overridable, + # and pinning them is what stops a fresh appdata volume from orphaning + # stored credentials (same reasoning as forgejo's SECRET_KEY). Left + # unset, the entrypoint generates them into /rails/storage/.secret_key_base + # and /rails/storage/.encryption_keys — fine until that directory is lost. + SECRET_KEY_BASE: ${SECRET_KEY_BASE} + # Encrypt every credential Shelfarr stores (indexer API keys, download + # client passwords). All three must be set together; the entrypoint only + # falls back to its generated file when the primary key is absent. + ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY: ${ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY} + ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY: ${ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY} + ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT: ${ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT} + # --- OIDC (Authentik) ------------------------------------------------ + # + # Upstream maps environment onto the Admin → Settings store by + # `SHELFARR_SETTING_` + the uppercased setting key, so these ten are the + # whole of the SSO config; there is no redirect-URI variable, the + # callback path is fixed at /auth/oidc/callback and is registered on the + # Authentik side. + # + # Every one has to be named here. The .env `compose_stack` renders holds + # them already, but Compose reads .env only to interpolate `${...}` in + # this file — it does not inject it into the container. An unreferenced + # var reaches the host's Compose process and stops there, which is why + # OIDC stayed off with the values apparently set. + SHELFARR_SETTING_OIDC_ENABLED: ${SHELFARR_SETTING_OIDC_ENABLED} + SHELFARR_SETTING_OIDC_PROVIDER_NAME: ${SHELFARR_SETTING_OIDC_PROVIDER_NAME} + SHELFARR_SETTING_OIDC_ISSUER: ${SHELFARR_SETTING_OIDC_ISSUER} + SHELFARR_SETTING_OIDC_CLIENT_ID: ${SHELFARR_SETTING_OIDC_CLIENT_ID} + # Secret — Vault key of the same name, not either vars.yml. + SHELFARR_SETTING_OIDC_CLIENT_SECRET: ${SHELFARR_SETTING_OIDC_CLIENT_SECRET} + SHELFARR_SETTING_OIDC_SCOPES: ${SHELFARR_SETTING_OIDC_SCOPES} + SHELFARR_SETTING_OIDC_AUTO_CREATE_USERS: ${SHELFARR_SETTING_OIDC_AUTO_CREATE_USERS} + SHELFARR_SETTING_OIDC_DEFAULT_ROLE: ${SHELFARR_SETTING_OIDC_DEFAULT_ROLE} + SHELFARR_SETTING_OIDC_LINK_EXISTING_USERS: ${SHELFARR_SETTING_OIDC_LINK_EXISTING_USERS} + SHELFARR_SETTING_OIDC_AUTO_REDIRECT: ${SHELFARR_SETTING_OIDC_AUTO_REDIRECT} + volumes: + - ${SHELFARR_DATA_PATH}:/rails/storage + - ${SHELFARR_AUDIOBOOKS_PATH}:/audiobooks + - ${SHELFARR_EBOOKS_PATH}:/ebooks + - ${SHELFARR_DOWNLOADS_PATH}:/downloads + ports: + - "${SHELFARR_HTTP_PORT}:80" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:80/up"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + + # Indexer manager for the rest of the stack. Shelfarr doesn't talk to + # trackers itself — it asks Prowlarr, which owns the indexer definitions and + # their credentials. Both are on `proxy`, so Shelfarr reaches it as + # `http://prowlarr:9696` (Admin → Settings → Indexers) rather than via nas1's + # LAN IP and the published port. + prowlarr: + image: lscr.io/linuxserver/prowlarr:${PROWLARR_VERSION} + container_name: prowlarr + restart: unless-stopped + # Same reasoning as the labels on `shelfarr` above — a Compose stack has no + # dockerMan template, so without these the container is a question mark + # with no WebUI entry. STACK_ICON_PROWLARR rather than STACK_ICON: labels + # are per-service, and ${STACK_ICON} is the stack's own icon.png, which is + # Shelfarr's. This one comes from the committed icon-prowlarr.png in this + # directory — see README.md → "Icons on the Unraid Docker page". + labels: + net.unraid.docker.icon: ${STACK_ICON_PROWLARR} + net.unraid.docker.webui: "http://[IP]:${PROWLARR_HTTP_PORT}/" + # Alpine-based, so `sh` — same as forgejo, not `bash` like the + # Debian-based shelfarr above. + net.unraid.docker.shell: sh + networks: + - proxy + environment: + PUID: ${PUID} + PGID: ${PGID} + # LinuxServer's init sets the container clock from this. It matters more + # here than for Shelfarr: Prowlarr timestamps search history and applies + # per-indexer rate limits, and both read wrong at UTC. + TZ: ${TZ} + # Prowlarr binds its config file to environment via `PROWLARR__` plus the + # config section and key, double-underscore separated. Pinning the API + # key means Shelfarr's indexer config can be written from the same Vault + # path instead of being copied by hand out of the UI after first boot — + # the same argument as SECRET_KEY_BASE above, applied to a value that is + # otherwise generated into /config/config.xml on first start. + # + # Older releases read this as PROWLARR__APIKEY (no section). If the key + # in the UI doesn't match Vault after a deploy, that's which form this + # image wants; an unrecognised variable is ignored silently rather than + # failing the container, so the symptom is a generated key, not a crash. + PROWLARR__AUTH__APIKEY: ${PROWLARR_API_KEY} + volumes: + # config.xml, the indexer definitions, and Prowlarr's own SQLite + # database. This directory *is* the application state — same standing as + # SHELFARR_DATA_PATH, back it up the same way. + - ${PROWLARR_CONFIG_PATH}:/config + ports: + - "${PROWLARR_HTTP_PORT}:9696" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9696/ping"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + +networks: + # `external: true` means "attach to a network that already exists; don't + # create it, don't delete it on `down`". Without it Compose would create a + # *new* network named `arr_` — the deploy would still succeed and + # Caddy would silently fail to resolve `arr`. `name:` is what stops the + # project-name prefix being applied. + # + # The local alias is a fixed `proxy` so the compose file stays static; which + # real network it maps to is a per-host decision in vars.yml + # (SHELFARR_NETWORK). Same shape as forgejo's `shared` → `unraid_shared`. + # + # The alias MUST be a literal, not `${SHELFARR_NETWORK}:`. Compose + # interpolates values, never mapping keys, so a variable used as the key + # stays the literal string `${SHELFARR_NETWORK}` while the service's + # `networks:` entry (a list value) expands to `caddy-net` — and the two no + # longer match: + # service "arr" refers to undefined network caddy-net + # `name:` below is a value, so it interpolates normally. That asymmetry is + # the whole reason for the alias indirection. + # + # caddy-net is defined in Unraid's Docker settings on nas1 rather than by a + # stack in this repo, so nothing here has to deploy before Shelfarr does. + # But it must exist on the target host — an absent external network fails + # the deploy outright, which is why `unraid_shared` (created by the shared + # Postgres stack, and that runs on nas2) is deliberately not used here. + # Shelfarr is SQLite-only and needs no database anyway. + proxy: + name: ${SHELFARR_NETWORK} + external: true + +# Now that Caddy and Shelfarr share a network, Caddy should proxy to +# `arr:80` — the container name on a user-defined network — rather than +# to nas1's LAN IP and the published port. The `ports:` mapping above is kept +# only for direct access that bypasses the proxy; drop it if you don't want +# 5056 reachable on the LAN. +# +# Reaching the download client and indexers stays a runtime concern, not a +# compose one: they're configured in Admin → Settings. Anything not on +# caddy-net is addressed via nas1's LAN IP and its published port +# (e.g. http://192.168.50.1:8080), which works whichever network it's on. + +# --- Import layout --------------------------------------------------------- +# +# The three media binds above follow upstream's documented layout, which works +# regardless of how your shares are arranged. The cost: /downloads and +# /audiobooks are separate mounts inside the container, and rename() across +# mounts returns EXDEV even when both sit on the same Unraid filesystem — so +# every import is a full copy rather than an instant move. +# +# If downloads and the library live under one share, replace the three binds +# with a single one and set the paths in Admin → Settings to match: +# +# - ${SHELFARR_MEDIA_PATH}:/media # e.g. /mnt/user/media +# +# Admin → Settings: /media/downloads, /media/audiobooks, /media/ebooks +# +# One mount means one filesystem, so imports become atomic moves and hardlinks +# work. Worth doing if your download client is already writing under the same +# share. + +# --- Libation companion (Audible backup, beta) ----------------------------- +# +# Left out for now; purely additive. To enable it: uncomment the service and +# the `volumes:` block below, add these three to the main service's +# environment, and add the two mounts to its `volumes:`, then redeploy. The +# appdata bind survives the container recreate, so nothing is lost. +# +# environment: +# SHELFARR_LIBATION_URL: http://shelfarr-libation:8080 +# SHELFARR_LIBATION_TOKEN_FILE: /run/shelfarr-libation/token +# SHELFARR_LIBATION_IMPORT_ROOT: /imports/libation +# volumes: +# - ${LIBATION_BOOKS_PATH}:/imports/libation:ro +# - ${LIBATION_CONTROL_PATH}:/run/shelfarr-libation:ro +# +# Note the two services must then share a user-defined network so the main +# container can resolve `shelfarr-libation` by name — the default bridge has no +# DNS between containers. +# +# shelfarr-libation: +# image: ghcr.io/pedro-revez-silva/shelfarr-libation:${SHELFARR_VERSION} +# container_name: shelfarr-libation +# restart: unless-stopped +# expose: +# - "8080" +# environment: +# PUID: ${PUID} +# PGID: ${PGID} +# CHOWN_ON_START: ${CHOWN_ON_START} +# LIBATION_FILES_DIR: /config +# LIBATION_BOOKS_DIR: /data +# LIBATION_IN_PROGRESS_DIR: /config/in-progress +# COMPANION_STATE_DIR: /config/shelfarr-companion +# COMPANION_TOKEN_FILE: /control/token +# ASPNETCORE_URLS: http://0.0.0.0:8080 +# volumes: +# - ${LIBATION_CONFIG_PATH}:/config +# - ${LIBATION_BOOKS_PATH}:/data +# - ${LIBATION_CONTROL_PATH}:/control +# healthcheck: +# test: ["CMD", "/companion/Shelfarr.Libation.Companion", "--healthcheck"] +# interval: 30s +# timeout: 10s +# retries: 3 +# start_period: 40s +# +# Upstream uses named volumes for libation_config/books/control. Bind mounts +# under /mnt/user/appdata/arr/ instead, to match every other stack here +# and so the data is visible on the array rather than buried in Docker's +# storage. Add the paths to ansible/unraid/vars.yml when enabling. diff --git a/src/arr/ansible/unraid/icon-prowlarr.png b/src/arr/ansible/unraid/icon-prowlarr.png new file mode 100644 index 0000000..c1b5f2d Binary files /dev/null and b/src/arr/ansible/unraid/icon-prowlarr.png differ diff --git a/src/arr/ansible/unraid/icon-shelfarr.png b/src/arr/ansible/unraid/icon-shelfarr.png new file mode 100644 index 0000000..f81a6aa Binary files /dev/null and b/src/arr/ansible/unraid/icon-shelfarr.png differ diff --git a/src/arr/ansible/unraid/vars.yml b/src/arr/ansible/unraid/vars.yml new file mode 100644 index 0000000..ee9829d --- /dev/null +++ b/src/arr/ansible/unraid/vars.yml @@ -0,0 +1,59 @@ +--- +# Unraid-specific overrides, merged over ../../common/vars.yml (these win). +env_defaults: + # SQLite lives here — all four production databases (primary, cache, queue, + # cable) plus Active Storage. Shelfarr has no external database, so unlike + # forgejo this directory *is* the application state, and it is also where + # everything configured in Admin → Settings is kept. Back it up accordingly. + SHELFARR_DATA_PATH: /mnt/user/appdata/arr/data + + # CHECK THESE AGAINST YOUR OWN SHARES before the first deploy — they are + # conventional Unraid layouts, not something read off nas1. + # + # `/mnt/user/...`, never `/mnt/cache/...` or `/mnt/disk1/...`: the latter + # bypass the user-share layer and strand files where the mover won't find + # them. + # + # See the note in docker-compose.yml about keeping downloads and the library + # on one mount — as three separate binds, imports are copies rather than + # instant moves. + SHELFARR_AUDIOBOOKS_PATH: /mnt/user/Media/Books/audio + SHELFARR_EBOOKS_PATH: /mnt/user/Media/Books/ebooks + SHELFARR_DOWNLOADS_PATH: /mnt/user/downloads + + # 99:100 is nobody:users — what Unraid owns share content as. Upstream + # defaults to 1000:1000, which is a normal Linux desktop user and wrong + # here; files written as 1000 are invisible to everything else on the box. + PUID: "99" + PGID: "100" + + # `auto` chowns only when it has to. `always` would fight Unraid's own + # permissions handling on every restart; `never` risks a container that + # can't write to its own appdata after a share rebuild. + CHOWN_ON_START: auto + + # Prowlarr's config.xml, indexer definitions and SQLite database. Sibling of + # SHELFARR_DATA_PATH under the same stack directory — one appdata folder per + # stack, one subdirectory per container. + PROWLARR_CONFIG_PATH: /mnt/user/appdata/arr/prowlarr + + # Container clock. Prowlarr timestamps its search history and enforces + # per-indexer rate limits against it; Shelfarr takes its time from the Rails + # default and doesn't read this. + TZ: Europe/London + + # The stack's Docker network — both containers join it, which is how + # Shelfarr resolves `prowlarr` by name. + SHELFARR_NETWORK: caddy-net + + # --- Unraid Docker page presentation (net.unraid.docker.* labels) -------- + # + # STACK_ICON is normally left to the compose_stack role, which points it at + # /mnt/user/appdata/icons/.png when this directory contains an + # icon.png. Uncomment to use a hosted icon instead of committing one: + # + # STACK_ICON: https://example.org/arr.png + # + # PNG only — Unraid renders its placeholder for SVG and nothing at all for + # WebP, and a URL it can't reach when the Docker page renders fails the same + # way, which is the argument for committing the file. diff --git a/src/arr/common/vars.yml b/src/arr/common/vars.yml new file mode 100644 index 0000000..69ffa1d --- /dev/null +++ b/src/arr/common/vars.yml @@ -0,0 +1,64 @@ +--- +# Portable config — values that hold regardless of where Shelfarr runs. +# Platform-specific values (filesystem paths, the PUID/PGID the container +# writes as) live in ../ansible//vars.yml and are merged over these +# at deploy time. +env_defaults: + # Pinned rather than `latest`. Upstream tags releases YYYY.MM.DD.N and the + # image tag carries no leading `v`, unlike the git tag — `v2026.08.05.1` is + # the release, `2026.08.05.1` is the image. One tag pins both the app image + # and the (currently unused) Libation companion. + SHELFARR_VERSION: "2026.08.09.1" + + # Published port. 5056 is upstream's convention; the container listens on 80 + # internally, which is left at its default rather than being made + # configurable — there is nothing else in the container to collide with. + SHELFARR_HTTP_PORT: "5056" + + # SECRET_KEY_BASE and the three ACTIVE_RECORD_ENCRYPTION_* values are + # secrets — sourced from Vault (homelab/arr), never set here. The + # encryption trio is not just a session key: it encrypts every credential + # Shelfarr stores (indexer API keys, download client passwords), so losing + # it means re-entering all of them. + # + # RAILS_MASTER_KEY is deliberately absent. It belongs to the + # config/credentials.yml.enc compiled into the image, so any value you + # generate yourself fails to decrypt it and the container won't boot. See + # the note in ../ansible/unraid/docker-compose.yml. + + SHELFARR_SETTING_OIDC_ENABLED: "true" + SHELFARR_SETTING_OIDC_PROVIDER_NAME: Authentik + SHELFARR_SETTING_OIDC_ISSUER: https://auth.turtlesystems.uk/application/o/shelfarr/ + SHELFARR_SETTING_OIDC_CLIENT_ID: jf1IKMxxYjgZxZikvONdFfOTauFrFiC4AYmJEbmN + SHELFARR_SETTING_OIDC_SCOPES: openid profile email + SHELFARR_SETTING_OIDC_AUTO_CREATE_USERS: "true" + SHELFARR_SETTING_OIDC_DEFAULT_ROLE: user + SHELFARR_SETTING_OIDC_LINK_EXISTING_USERS: "true" + SHELFARR_SETTING_OIDC_AUTO_REDIRECT: "false" + + # --- Prowlarr -------------------------------------------------------------- + # + # LinuxServer's tags are the upstream version plus their own build suffix + # (`1.37.0.5076-ls117`), and the suffix can't be derived from the release + # number — so a pin has to be read off the registry rather than guessed. + # `latest` until then; check + # https://github.com/linuxserver/docker-prowlarr/pkgs/container/prowlarr and + # replace this with the full tag, which is what every other version in this + # repo does. + PROWLARR_VERSION: latest + + # Published port. 9696 is upstream's default and what the container listens + # on internally; kept the same on the host so the WebUI label, the LAN URL + # and the docs all read alike. + PROWLARR_HTTP_PORT: "9696" + + # PROWLARR_API_KEY is a secret — Vault (homelab/arr), never set here. + # Generate a 32-character hex string: openssl rand -hex 16 + # Prowlarr generates its own into /config/config.xml if this is left empty, + # which works fine but means the value is only discoverable from the UI. + + # STACK_ICON_PROWLARR is supplied by the compose_stack role, from the + # committed ../ansible/unraid/icon-prowlarr.png — not set here. Uncomment to + # use a hosted icon instead, exactly as with STACK_ICON: + # + # STACK_ICON_PROWLARR: https://example.org/prowlarr.png diff --git a/src/arr/terraform/README.md b/src/arr/terraform/README.md new file mode 100644 index 0000000..92da798 --- /dev/null +++ b/src/arr/terraform/README.md @@ -0,0 +1,11 @@ +# terraform/ — arr + +Not built, and not expected to be. Terraform's only job in this repo is +creating Proxmox LXCs; Shelfarr runs as a Compose stack on nas1, where the +host already exists. + +This folder is a placeholder so the app keeps the same +`common/` + `ansible/` + `terraform/` shape as every other one. If Shelfarr +ever moves to Proxmox it would need an LXC defined here and wired into +`build/config/terraform/main.tf` — see `../ansible/proxmox/README.md` first +for why that move is less straightforward than it looks. diff --git a/src/authentik/ansible/kubernetes/helmchart.yaml.j2 b/src/authentik/ansible/kubernetes/helmchart.yaml.j2 new file mode 100644 index 0000000..ec35b7b --- /dev/null +++ b/src/authentik/ansible/kubernetes/helmchart.yaml.j2 @@ -0,0 +1,130 @@ +{# + Managed by Ansible (roles/k3s_app) — do not edit on the node. + + A HelmChart CR for k3s's bundled helm-controller, the same mechanism every + cluster service here already uses (roles/k3s_metallb, k3s_monitoring, + k3s_postgres, k3s_cert_manager). The difference is only in who owns the + file: those roles each carry their own template, whereas this one belongs + to the app and roles/k3s_app renders whatever *.yaml.j2 it finds here. + + This is the k3s analogue of src//ansible/unraid/docker-compose.yml. + It is not static the way the compose files are — Helm values have no + ${VAR} interpolation to defer to a rendered .env, so config is substituted + at render time from `app_config` (common/vars.yml + this platform's + vars.yml). What survives from the Unraid convention is the part that + matters: no secret is ever interpolated into this file. Secrets are + referenced by secretKeyRef against the Secret roles/k3s_app renders from + Vault, so this manifest stays 0644 on the node and safe to read. + + No Redis anywhere below, deliberately: recent Authentik no longer requires + it, and the chart carries no Redis dependency (its only conditional + subchart is Bitnami Postgres, which is disabled here in favour of the + cluster's shared CNPG instance). +-#} +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: authentik + namespace: kube-system +spec: + chart: authentik + repo: https://charts.goauthentik.io + # Chart version and app version are the same string upstream, so + # AUTHENTIK_VERSION pins both. See common/vars.yml — this must match the + # version the database was dumped from at cutover time. + version: "{{ app_config.AUTHENTIK_VERSION }}" + targetNamespace: {{ app_config.K8S_NAMESPACE }} + createNamespace: true + valuesContent: |- + global: + # Explicit `env` entries rather than an `envFrom: [secretRef]`, for a + # precedence reason rather than a stylistic one: the chart builds its + # own env for the values under `authentik:` below, and container `env` + # deterministically wins over anything arriving via `envFrom`, whereas + # two envFrom sources resolve by list order the chart controls. This + # way the non-secret half stays readable as ordinary Helm values and + # the secret half is unambiguously authoritative. + env: + # Signs session cookies. Carried over from the Unraid deployment, not + # generated — a new value logs every user out the moment DNS flips. + - name: AUTHENTIK_SECRET_KEY + valueFrom: + secretKeyRef: + name: authentik-secrets + key: AUTHENTIK_SECRET_KEY + - name: AUTHENTIK_POSTGRESQL__PASSWORD + valueFrom: + secretKeyRef: + name: authentik-secrets + key: AUTHENTIK_POSTGRESQL__PASSWORD + + authentik: + log_level: info + # Off: this is a homelab, and the default ships crash reports to + # Sentry. + error_reporting: + enabled: false + postgresql: + host: {{ app_config.DB_HOST }} + port: {{ app_config.DB_PORT }} + name: {{ app_config.DB_NAME }} + user: {{ app_config.DB_USER }} + # password comes from global.env above, never from here. + + # The chart's bundled Bitnami Postgres. Off — this cluster has a shared + # CNPG instance (roles/k3s_postgres) and an app getting its own database + # server would defeat the point of having one. + postgresql: + enabled: false + + server: + replicas: 1 + resources: + requests: + cpu: 100m + memory: {{ app_config.SERVER_MEMORY_REQUEST }} + limits: + memory: {{ app_config.SERVER_MEMORY_LIMIT }} + + ingress: + enabled: true + ingressClassName: {{ app_config.INGRESS_CLASS }} + annotations: + # cert-manager watches Ingresses for this annotation and creates + # the Certificate (and hence the Secret named under tls: below) on + # its own — no Certificate resource of ours to keep in sync. + cert-manager.io/cluster-issuer: {{ app_config.CERT_ISSUER }} + hosts: + - {{ app_config.AUTHENTIK_HOST }} + tls: + - secretName: authentik-tls + hosts: + - {{ app_config.AUTHENTIK_HOST }} + + # /media — uploaded icons and flow backgrounds, the only Authentik + # state that isn't in Postgres. The same PVC is mounted by the worker + # below; see vars.yml for why one ReadWriteOnce volume across two pods + # is fine here and what it costs. + volumes: + - name: media + persistentVolumeClaim: + claimName: authentik-media + volumeMounts: + - name: media + mountPath: /media + + worker: + replicas: 1 + resources: + requests: + cpu: 100m + memory: {{ app_config.WORKER_MEMORY_REQUEST }} + limits: + memory: {{ app_config.WORKER_MEMORY_LIMIT }} + volumes: + - name: media + persistentVolumeClaim: + claimName: authentik-media + volumeMounts: + - name: media + mountPath: /media diff --git a/src/authentik/ansible/kubernetes/media-pvc.yaml.j2 b/src/authentik/ansible/kubernetes/media-pvc.yaml.j2 new file mode 100644 index 0000000..ff319ef --- /dev/null +++ b/src/authentik/ansible/kubernetes/media-pvc.yaml.j2 @@ -0,0 +1,33 @@ +{# + Managed by Ansible (roles/k3s_app) — do not edit on the node. + + Authentik's /media volume, claimed here rather than left to the chart: the + chart has no key that creates a PVC, only `volumes:`/`volumeMounts:` that + reference one, so something has to declare it. + + Owning it separately turns out to be the safer arrangement anyway. A PVC + created by the chart would carry Helm's ownership metadata and be a + candidate for deletion whenever the HelmChart CR is uninstalled or the + chart's templates change shape across an upgrade. This one is a plain + manifest whose lifecycle is its own, so `state: absent` on the app — which + deletes the manifests and lets k3s garbage-collect them — is the only thing + that touches it, and roles/k3s_app/tasks/remove.yml documents PVCs as + deliberately surviving a default teardown. + + WaitForFirstConsumer on the local-path StorageClass means this stays + Pending until the first Authentik pod is scheduled; that is the provisioner + working as intended, not a failure, and it's what pins the volume — and + therefore both pods — to a single worker Pi. +-#} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: authentik-media + namespace: {{ app_config.K8S_NAMESPACE }} +spec: + accessModes: + - ReadWriteOnce + storageClassName: {{ app_config.MEDIA_STORAGE_CLASS }} + resources: + requests: + storage: {{ app_config.MEDIA_STORAGE_SIZE }} diff --git a/src/authentik/ansible/kubernetes/vars.yml b/src/authentik/ansible/kubernetes/vars.yml new file mode 100644 index 0000000..7c61b2d --- /dev/null +++ b/src/authentik/ansible/kubernetes/vars.yml @@ -0,0 +1,63 @@ +--- +# Kubernetes-specific overrides, merged over ../../common/vars.yml (these +# win). Consumed by roles/k3s_app — the same "portable config + platform +# overrides" split compose_stack and lxc_app apply, on the third platform. +env_defaults: + K8S_NAMESPACE: authentik + + # --- Database --------------------------------------------------------- + # + # CNPG's read-write Service, by in-cluster DNS. This is the *primary*, and + # it follows a failover — CNPG re-points the selector when it promotes the + # replica, so the name stays correct without Authentik being restarted. + # + # Deliberately not the LoadBalancer address (192.168.50.81): that exists so + # the Ansible controller can provision databases from off-cluster, and + # sending in-cluster traffic out to the LAN and back would put MetalLB's + # L2 speaker in the path of every query for no benefit. See + # roles/k3s_app/tasks/deploy.yml, which uses the LB address for exactly the + # one thing that needs it. + DB_HOST: shared-postgres-rw.shared-postgres.svc.cluster.local + DB_PORT: "5432" + + # --- Ingress ---------------------------------------------------------- + # + # k3s's bundled Traefik, already running on 192.168.50.80 (MetalLB's first + # pool address). Nothing here pins that IP — the Ingress attaches to the + # class and Traefik's own Service owns the address. + INGRESS_CLASS: traefik + + # The ClusterIssuer roles/k3s_cert_manager creates. Must match + # k3s_cert_manager_issuer_name in inventory/group_vars/k3s_cluster.yml. + CERT_ISSUER: letsencrypt + + # --- Storage ---------------------------------------------------------- + # + # /media holds uploaded application icons and flow backgrounds — the things + # that make the login page look like yours rather than stock. Small, but + # the one piece of Authentik's state that isn't in Postgres, so it has to + # be copied across at cutover (docs/authentik-migration.md) or the migration + # is visibly incomplete. + # + # ReadWriteOnce on k3s's local-path provisioner, and both the server and + # worker pods mount it. That works without any affinity rules of our own: + # local-path PVs carry node affinity, so once the volume binds to whichever + # worker the first pod lands on, the scheduler is obliged to place the + # second pod on that same node — and RWO permits multiple pods per node. + # The cost is that both pods are pinned to one Pi and neither can be + # rescheduled while it's down. Acceptable for a homelab SSO that is already + # a single logical instance; the fix, if it ever matters, is + # ReadWriteMany-capable storage, not a second PVC. + MEDIA_STORAGE_SIZE: 2Gi + MEDIA_STORAGE_CLASS: local-path + + # --- Sizing ----------------------------------------------------------- + # + # Pi 4, 3.8Gi usable, already running CNPG + Prometheus + MetalLB. The + # worker gets more headroom than the server because it's the half that runs + # migrations on startup and processes outposts/policies in the background; + # it is the one that gets OOM-killed if these are set naively equal. + SERVER_MEMORY_REQUEST: 384Mi + SERVER_MEMORY_LIMIT: 768Mi + WORKER_MEMORY_REQUEST: 512Mi + WORKER_MEMORY_LIMIT: 1Gi diff --git a/src/authentik/ansible/proxmox/README.md b/src/authentik/ansible/proxmox/README.md new file mode 100644 index 0000000..e3e3773 --- /dev/null +++ b/src/authentik/ansible/proxmox/README.md @@ -0,0 +1,15 @@ +# Authentik on Proxmox — placeholder + +Not deployed on Proxmox. Authentik runs on the k3s cluster +(`../kubernetes/`), having moved off a hand-made Unraid container — see +`docs/authentik-migration.md`. + +To add it here later: `vars.yml` of Proxmox-specific overrides, `install.yml` +of install steps, and templates for the app's config and systemd unit, plus +`src/authentik/terraform/` defining its LXC. See `src/forgejo/ansible/proxmox/` +for a worked example, and CLAUDE.md → "Adding a new app" — in particular that +an app must only ever be live on one platform at a time. + +This folder is kept empty-but-present on purpose: every app carries the same +`common/` + `ansible/` + `terraform/` shape whether or not each platform is +in use, so adding one later doesn't mean restructuring. diff --git a/src/authentik/ansible/unraid/README.md b/src/authentik/ansible/unraid/README.md new file mode 100644 index 0000000..921d8ce --- /dev/null +++ b/src/authentik/ansible/unraid/README.md @@ -0,0 +1,23 @@ +# Authentik on Unraid — deliberately not managed here + +Authentik **does** currently run on Unraid, but it was created by hand +through the Unraid UI, not by this repo. There is no `docker-compose.yml` +here and there should not be one: adding it would put Authentik in `nas01`'s +`stacks:` list, and CLAUDE.md → "Adding a new app" is explicit that an app +must never be declared on two platforms at once — they would share a Vault +path and a database, and two live deployments would corrupt each other's +state. + +The k3s deployment under `../kubernetes/` is where Authentik is going. See +`docs/authentik-migration.md` for the cutover: dump the database off the +Unraid Postgres, restore onto the cluster's CNPG instance, bring Authentik up +on k3s on the *same* version, verify, then move DNS. + +Once DNS has moved and the k3s instance has been trusted for long enough, +stop and delete the hand-made Unraid container. That teardown is a UI action, +not a `state: absent` run — this repo never deployed it, so it has nothing to +tear down. + +This folder stays as a placeholder rather than being deleted, per CLAUDE.md: +the shape is the point, and Authentik moving back to Unraid one day should +mean writing a compose file here, not restructuring the app. diff --git a/src/authentik/common/vars.yml b/src/authentik/common/vars.yml new file mode 100644 index 0000000..6437ce2 --- /dev/null +++ b/src/authentik/common/vars.yml @@ -0,0 +1,38 @@ +--- +# Portable config — values that hold regardless of where Authentik runs. +# Platform-specific values (namespace, storage class, ingress class, how the +# database is reached) live in ../ansible//vars.yml and are merged +# over these at deploy time. +env_defaults: + # >>> CONFIRM BEFORE THE CUTOVER <<< + # + # This must match the version currently running on Unraid at the moment + # the database is dumped. Authentik runs its Django migrations on startup + # against whatever schema it finds, and those migrations are one-way: a + # newer Authentik pointed at an older dump will silently upgrade the + # schema, and there is no downgrade path if the cutover then has to be + # rolled back. Deploy on the *same* version, confirm it works, flip DNS, + # and only then bump this in a separate commit. + # + # The chart version and the app version are the same string upstream (see + # ../ansible/kubernetes/helmchart.yaml.j2, which uses this for both), so + # this is the only place a version is written down. + AUTHENTIK_VERSION: "2026.5.4" + + AUTHENTIK_HOST: auth.turtlesystems.uk + AUTHENTIK_URL: "https://auth.turtlesystems.uk" + + # Authentik's own HTTP port inside the container. Not the published port — + # on k3s the Ingress terminates and forwards here; there is no host port. + AUTHENTIK_HTTP_PORT: "9000" + + DB_NAME: authentik + DB_USER: authentik + # AUTHENTIK_SECRET_KEY and AUTHENTIK_POSTGRESQL__PASSWORD are secrets — + # sourced from Vault (homelab/authentik), never set here. + # + # AUTHENTIK_SECRET_KEY specifically must be carried over from the existing + # Unraid deployment rather than generated fresh: it signs session cookies, + # so a new value logs every user out at the moment DNS flips, and the + # cutover stops looking like the no-op it's meant to be. See + # docs/authentik-migration.md. diff --git a/src/authentik/terraform/README.md b/src/authentik/terraform/README.md new file mode 100644 index 0000000..23773f6 --- /dev/null +++ b/src/authentik/terraform/README.md @@ -0,0 +1,11 @@ +# Authentik Terraform — placeholder + +No Terraform for Authentik. It runs on the k3s cluster, and the k3s cluster +is 4 Raspberry Pis that already exist — there is no guest for Terraform to +provision, the same reason `homelab-utils` has no `src//terraform/` +entries of its own (CLAUDE.md → "K3s cluster"). + +Terraform would only come into this if Authentik moved to Proxmox, where it +would need a module defining its LXC (template, cores, memory, disk, IP), +called from `build/config/terraform/main.tf`. See `src/forgejo/terraform/` +for the shape. diff --git a/src/forgejo/ansible/proxmox/README.md b/src/forgejo/ansible/proxmox/README.md new file mode 100644 index 0000000..491fd72 --- /dev/null +++ b/src/forgejo/ansible/proxmox/README.md @@ -0,0 +1,94 @@ +# ansible/proxmox/ — forgejo + +Native install into the Terraform-provisioned LXC (`../../terraform/`). Run by +the shared `lxc_app` role, which merges `../../common/vars.yml` with `vars.yml` +here, fetches `homelab/forgejo` from Vault, provisions the database, then +includes `install.yml`. + +| File | What it is | +|---|---| +| `vars.yml` | Proxmox-specific overrides — paths, service user, exact release, LAN address of the shared Postgres | +| `install.yml` | Packages, `git` user, directory layout, binary download, config/unit rendering, dump schedule | +| `templates/app.ini.j2` | Forgejo's config — the equivalent of the `FORGEJO__section__KEY` env vars in `../unraid/docker-compose.yml` | +| `templates/forgejo.service.j2` | systemd unit, replacing Compose's `restart: unless-stopped` | +| `templates/forgejo-dbdump.*` | Nightly `pg_dump`, script + service + timer | +| `templates/pgpass.j2` | Credentials for that dump, mode 0600 | + +## How this is backed up + +Nothing is bind-mounted in from the NAS. Every path Forgejo writes to — +repositories, LFS, attachments, indexers, queues, sessions — is left at its +default under `APP_DATA_PATH` on the container's own disk, so a vzdump of the +guest is a complete copy of its filesystem state. Proxmox excludes bind mounts +from vzdump, so an NFS-mounted repository directory would have been the one +thing missing from the backup that was made to protect it. + +The off-box copy is the vzdump job in +`build/config/ansible/roles/pve_backup`, writing to an NFS storage on the NAS. + +**The database is the other half.** Postgres runs elsewhere — the shared +Postgres LXC at 192.168.50.54, a different guest and therefore a different +archive — so a restored container would come back with every repository and no +issues, pull requests, users or permissions. `forgejo-dbdump.timer` runs +`pg_dump` into `FORGEJO_BACKUP_PATH` on the container's own disk, which puts +the dump inside the same vzdump archive as the repositories it belongs to. One +artifact, one restore. + +That only holds if the dump finishes before the backup window, so the two +schedules are a pair: + +| | Set in | Default | +|---|---|---| +| Database dump | `vars.yml` → `FORGEJO_DB_DUMP_ONCALENDAR` | 01:30 | +| vzdump job | `group_vars/proxmox_nodes.yml` → `pve_backup_schedule` | 02:00 | + +Move one and move the other. Retention is independent: `pg_dump` output is +pruned inside the container by `FORGEJO_BACKUP_KEEP_DAYS`, while the archives +themselves are pruned by the storage's `prune-backups` settings — Proxmox +prunes archives, not their contents. + +### Restoring + +1. Restore the container from the archive (`pct restore`, or the UI). +2. Recreate the database and role if they're gone — + `ansible-playbook playbooks/proxmox.yml -e only_stacks=forgejo` does it + idempotently. +3. Load the newest dump from `/var/lib/forgejo/backups`: + ```sh + systemctl stop forgejo + zcat /var/lib/forgejo/backups/forgejo-.sql.gz \ + | psql -h -U forgejo -d forgejo + systemctl start forgejo + ``` + +## Upgrading Forgejo + +`FORGEJO_VERSION` in `../../common/vars.yml` is `10` — a Docker tag, a rolling +pointer at the latest 10.x, which is all the Compose stack needs. A native +install downloads one specific artifact, so `vars.yml` here carries +`FORGEJO_RELEASE` with the exact version. Bump it and redeploy: the new binary +lands alongside the running one, the `/usr/local/bin/forgejo` symlink flips, +and the service restarts. The old binary stays on disk, so rolling back is +editing `FORGEJO_RELEASE` and redeploying. + +## Differences from the Unraid stack + +- **SSH port 22, not 2222.** On Unraid, 2222 was a published Docker port + dodging the host's own sshd. Here the container has its own IP and its own + sshd, which Forgejo uses directly (`START_SSH_SERVER = false`) — it manages + the `git` user's `authorized_keys` rather than running its own server. This + changes the SSH clone URLs Forgejo advertises, so existing remotes need + updating after a migration. +- **`INSTALL_LOCK = true`.** `app.ini` is rendered in full from Vault, so the + web installer is skipped entirely. Nothing is left for Forgejo to generate + on first boot — which is why `LFS_JWT_SECRET` has to exist in Vault here + even though the Compose stack lets Forgejo invent one. +- **`DB_HOST` is a LAN address**, not the `shared-postgres` Docker network + alias — there's no `unraid_shared` network to resolve it through. + +## One platform at a time + +Both platforms use the same Vault path and the same database, so they must not +run at once. This was why `host_vars/forgejo.yml` carried an empty `apps:` +list for so long; nas2 has since been retired, the list is live, and the +bring-up runbook is `docs/forgejo-proxmox.md`. diff --git a/src/forgejo/ansible/proxmox/install.yml b/src/forgejo/ansible/proxmox/install.yml new file mode 100644 index 0000000..27cc872 --- /dev/null +++ b/src/forgejo/ansible/proxmox/install.yml @@ -0,0 +1,190 @@ +--- +# App-specific install steps, included by the `lxc_app` role after it has +# merged ../../common/vars.yml with ./vars.yml into `app_config` and fetched +# `vault_secrets` from the app's Vault path. The role handles enabling and +# starting the service once this file has put the unit in place. +# +# Template paths are absolute rather than relative: this file is included from +# a role, so a bare `src:` would be looked up against that role's templates/ +# directory, not this app's. + +# A native install renders app.ini in full and sets INSTALL_LOCK, so nothing +# is left for Forgejo to generate on first boot. A missing key would otherwise +# surface as an undefined-variable error three tasks later. +- name: Check that every required Vault key is present + ansible.builtin.assert: + that: + - vault_secrets.DB_PASSWORD is defined + - vault_secrets.SECRET_KEY is defined + - vault_secrets.INTERNAL_TOKEN is defined + - vault_secrets.JWT_SECRET is defined + - vault_secrets.LFS_JWT_SECRET is defined + fail_msg: >- + Vault path {{ app.vault_path }} is missing one or more of DB_PASSWORD, + SECRET_KEY, INTERNAL_TOKEN, JWT_SECRET, LFS_JWT_SECRET. See + docs/vault-secrets.md — LFS_JWT_SECRET is needed by the native install + even though the Unraid Compose stack lets Forgejo generate it. + quiet: true + +- name: Install packages Forgejo needs + ansible.builtin.apt: + name: + - git + - gzip + - ca-certificates + # pg_dump for the nightly database dump at the end of this file. The + # distro client and the shared Postgres LXC (17, from the PGDG archive) + # are close enough that either direction works, but the supported one is + # a client at least as new as the server — check this if the dump ever + # starts failing after a Postgres major upgrade over there. + - postgresql-client + state: present + update_cache: true + cache_valid_time: 3600 + +- name: Create the Forgejo service group + ansible.builtin.group: + name: "{{ app_config.FORGEJO_USER }}" + system: true + state: present + +- name: Create the Forgejo service user + ansible.builtin.user: + name: "{{ app_config.FORGEJO_USER }}" + group: "{{ app_config.FORGEJO_USER }}" + system: true + # A real shell, not nologin: this account is also the SSH login clients + # use for git@host:owner/repo.git. + shell: /bin/bash + home: "{{ app_config.FORGEJO_HOME }}" + create_home: true + comment: Forgejo + +- name: Create Forgejo directories + ansible.builtin.file: + path: "{{ item.path }}" + state: directory + owner: "{{ item.owner | default(app_config.FORGEJO_USER) }}" + group: "{{ item.group | default(app_config.FORGEJO_USER) }}" + mode: "{{ item.mode }}" + loop: + - { path: "{{ app_config.FORGEJO_HOME }}", mode: "0750" } + - { path: "{{ app_config.FORGEJO_DATA_PATH }}", mode: "0750" } + - { path: "{{ app_config.FORGEJO_LOG_PATH }}", mode: "0750" } + # Forgejo writes the git user's authorized_keys here itself. + - { path: "{{ app_config.FORGEJO_HOME }}/.ssh", mode: "0700" } + # Nightly database dumps. On the container's own disk on purpose — that is + # what gets them into the vzdump archive. + - { path: "{{ app_config.FORGEJO_BACKUP_PATH }}", mode: "0700" } + # Config is root-owned and group-readable by Forgejo: it holds the + # database password and the three signing secrets, and Forgejo has no + # business rewriting it — everything in it is rendered from Vault below. + - { path: "{{ app_config.FORGEJO_CONFIG_DIR }}", owner: root, mode: "0750" } + - { path: "{{ app_config.FORGEJO_INSTALL_DIR }}", owner: root, group: root, mode: "0755" } + loop_control: + label: "{{ item.path }}" + +- name: Set Forgejo release facts + ansible.builtin.set_fact: + forgejo_artifact: >- + forgejo-{{ app_config.FORGEJO_RELEASE }}-linux-{{ app_config.FORGEJO_ARCH }} + forgejo_release_url: >- + https://codeberg.org/forgejo/forgejo/releases/download/v{{ app_config.FORGEJO_RELEASE }} + +- name: Download the Forgejo binary + ansible.builtin.get_url: + url: "{{ forgejo_release_url }}/{{ forgejo_artifact }}" + dest: "{{ app_config.FORGEJO_INSTALL_DIR }}/{{ forgejo_artifact }}" + # Checksum from the release's own .sha256 file. Ansible fetches it and + # matches the line for this artifact — which is why `dest` keeps the + # upstream filename rather than being renamed to plain `forgejo`. + checksum: "sha256:{{ forgejo_release_url }}/{{ forgejo_artifact }}.sha256" + owner: root + group: root + mode: "0755" + register: forgejo_binary + +# Versioned file plus a symlink, rather than overwriting one path: an upgrade +# lands the new binary alongside the running one and flips the link, so a +# failed download can't leave a half-written executable in place. +- name: Link the current Forgejo binary onto PATH + ansible.builtin.file: + src: "{{ app_config.FORGEJO_INSTALL_DIR }}/{{ forgejo_artifact }}" + dest: /usr/local/bin/forgejo + state: link + register: forgejo_link + +- name: Render Forgejo configuration + ansible.builtin.template: + src: "{{ app_local_dir }}/ansible/proxmox/templates/app.ini.j2" + dest: "{{ app_config.FORGEJO_CONFIG_PATH }}" + owner: root + group: "{{ app_config.FORGEJO_USER }}" + mode: "0640" + register: forgejo_config + no_log: true + +- name: Install the Forgejo systemd unit + ansible.builtin.template: + src: "{{ app_local_dir }}/ansible/proxmox/templates/forgejo.service.j2" + dest: /etc/systemd/system/forgejo.service + owner: root + group: root + mode: "0644" + register: forgejo_unit + +# `lxc_app` starts the service after this file, but starting an already-running +# service is a no-op — it would not pick up a changed app.ini or a new binary. +- name: Restart Forgejo if its binary or configuration changed + ansible.builtin.systemd_service: + name: forgejo + state: restarted + daemon_reload: true + when: forgejo_binary.changed or forgejo_link.changed + or forgejo_config.changed or forgejo_unit.changed + +# --- Database dump ------------------------------------------------------- +# +# The vzdump job on the Proxmox node captures this container's filesystem, but +# Forgejo's database is not in it — Postgres runs elsewhere, in the shared +# Postgres LXC at 192.168.50.54, which is a *different guest* and so a +# different archive. Restoring the container alone would give back every repository with +# no issues, pull requests, users or permissions. So dump the database into the +# container's own filesystem on a schedule that finishes before the vzdump +# window, and the one archive holds both halves. + +- name: Render the Postgres password file for the dump job + ansible.builtin.template: + src: "{{ app_local_dir }}/ansible/proxmox/templates/pgpass.j2" + dest: "{{ app_config.FORGEJO_HOME }}/.pgpass" + owner: "{{ app_config.FORGEJO_USER }}" + group: "{{ app_config.FORGEJO_USER }}" + mode: "0600" + no_log: true + +- name: Install the database dump script + ansible.builtin.template: + src: "{{ app_local_dir }}/ansible/proxmox/templates/forgejo-dbdump.sh.j2" + dest: /usr/local/bin/forgejo-dbdump + owner: root + group: root + mode: "0755" + +- name: Install the database dump systemd units + ansible.builtin.template: + src: "{{ app_local_dir }}/ansible/proxmox/templates/{{ item }}.j2" + dest: "/etc/systemd/system/{{ item }}" + owner: root + group: root + mode: "0644" + loop: + - forgejo-dbdump.service + - forgejo-dbdump.timer + register: forgejo_dbdump_units + +- name: Enable the nightly database dump timer + ansible.builtin.systemd_service: + name: forgejo-dbdump.timer + enabled: true + state: started + daemon_reload: "{{ forgejo_dbdump_units.changed }}" diff --git a/src/forgejo/ansible/proxmox/templates/app.ini.j2 b/src/forgejo/ansible/proxmox/templates/app.ini.j2 new file mode 100644 index 0000000..b9e98d7 --- /dev/null +++ b/src/forgejo/ansible/proxmox/templates/app.ini.j2 @@ -0,0 +1,116 @@ +{# + Rendered by Ansible from src/forgejo/common/vars.yml merged with + ../vars.yml (as `app_config`), plus Vault path homelab/forgejo (as + `vault_secrets`). The native-install equivalent of the + FORGEJO__section__KEY environment variables in ../../unraid/docker-compose.yml. + + Do not edit this file on the host — the next deploy overwrites it. It is + root-owned and group-readable by Forgejo precisely so Forgejo cannot rewrite + it behind Ansible's back. + + Note what is NOT configured here: every path Forgejo stores state under is + left at its default beneath APP_DATA_PATH — indexers, queues, sessions, + avatars, packages. That is deliberate. All of it sitting on the container's + own disk is what makes a vzdump of this guest a complete copy of Forgejo's + filesystem state. +#} +APP_NAME = Forgejo +RUN_USER = {{ app_config.FORGEJO_USER }} +RUN_MODE = prod +WORK_PATH = {{ app_config.FORGEJO_WORK_PATH }} + +[server] +PROTOCOL = http +DOMAIN = {{ app_config.FORGEJO_DOMAIN }} +ROOT_URL = {{ app_config.FORGEJO_ROOT_URL }} +HTTP_ADDR = 0.0.0.0 +HTTP_PORT = {{ app_config.FORGEJO_HTTP_PORT }} +APP_DATA_PATH = {{ app_config.FORGEJO_DATA_PATH }} +{# The container's own sshd serves git over SSH, so Forgejo only advertises + the address and manages the git user's authorized_keys. On Unraid this was + Forgejo's built-in server behind a published Docker port. #} +SSH_DOMAIN = {{ app_config.FORGEJO_DOMAIN }} +SSH_PORT = {{ app_config.FORGEJO_SSH_PORT }} +START_SSH_SERVER = false +DISABLE_SSH = false +LFS_START_SERVER = true +LFS_JWT_SECRET = {{ vault_secrets.LFS_JWT_SECRET }} + +[database] +DB_TYPE = postgres +HOST = {{ app_config.DB_HOST }}:{{ app_config.DB_PORT }} +NAME = {{ app_config.DB_NAME }} +USER = {{ app_config.DB_USER }} +PASSWD = {{ vault_secrets.DB_PASSWORD }} +{# The shared Postgres publishes a plain port on the LAN and has no TLS + configured — see src/shared/postgres/ansible/unraid/docker-compose.yml. + Change both together if that ever gains a certificate. #} +SSL_MODE = disable + +[repository] +ROOT = {{ app_config.FORGEJO_DATA_PATH }}/forgejo-repositories + +[security] +{# Fully configured from Vault, so skip the web installer entirely — without + this, a fresh container serves the setup wizard on first boot and would + write its own app.ini over this one. #} +INSTALL_LOCK = true +SECRET_KEY = {{ vault_secrets.SECRET_KEY }} +INTERNAL_TOKEN = {{ vault_secrets.INTERNAL_TOKEN }} + +[oauth2] +JWT_SECRET = {{ vault_secrets.JWT_SECRET }} + +[oauth2_client] +{# Policy for signing in through an external OIDC provider (Authentik). The + provider itself is NOT configured here and cannot be: Forgejo keeps + authentication sources in its database, added once with + `forgejo admin auth add-oauth` — see step 8 of docs/forgejo-proxmox.md. + This section only decides what happens to a user who has just + authenticated there. #} + +{# Create a Forgejo account on first successful OIDC login. Without this, + Authentik authenticates the user and Forgejo then has nobody to log in as. + Gated by the registration settings below, not independent of them. #} +ENABLE_AUTO_REGISTRATION = true + +{# Attach an OIDC identity to an existing local account when the email + matches, rather than creating a second one alongside it. This is what lets + the break-glass admin from step 4b become the same account you arrive as + through Authentik, instead of demoting you to a fresh unprivileged user on + first SSO login. `login` would ask the user to confirm by entering their + local password; `auto` links silently, which is only safe because + Authentik is the sole source of verified addresses here. #} +ACCOUNT_LINKING = auto + +{# Take the Forgejo username from the provider's preferred_username claim. + The alternative, `userid`, would name accounts after Authentik's opaque + subject UUID. #} +USERNAME = nickname + +[service] +{# INSTALL_LOCK skips the wizard, which is also where these would have been + chosen. + + These two are a pair and the combination is deliberate. DISABLE_REGISTRATION + cannot stay `true` once SSO is wanted: it blocks OIDC auto-registration as + well as the local signup form, so Authentik logins authenticate correctly + and are then refused an account — which reads like a broken provider rather + than a policy setting. ALLOW_ONLY_EXTERNAL_REGISTRATION restores exactly + the property that was wanted: no self-service signup, accounts only through + a configured provider. + + Safe to have in place before the Authentik source exists. With no external + provider configured, "only external registration" permits nothing, so this + is not a window during which the forge is open — flipping the first value + alone would have been. #} +DISABLE_REGISTRATION = false +ALLOW_ONLY_EXTERNAL_REGISTRATION = true +REQUIRE_SIGNIN_VIEW = false + +[log] +{# journald, via systemd capturing stdout — `journalctl -u forgejo`. ROOT_PATH + still matters: some subsystems write their own files regardless. #} +MODE = console +LEVEL = info +ROOT_PATH = {{ app_config.FORGEJO_LOG_PATH }} diff --git a/src/forgejo/ansible/proxmox/templates/forgejo-dbdump.service.j2 b/src/forgejo/ansible/proxmox/templates/forgejo-dbdump.service.j2 new file mode 100644 index 0000000..b8710ce --- /dev/null +++ b/src/forgejo/ansible/proxmox/templates/forgejo-dbdump.service.j2 @@ -0,0 +1,16 @@ +{# Rendered by Ansible — see ../install.yml. #} +[Unit] +Description=Dump the Forgejo database into this container's filesystem +Documentation=file://{{ app_config.FORGEJO_CONFIG_PATH }} +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User={{ app_config.FORGEJO_USER }} +Group={{ app_config.FORGEJO_USER }} +ExecStart=/usr/local/bin/forgejo-dbdump +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=full +ReadWritePaths={{ app_config.FORGEJO_BACKUP_PATH }} diff --git a/src/forgejo/ansible/proxmox/templates/forgejo-dbdump.sh.j2 b/src/forgejo/ansible/proxmox/templates/forgejo-dbdump.sh.j2 new file mode 100644 index 0000000..49a8d02 --- /dev/null +++ b/src/forgejo/ansible/proxmox/templates/forgejo-dbdump.sh.j2 @@ -0,0 +1,50 @@ +#!/bin/bash +{# Rendered by Ansible — see ../install.yml. Do not edit on the host. #} +# +# Dumps the Forgejo database into this container's filesystem, where the +# Proxmox vzdump job will pick it up along with everything else. Postgres runs +# elsewhere (shared, on Unraid), so without this a restored container would +# have every repository and no issues, pull requests, users or permissions. +# +# Run by forgejo-dbdump.timer, not by hand — though running it by hand is +# harmless and is the quickest way to check the credentials work. + +# pipefail matters more than usual here: pg_dump feeds gzip, and without it a +# failed dump still exits 0 through gzip and gets published as a valid-looking +# but truncated archive. +set -euo pipefail + +readonly BACKUP_DIR="{{ app_config.FORGEJO_BACKUP_PATH }}" +readonly KEEP_DAYS="{{ app_config.FORGEJO_BACKUP_KEEP_DAYS }}" + +export PGPASSFILE="{{ app_config.FORGEJO_HOME }}/.pgpass" + +umask 077 + +dest="${BACKUP_DIR}/{{ app_config.DB_NAME }}-$(date -u +%Y%m%dT%H%M%SZ).sql.gz" + +# Write to .part and rename only on success, so a dump interrupted by the +# vzdump window (or a reboot) never leaves a partial file that looks complete +# to the retention sweep below. +pg_dump \ + --host="{{ app_config.DB_HOST }}" \ + --port="{{ app_config.DB_PORT }}" \ + --username="{{ app_config.DB_USER }}" \ + --dbname="{{ app_config.DB_NAME }}" \ + --format=plain \ + --no-owner \ + --no-privileges \ + | gzip -9 > "${dest}.part" + +mv "${dest}.part" "${dest}" + +# Retention is here rather than in the storage's prune settings because these +# live inside the container: Proxmox prunes backup archives, not their +# contents. Only complete dumps are counted, so a stale .part never displaces +# a good one. +find "${BACKUP_DIR}" -maxdepth 1 -type f -name '*.sql.gz' \ + -mtime "+${KEEP_DAYS}" -delete +find "${BACKUP_DIR}" -maxdepth 1 -type f -name '*.sql.gz.part' \ + -mtime +1 -delete + +echo "wrote ${dest}" diff --git a/src/forgejo/ansible/proxmox/templates/forgejo-dbdump.timer.j2 b/src/forgejo/ansible/proxmox/templates/forgejo-dbdump.timer.j2 new file mode 100644 index 0000000..725b66e --- /dev/null +++ b/src/forgejo/ansible/proxmox/templates/forgejo-dbdump.timer.j2 @@ -0,0 +1,16 @@ +{# Rendered by Ansible — see ../install.yml. #} +[Unit] +Description=Nightly Forgejo database dump + +[Timer] +{# Must land before the vzdump window on the Proxmox node + (build/config/ansible/inventory/host_vars/pve.yml). The archive is only a + complete restore point if the dump inside it is from the same night. #} +OnCalendar={{ app_config.FORGEJO_DB_DUMP_ONCALENDAR }} +{# Catch up after a reboot rather than skipping a night silently. #} +Persistent=true +{# No randomised delay: the gap to the backup window is the whole point. #} +AccuracySec=1min + +[Install] +WantedBy=timers.target diff --git a/src/forgejo/ansible/proxmox/templates/forgejo.service.j2 b/src/forgejo/ansible/proxmox/templates/forgejo.service.j2 new file mode 100644 index 0000000..5086ded --- /dev/null +++ b/src/forgejo/ansible/proxmox/templates/forgejo.service.j2 @@ -0,0 +1,33 @@ +{# Rendered by Ansible — see ../install.yml. The native-install replacement for + Compose's `restart: unless-stopped`. #} +[Unit] +Description=Forgejo +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User={{ app_config.FORGEJO_USER }} +Group={{ app_config.FORGEJO_USER }} +WorkingDirectory={{ app_config.FORGEJO_WORK_PATH }} +ExecStart=/usr/local/bin/forgejo web --config {{ app_config.FORGEJO_CONFIG_PATH }} +Restart=always +RestartSec=2s +Environment=USER={{ app_config.FORGEJO_USER }} +Environment=HOME={{ app_config.FORGEJO_HOME }} +Environment=GITEA_WORK_DIR={{ app_config.FORGEJO_WORK_PATH }} + +# HTTP_PORT is 3000, so no CAP_NET_BIND_SERVICE is needed. Forgejo does need +# to write the git user's authorized_keys and its own data, so the filesystem +# is not made read-only wholesale. +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=full +ProtectHome=false +ReadWritePaths={{ app_config.FORGEJO_HOME }} +{# Restarting mid-push should not orphan git subprocesses. #} +KillMode=control-group +TimeoutStopSec=30s + +[Install] +WantedBy=multi-user.target diff --git a/src/forgejo/ansible/proxmox/templates/pgpass.j2 b/src/forgejo/ansible/proxmox/templates/pgpass.j2 new file mode 100644 index 0000000..e4557a8 --- /dev/null +++ b/src/forgejo/ansible/proxmox/templates/pgpass.j2 @@ -0,0 +1,8 @@ +{# Rendered by Ansible — see ../install.yml. Lets the nightly dump job + authenticate without the password appearing in a command line or a systemd + unit. Mode 0600, owned by the Forgejo service user; libpq refuses to read it + otherwise. + + Format is host:port:database:user:password, with literal backslashes and + colons in the password escaped. #} +{{ app_config.DB_HOST }}:{{ app_config.DB_PORT }}:{{ app_config.DB_NAME }}:{{ app_config.DB_USER }}:{{ vault_secrets.DB_PASSWORD | replace('\\', '\\\\') | replace(':', '\\:') }} diff --git a/src/forgejo/ansible/proxmox/vars.yml b/src/forgejo/ansible/proxmox/vars.yml new file mode 100644 index 0000000..4cefc97 --- /dev/null +++ b/src/forgejo/ansible/proxmox/vars.yml @@ -0,0 +1,50 @@ +--- +# Proxmox-specific overrides, merged over ../../common/vars.yml (these win). +# Reachable in install.yml and the templates as `app_config`. +env_defaults: + # ../../common/vars.yml carries FORGEJO_VERSION: "10" — that's a Docker tag, + # a rolling pointer at the latest 10.x. A native install downloads one + # specific release artifact, so it needs the exact version. Bump this to + # upgrade; the next deploy fetches the new binary and restarts the service. + FORGEJO_RELEASE: "10.0.3" + FORGEJO_ARCH: amd64 + + # Service account and filesystem layout — the Proxmox counterpart to the + # Unraid appdata path. `git` is the conventional user name for a forge: it's + # also the SSH login clients use (git@host:owner/repo.git). + FORGEJO_USER: git + FORGEJO_HOME: /var/lib/forgejo + FORGEJO_WORK_PATH: /var/lib/forgejo + FORGEJO_DATA_PATH: /var/lib/forgejo/data + FORGEJO_LOG_PATH: /var/lib/forgejo/log + FORGEJO_CONFIG_DIR: /etc/forgejo + FORGEJO_CONFIG_PATH: /etc/forgejo/app.ini + FORGEJO_INSTALL_DIR: /usr/local/lib/forgejo + + # 22, not the 2222 in common/vars.yml. On Unraid that was a published Docker + # port dodging the host's own sshd; here Forgejo uses the container's sshd + # directly, and the container has a whole IP to itself. Note this changes the + # SSH clone URLs Forgejo advertises — see README.md in this directory. + FORGEJO_SSH_PORT: "22" + + # No `unraid_shared` Docker network here to resolve `shared-postgres` + # through, so the shared Postgres is reached over the LAN, at the address of + # the LXC running it. Keep in step with `ip_address` on the `postgres` + # module in build/config/terraform/main.tf, which is the authoritative copy + # (it is also published as the `postgres_address` output). + # + # Was 192.168.50.2 — the Unraid host nas2, now retired. Note that address + # has since been reassigned to turtle-proxmox-01, so a stale copy of this + # value fails by connecting to the wrong machine rather than by timing out. + DB_HOST: 192.168.50.54 + DB_PORT: "5432" + + # Nightly pg_dump of the Forgejo database, written to the container's own + # disk so that the vzdump archive of this guest carries it off-box along with + # everything else. Must finish before the vzdump window in + # build/config/ansible/inventory/group_vars/proxmox_nodes.yml (02:00, half an + # hour after this) — see the README in this directory for why the ordering + # matters. + FORGEJO_BACKUP_PATH: /var/lib/forgejo/backups + FORGEJO_BACKUP_KEEP_DAYS: "14" + FORGEJO_DB_DUMP_ONCALENDAR: "*-*-* 01:30:00" diff --git a/src/forgejo/ansible/unraid/.env.example b/src/forgejo/ansible/unraid/.env.example new file mode 100644 index 0000000..14bc60d --- /dev/null +++ b/src/forgejo/ansible/unraid/.env.example @@ -0,0 +1,28 @@ +# Reference only — real values are rendered by Ansible from vars.yml (non- +# secret) and Vault path homelab/forgejo (secret). Do not fill this in and +# deploy it directly. + +# Non-secret (see vars.yml) +FORGEJO_VERSION=10 +FORGEJO_DATA_PATH=/mnt/user/appdata/forgejo/data +FORGEJO_HTTP_PORT=3000 +FORGEJO_SSH_PORT=2222 +FORGEJO_DOMAIN=git.example.internal +FORGEJO_ROOT_URL=http://git.example.internal:3000/ +DB_HOST=shared-postgres +DB_NAME=forgejo +DB_USER=forgejo + +# Non-secret, supplied by the compose_stack role rather than either vars.yml — +# the path it copies ansible/unraid/icon.png to on the host, for the +# net.unraid.docker.icon label. Empty when the stack ships no icon.png, which +# just leaves Unraid's question-mark placeholder. Override it in vars.yml to +# point at a hosted PNG instead. +STACK_ICON=/mnt/user/appdata/icons/forgejo.png + +# Secrets — Vault keys: DB_PASSWORD, SECRET_KEY, INTERNAL_TOKEN, JWT_SECRET +# Generate the latter three with: docker run --rm codeberg.org/forgejo/forgejo:10 forgejo generate secret +DB_PASSWORD= +SECRET_KEY= +INTERNAL_TOKEN= +JWT_SECRET= diff --git a/src/forgejo/ansible/unraid/docker-compose.yml b/src/forgejo/ansible/unraid/docker-compose.yml new file mode 100644 index 0000000..04b0118 --- /dev/null +++ b/src/forgejo/ansible/unraid/docker-compose.yml @@ -0,0 +1,48 @@ +services: + forgejo: + image: codeberg.org/forgejo/forgejo:${FORGEJO_VERSION} + container_name: forgejo + restart: unless-stopped + # Unraid's Docker page normally reads icon, WebUI link and console shell + # from the dockerMan template that created the container. A Compose stack + # has no template, so 6.10+ falls back to these labels; without them the + # container renders as a question mark with no WebUI or Console entry. + # + # STACK_ICON comes from the compose_stack role (a path under + # /mnt/user/appdata/icons, or a URL if vars.yml overrides it). `[IP]` is + # substituted by the webgui with the host's address; the port has to be + # the published one, hence FORGEJO_HTTP_PORT rather than the container's + # 3000. Alpine image, so Console gets sh, not bash. + labels: + net.unraid.docker.icon: ${STACK_ICON} + net.unraid.docker.webui: "http://[IP]:${FORGEJO_HTTP_PORT}/" + net.unraid.docker.shell: sh + environment: + USER_UID: "1000" + USER_GID: "1000" + FORGEJO__database__DB_TYPE: postgres + FORGEJO__database__HOST: ${DB_HOST} + FORGEJO__database__NAME: ${DB_NAME} + FORGEJO__database__USER: ${DB_USER} + FORGEJO__database__PASSWD: ${DB_PASSWORD} + FORGEJO__server__DOMAIN: ${FORGEJO_DOMAIN} + FORGEJO__server__ROOT_URL: ${FORGEJO_ROOT_URL} + FORGEJO__server__SSH_DOMAIN: ${FORGEJO_DOMAIN} + FORGEJO__server__SSH_PORT: ${FORGEJO_SSH_PORT} + # Fixed rather than left to be auto-generated, so recreating the + # container from a fresh volume doesn't invalidate sessions/tokens. + FORGEJO__security__SECRET_KEY: ${SECRET_KEY} + FORGEJO__security__INTERNAL_TOKEN: ${INTERNAL_TOKEN} + FORGEJO__oauth2__JWT_SECRET: ${JWT_SECRET} + volumes: + - ${FORGEJO_DATA_PATH}:/data + ports: + - "${FORGEJO_HTTP_PORT}:3000" + - "${FORGEJO_SSH_PORT}:22" + networks: + - shared + +networks: + shared: + name: unraid_shared + external: true diff --git a/src/forgejo/ansible/unraid/vars.yml b/src/forgejo/ansible/unraid/vars.yml new file mode 100644 index 0000000..9b68c40 --- /dev/null +++ b/src/forgejo/ansible/unraid/vars.yml @@ -0,0 +1,21 @@ +--- +# Unraid-specific overrides, merged over ../../common/vars.yml (these win). +env_defaults: + FORGEJO_DATA_PATH: /mnt/user/appdata/forgejo/data + # Docker network alias of the shared Postgres container — reachable because + # both stacks join the `unraid_shared` network. Not the same as + # host_vars/*.yml stack.db.provision_host, which Ansible uses from the + # controller, outside Docker. + DB_HOST: shared-postgres + + # --- Unraid Docker page presentation (net.unraid.docker.* labels) -------- + # + # STACK_ICON is normally left to the compose_stack role, which points it at + # /mnt/user/appdata/icons/.png when this directory contains an + # icon.png. Uncomment to use a hosted icon instead of committing one: + # + # STACK_ICON: https://example.org/forgejo.png + # + # PNG only — Unraid renders its placeholder for SVG and nothing at all for + # WebP, and a URL it can't reach when the Docker page renders fails the same + # way, which is the argument for committing the file. diff --git a/src/forgejo/common/vars.yml b/src/forgejo/common/vars.yml new file mode 100644 index 0000000..17c31b9 --- /dev/null +++ b/src/forgejo/common/vars.yml @@ -0,0 +1,15 @@ +--- +# Portable config — values that hold regardless of whether Forgejo runs as a +# Compose stack on Unraid or natively in a Proxmox LXC. Platform-specific +# values (filesystem paths, how the database is reached) live in +# ../ansible//vars.yml and are merged over these at deploy time. +env_defaults: + FORGEJO_VERSION: "10" + FORGEJO_HTTP_PORT: "3000" + FORGEJO_SSH_PORT: "2222" + FORGEJO_DOMAIN: git.turtlesystems.uk + FORGEJO_ROOT_URL: "https://git.turtlesystems.uk" + DB_NAME: forgejo + DB_USER: forgejo + # DB_PASSWORD, SECRET_KEY, INTERNAL_TOKEN, JWT_SECRET are secrets — sourced + # from Vault (homelab/forgejo), never set here. diff --git a/src/forgejo/terraform/README.md b/src/forgejo/terraform/README.md new file mode 100644 index 0000000..7692490 --- /dev/null +++ b/src/forgejo/terraform/README.md @@ -0,0 +1,52 @@ +# terraform/ — forgejo + +The LXC Forgejo runs in on Proxmox: container template, cores, memory, disk, +static IP. Called from `build/config/terraform/main.tf`, which holds the +provider and the `backend "pg"` state config. + +Terraform's scope stops at the guest — installing Forgejo into it is +`../ansible/proxmox/`. + +## Required inputs + +Everything else has a default (see `variables.tf`): + +| Variable | Example | +|---|---| +| `node_name` | `pve` | +| `template_file_id` | `local:vztmpl/debian-12-standard_12.7-1_amd64.tar.zst` | +| `ip_address` | `192.168.50.52/24` | +| `gateway` | `192.168.50.254` | +| `ssh_public_keys` | the public half of the key in Vault at `homelab/ci/ssh` | + +## No bind mounts, on purpose + +There is no `mount_point` block. All of Forgejo's state — repositories, LFS, +attachments, Bleve indexers, queues — sits on the container's own rootfs, so +one vzdump archive is a complete copy of it. A bind-mounted NFS share would +break that guarantee: vzdump deliberately excludes bind mounts, so the thing +you most wanted backed up would be the one thing missing from the backup. + +Getting a copy off the box is the backup job's job, not the container's: +`build/config/ansible/roles/pve_backup` defines an NFS storage pointing at the +NAS and a vzdump schedule that writes there. The Postgres database lives +off-container and is handled separately — see `../ansible/proxmox/README.md`. + +This is why `disk_size` defaults to 64 GiB rather than something minimal: the +repositories are on it. + +## `prevent_destroy` + +`main.tf` sets `lifecycle { prevent_destroy = true }`. Several innocuous-looking +changes (shrinking `disk_size`, changing `template_file_id`) are replacements +rather than in-place updates, and replacing this container destroys every +repository on it. The flag turns those into a failed plan instead of a +successful one. To genuinely retire the guest, remove the block first — it +can't be overridden from the command line. + +## Sizing + +`cores = 2`, `memory = 2048`, `disk_size = 64`. Memory is sized for Forgejo +plus its indexers; disk for repositories plus headroom for push staging and +the nightly database dumps kept under `FORGEJO_BACKUP_PATH`. Growing the disk +later is safe. diff --git a/src/forgejo/terraform/main.tf b/src/forgejo/terraform/main.tf new file mode 100644 index 0000000..26e7b02 --- /dev/null +++ b/src/forgejo/terraform/main.tf @@ -0,0 +1,116 @@ +# The LXC Forgejo runs in on Proxmox. Terraform's scope stops here: it creates +# the guest and nothing else, because the Proxmox provider has no way to +# install an application into one. Ansible (`lxc_app` + +# ../ansible/proxmox/install.yml) picks up from the point this container +# exists. +# +# Deliberately no `mount_point` block. Everything Forgejo owns — repositories, +# LFS, attachments, indexers, queues — lives on this container's own rootfs, so +# a vzdump of the guest is a complete copy of its filesystem state. Bind mounts +# would break that: vzdump skips them by design. The off-box copy is the +# backup job in build/config/ansible/roles/pve_backup, which writes vzdump +# archives to the NAS over NFS. See ../ansible/proxmox/README.md for why the +# database is handled separately. + +resource "proxmox_virtual_environment_container" "forgejo" { + node_name = var.node_name + vm_id = var.vm_id + tags = var.tags + + # Unprivileged: nothing here needs host privileges. Forgejo binds 3000, not + # a low port, and git operations are ordinary userspace file I/O. + unprivileged = true + start_on_boot = true + + # systemd >= 252 in an unprivileged container needs a cgroup namespace of + # its own to set up delegation, which LXC only grants with nesting on. + # Without it systemd still boots, but units using the sandboxing directives + # (PrivateTmp, ProtectSystem, ProtectHome) fail to start — and the unit + # ../ansible/proxmox/ installs is exactly that shape. This is what the + # provider's "WARN: Systemd NNN detected. You may need to enable nesting." + # is pointing at, and it is the Proxmox UI's own default for unprivileged + # containers. + # + # Safe here precisely *because* the container is unprivileged: nesting on a + # privileged container widens a real escape surface, on an unprivileged one + # it grants a namespace the kernel already refuses to let root out of. + features { + nesting = true + } + + # Setting this on an *already existing* container takes root@pam. PVE + # hard-codes that for feature flags — it is not an ACL and no role grants + # it — and while its own error message exempts nesting, the provider sends + # the other flags alongside, so the exemption doesn't apply. A 403 reading + # "changing feature flags (except nesting) is only allowed for root@pam" + # means this container predates the block; fix it on the node with + # + # pct set --features nesting=1 && pct reboot + # + # after which the plan reads clean, because the provider only sends + # `features` when it differs. Whether a *fresh* create hits the same wall is + # untested — CT 161 was already up when this block was added. + + initialization { + hostname = var.hostname + + ip_config { + ipv4 { + address = var.ip_address + gateway = var.gateway + } + } + + # Omitted entirely when unset, so the container inherits the node's + # resolver rather than being handed an empty server list. + dynamic "dns" { + for_each = var.dns_servers == null ? [] : [1] + + content { + servers = var.dns_servers + } + } + + # Ansible connects as root over SSH, same as the Unraid hosts — see + # "SSH access" in the top-level README.md. Without a key here the guest + # comes up unreachable and playbooks/proxmox.yml can't do anything with it. + user_account { + keys = var.ssh_public_keys + } + } + + cpu { + cores = var.cores + } + + memory { + dedicated = var.memory + swap = var.swap + } + + disk { + datastore_id = var.datastore_id + size = var.disk_size + } + + network_interface { + name = "eth0" + bridge = var.network_bridge + vlan_id = var.network_vlan_id + firewall = var.network_firewall + enabled = true + } + + operating_system { + template_file_id = var.template_file_id + type = var.operating_system_type + } + + # `disk.size` can be grown in place but not shrunk, and the provider replaces + # the container rather than failing if you try. Replacing this guest means + # losing every repository on it, so refuse the plan instead and let a resize + # be a deliberate act. + lifecycle { + prevent_destroy = true + } +} diff --git a/src/forgejo/terraform/outputs.tf b/src/forgejo/terraform/outputs.tf new file mode 100644 index 0000000..a2a89e4 --- /dev/null +++ b/src/forgejo/terraform/outputs.tf @@ -0,0 +1,25 @@ +# Ansible finds the guest through the Proxmox API (inventory/proxmox.yml), not +# by having these wired into inventory — see the note at the top of +# build/config/terraform/main.tf. They exist so the backup job and anything +# else in the root module can refer to the guest without restating its +# identity. + +output "hostname" { + description = "Container hostname — the name this guest appears under in Ansible inventory." + value = proxmox_virtual_environment_container.forgejo.initialization[0].hostname +} + +output "vm_id" { + description = "Proxmox VMID. Pass to the vzdump job if it is pinned to an explicit VMID list." + value = proxmox_virtual_environment_container.forgejo.vm_id +} + +output "ip_address" { + description = "IPv4 address without the prefix length." + value = split("/", var.ip_address)[0] +} + +output "node_name" { + description = "Proxmox node the container was created on." + value = proxmox_virtual_environment_container.forgejo.node_name +} diff --git a/src/forgejo/terraform/providers.tf b/src/forgejo/terraform/providers.tf new file mode 100644 index 0000000..6714e16 --- /dev/null +++ b/src/forgejo/terraform/providers.tf @@ -0,0 +1,13 @@ +terraform { + required_version = ">= 1.6" + + required_providers { + proxmox = { + source = "bpg/proxmox" + # Deliberately unpinned until the first real `terraform init` — pin to + # whatever it resolves, and commit .terraform.lock.hcl (which is not + # gitignored) so everyone and CI get the same provider. + # version = "~> 0.0" + } + } +} \ No newline at end of file diff --git a/src/forgejo/terraform/variables.tf b/src/forgejo/terraform/variables.tf new file mode 100644 index 0000000..be07067 --- /dev/null +++ b/src/forgejo/terraform/variables.tf @@ -0,0 +1,149 @@ +# Nothing environment-specific is hardcoded in the module — every value the +# caller could reasonably differ on is a variable, set from +# build/config/terraform/main.tf. + +variable "node_name" { + description = "Proxmox node to create the container on, e.g. `pve`." + type = string +} + +variable "vm_id" { + description = <<-EOT + Fixed VMID rather than letting Proxmox pick the next free one. The backup + job in build/config/ansible/roles/pve_backup can be pinned to an explicit + VMID list, and a guest whose ID moves between rebuilds silently drops out + of it. + EOT + type = number + default = 160 +} + +variable "hostname" { + description = <<-EOT + Container hostname. This is also the inventory hostname Ansible sees, via + the community.proxmox dynamic inventory — so it has to match the + build/config/ansible/inventory/host_vars/.yml that declares the + `apps:` list. + EOT + type = string + default = "forgejo" +} + +variable "cores" { + description = "CPU cores. Forgejo is not CPU-bound at homelab scale; 2 is comfortable." + type = number + default = 2 +} + +variable "memory" { + description = "RAM in MiB. 2048 covers Forgejo plus its Bleve indexers." + type = number + default = 2048 +} + +variable "swap" { + description = "Swap in MiB." + type = number + default = 512 +} + +variable "disk_size" { + description = <<-EOT + Root disk in GiB. This holds everything — the OS, the Forgejo binary, and + all repository/LFS/attachment data — because nothing is offloaded to an + NFS mount; that is what makes a vzdump of this guest a complete filesystem + backup. Size it for the repos you expect, plus headroom for push staging + and the local database dumps the app keeps under + `FORGEJO_BACKUP_PATH`. Growing it later is safe; shrinking is not (see the + `lifecycle` block in main.tf). + EOT + type = number + default = 64 +} + +variable "datastore_id" { + description = "Proxmox storage the root disk is created on, e.g. `local-lvm`." + type = string + default = "local-lvm" +} + +variable "template_file_id" { + description = <<-EOT + Container template volume ID, e.g. + `local:vztmpl/debian-12-standard_12.7-1_amd64.tar.zst`. Must already be + downloaded on the node — see "Prerequisites for Proxmox" in the top-level + README.md. List what you have with `pveam list local`. + EOT + type = string +} + +variable "operating_system_type" { + description = "Template OS family, as Proxmox labels it. Match the template above." + type = string + default = "debian" +} + +variable "ip_address" { + description = <<-EOT + Static IPv4 in CIDR form, e.g. `192.168.50.60/24`. Static rather than DHCP + because this is the address the `FORGEJO_ROOT_URL` domain resolves to and + the one Ansible connects to. + EOT + type = string + + validation { + condition = can(cidrnetmask(var.ip_address)) + error_message = "ip_address must include a prefix length, e.g. 192.168.50.60/24." + } +} + +variable "gateway" { + description = "IPv4 default gateway." + type = string +} + +variable "dns_servers" { + description = "Resolvers for the container. Leave null to inherit the Proxmox node's." + type = list(string) + default = null +} + +variable "network_bridge" { + description = "Proxmox bridge to attach eth0 to." + type = string + default = "vmbr0" +} + +variable "network_vlan_id" { + description = "VLAN tag for eth0, or null for an untagged bridge." + type = number + default = null +} + +variable "network_firewall" { + description = "Enable the Proxmox firewall on this interface." + type = bool + default = false +} + +variable "ssh_public_keys" { + description = <<-EOT + Public keys installed for root in the container. Ansible authenticates + with the matching private half (`homelab/ci/ssh` in Vault); without at + least one key here the guest is unreachable and playbooks/proxmox.yml has + nothing it can do with it. + EOT + type = list(string) +} + +variable "tags" { + description = <<-EOT + Proxmox tags, shown in the UI and read by the Ansible dynamic inventory. + `terraform` is load-bearing: build/config/ansible/inventory/proxmox.yml + filters on it to tell guests this repo creates apart from ones made by + hand on the node. Drop it and this container is simply never deployed + to — silently, with no error anywhere. + EOT + type = list(string) + default = ["forgejo", "terraform"] +} diff --git a/src/hashicorp-vault/config.hcl b/src/hashicorp-vault/config.hcl new file mode 100644 index 0000000..5c0cf7f --- /dev/null +++ b/src/hashicorp-vault/config.hcl @@ -0,0 +1,18 @@ +storage "raft" { + path = "/vault/file" + node_id = "nas-vault-01" +} + +listener "tcp" { + address = "0.0.0.0:8200" + tls_disable = "true" + x_forwarded_for_authorized_addrs = "192.168.50.51/32" + x_forwarded_for_reject_not_present = false +} + +api_addr = "https://safe.turtlesystems.uk" +cluster_addr = "https://vault:8201" + +ui = true + +disable_mlock = true diff --git a/src/hashicorp-vault/policies/unraid-deploy.hcl b/src/hashicorp-vault/policies/unraid-deploy.hcl new file mode 100644 index 0000000..910463d --- /dev/null +++ b/src/hashicorp-vault/policies/unraid-deploy.hcl @@ -0,0 +1,3 @@ +path "kv/data/homelab/*" { + capabilities = ["read"] +} \ No newline at end of file diff --git a/src/hashicorp-vault/policies/vault-readonly.hcl b/src/hashicorp-vault/policies/vault-readonly.hcl new file mode 100644 index 0000000..a5d6adc --- /dev/null +++ b/src/hashicorp-vault/policies/vault-readonly.hcl @@ -0,0 +1,6 @@ +path "secret/data/*" { + capabilities = ["read", "list"] +} +path "secret/metadata/*" { + capabilities = ["read", "list"] +} diff --git a/src/hashicorp-vault/policies/vault-write.hcl b/src/hashicorp-vault/policies/vault-write.hcl new file mode 100644 index 0000000..7423640 --- /dev/null +++ b/src/hashicorp-vault/policies/vault-write.hcl @@ -0,0 +1,3 @@ +path "secret/data/*" { + capabilities = ["create", "update", "delete"] +} diff --git a/src/scripts/configure_k3s_node.zsh b/src/scripts/configure_k3s_node.zsh new file mode 100644 index 0000000..e9f0ddf --- /dev/null +++ b/src/scripts/configure_k3s_node.zsh @@ -0,0 +1,15 @@ +#!/usr/bin/env zsh + +set -euo pipefail + +# Get the hostname that will be accessed +TARGET_HOST=$1 + +# Build up the command to add the ansible user and add to the right +# group to allow passwordless sudo +ssh $TARGET_HOST "sudo useradd -m -s /bin/bash ansible && sudo usermod -aG sudo ansible && echo 'ansible ALL=(ALL) NOPASSWD:ALL' | sudo tee /etc/sudoers.d/ansible" + +# Add the SSH public key for the ansible user +ssh $TARGET_HOST "sudo mkdir -p /home/ansible/.ssh && sudo chmod 700 /home/ansible/.ssh && sudo touch /home/ansible/.ssh/authorized_keys && sudo chmod 600 /home/ansible/.ssh/authorized_keys && sudo chown -R ansible:ansible /home/ansible/.ssh" +ssh $TARGET_HOST "echo '$(cat ~/.ssh/id_ed_ansible.pub)' | sudo tee -a /home/ansible/.ssh/authorized_keys" + diff --git a/src/shared/postgres/ansible/kubernetes/vars.yml b/src/shared/postgres/ansible/kubernetes/vars.yml new file mode 100644 index 0000000..4567576 --- /dev/null +++ b/src/shared/postgres/ansible/kubernetes/vars.yml @@ -0,0 +1,20 @@ +--- +# Kubernetes-specific overrides, merged over ../../common/vars.yml (these +# win) — the third platform for shared/postgres, alongside ansible/unraid/ +# and ansible/proxmox/. Consumed by roles/k3s_postgres, not compose_stack or +# lxc_app: there's no Compose file or systemd unit here, this cluster is +# deployed as a CloudNativePG Cluster CR (see that role's templates/), but +# the same "portable config + platform overrides" split still applies. +env_defaults: + # 1 primary + 1 replica, workers only — see roles/k3s_postgres/tasks/main.yml + # for the nodeAffinity that keeps both off the control-plane Pi. + POSTGRES_INSTANCES: "2" + + # k3s's built-in default StorageClass (local-path-provisioner) — each + # instance's PV is pinned to whichever worker it first schedules on. + # That's fine here: CNPG's resilience comes from Postgres-level streaming + # replication between instances, not from the storage layer, the same way + # losing one instance's local disk doesn't lose the data as long as the + # other instance is still up. + POSTGRES_STORAGE_SIZE: 10Gi + POSTGRES_STORAGE_CLASS: local-path diff --git a/src/shared/postgres/ansible/proxmox/README.md b/src/shared/postgres/ansible/proxmox/README.md new file mode 100644 index 0000000..677c82e --- /dev/null +++ b/src/shared/postgres/ansible/proxmox/README.md @@ -0,0 +1,76 @@ +# ansible/proxmox/ — postgres + +Native install into the Terraform-provisioned LXC (`../../terraform/`, VMID +161, 192.168.50.54). Deployed by the `lxc_app` role from the `apps:` list in +`build/config/ansible/inventory/host_vars/postgres.yml`. + +This is the shared Postgres for everything outside the k3s cluster. It is the +one app with no `db:` block of its own — the per-app databases are created +*against* it, by whichever `compose_stack` / `lxc_app` / `k3s_app` run +declares them, using `homelab/shared/postgres` as `admin_vault_path`. + +## Files + +| File | What it does | +|---|---| +| `vars.yml` | Proxmox-specific overrides merged over `../../common/vars.yml` — version, cluster name, listen address, allowed CIDRs, tuning | +| `install.yml` | PGDG repository, package install, config, and the superuser password from Vault | +| `templates/homelab.conf.j2` | A `conf.d` drop-in, so a package upgrade rewriting `postgresql.conf` can't revert it | +| `templates/pg_hba.conf.j2` | Rendered whole — there is no drop-in mechanism here and first-match-wins ordering has to be owned outright | + +## Postgres 17, not 13 + +The Unraid Compose stack was pinned to 13 because its data directory predated +this repo. This is a fresh cluster loaded by dump/restore, so there is nothing +to stay compatible with, and Postgres comes from the PGDG archive rather than +Debian's own so that the major version is a decision rather than whatever the +distro happens to ship. + +Bumping `POSTGRES_VERSION` on an existing host does **not** upgrade it. The +new major's package installs alongside the old one and `pg_createcluster` +makes a second, empty cluster on the next free port. Moving data across a +major is `pg_dumpall`/`psql` or `pg_upgradecluster` — a run-once operation, so +it belongs in a runbook rather than in this role. + +## Everything is on the rootfs, and here that is a correctness requirement + +`../../terraform/` puts the container on the `AppData` ZFS pool and creates a +replication job to the second node. Proxmox replicates by sending ZFS +snapshots, and it snapshots each of a guest's volumes independently. + +Splitting `PGDATA` and the WAL across two volumes would therefore replicate +them as two snapshots taken at different instants, and the far node could end +up holding a data directory and a WAL that disagree — which is not a state WAL +replay can recover from. One volume means one atomic snapshot, which is +exactly the crash-consistent image Postgres *is* designed to recover from. + +Same conclusion as Forgejo's "no bind mounts", arrived at from a different +direction: there it was about what `vzdump` includes, here it is about what a +snapshot is atomic across. + +## `full_page_writes = off` + +Set in `vars.yml`, and the one setting here that would be a data-loss bug if +copied to another platform. + +`full_page_writes` exists to survive a torn page — half of an 8 KiB write +reaching disk before a crash. ZFS is copy-on-write and never overwrites a live +block in place, so that failure mode does not arise, and the full-page images +are pure write amplification that on this host also becomes replication +traffic. It is safe **only** on copy-on-write storage. On ext4 or xfs the same +setting risks unrecoverable corruption, which is why it lives in the Proxmox +vars file and not in `../../common/vars.yml`. + +## Replication is not a backup + +The second copy protects against losing a node. It does nothing about a +dropped table, which replicates within five minutes. Two things cover that, +and both already exist: + +- `roles/pve_backup` — the node's vzdump schedule, writing to the NAS. +- An app-side dump, the way Forgejo writes a nightly `pg_dump` onto its own + disk so the vzdump archive carries a consistent copy of its database. + +Note also that replication is asynchronous. An unplanned failover loses every +transaction committed since the last successful run — up to +`replication_schedule`, five minutes by default. diff --git a/src/shared/postgres/ansible/proxmox/install.yml b/src/shared/postgres/ansible/proxmox/install.yml new file mode 100644 index 0000000..bfdd357 --- /dev/null +++ b/src/shared/postgres/ansible/proxmox/install.yml @@ -0,0 +1,157 @@ +--- +# App-specific install steps, included by the `lxc_app` role after it has +# merged ../../common/vars.yml with ./vars.yml into `app_config` and fetched +# `vault_secrets` from homelab/shared/postgres. The role enables and starts +# `app.service_name` once this file has finished. +# +# Template paths are absolute rather than relative: this file is included from +# a role, so a bare `src:` would resolve against that role's templates/ +# directory, not this app's. +# +# This is the shared database other things provision into, so it is the one +# app whose install has no `db:` block of its own — the per-app databases are +# created against it later, by whichever `compose_stack` / `lxc_app` / +# `k3s_app` run declares them. + +- name: Check that every required Vault key is present + ansible.builtin.assert: + that: + - vault_secrets.POSTGRES_SUPERUSER is defined + - vault_secrets.POSTGRES_SUPERUSER_PASSWORD is defined + fail_msg: >- + Vault path {{ app.vault_path }} is missing POSTGRES_SUPERUSER or + POSTGRES_SUPERUSER_PASSWORD. See docs/vault-secrets.md — the same path + seeds the Unraid, Proxmox and k3s instances. + quiet: true + +# Derived once here rather than restated in vars.yml, so a POSTGRES_VERSION +# bump can't leave one of them pointing at the previous major's cluster. +- name: Set cluster paths + ansible.builtin.set_fact: + pg_conf_dir: "/etc/postgresql/{{ app_config.POSTGRES_VERSION }}/{{ app_config.POSTGRES_CLUSTER }}" + pg_data_dir: "/var/lib/postgresql/{{ app_config.POSTGRES_VERSION }}/{{ app_config.POSTGRES_CLUSTER }}" + +- name: Install prerequisites for the PGDG repository + ansible.builtin.apt: + name: + - ca-certificates + - curl + - gnupg + # psycopg2, for the community.postgresql modules used below. They run on + # this host over the Unix socket, unlike the per-app provisioning in the + # `lxc_app` role, which is delegated to the controller and connects over + # TCP. + - python3-psycopg2 + state: present + update_cache: true + cache_valid_time: 3600 + +# Debian 12 ships Postgres 15 in its own archive. PGDG carries every supported +# major, which is what makes POSTGRES_VERSION a real choice — see vars.yml. +- name: Install the PGDG repository signing key + ansible.builtin.get_url: + url: https://www.postgresql.org/media/keys/ACCC4CF8.asc + dest: /usr/share/keyrings/postgresql.asc + mode: "0644" + +- name: Add the PGDG apt repository + ansible.builtin.apt_repository: + repo: >- + deb [signed-by=/usr/share/keyrings/postgresql.asc] + https://apt.postgresql.org/pub/repos/apt + {{ ansible_facts['distribution_release'] }}-pgdg main + filename: pgdg + state: present + update_cache: true + +- name: Install PostgreSQL + ansible.builtin.apt: + name: + - "postgresql-{{ app_config.POSTGRES_VERSION }}" + - "postgresql-client-{{ app_config.POSTGRES_VERSION }}" + state: present + # The package runs pg_createcluster itself, so by the end of this task a + # cluster exists and is running with stock settings on 5432. Everything + # below is reconfiguration of that cluster, not creation of one. + register: pg_install + +# Both templates are rendered as drop-ins and full files respectively, never +# by editing what the packaging wrote: postgresql.conf is Debian's and carries +# an `include_dir = 'conf.d'` at the bottom for exactly this purpose, so a +# drop-in survives a package upgrade rewriting the main file. +- name: Configure the cluster + ansible.builtin.template: + src: "{{ app_local_dir }}/ansible/proxmox/templates/homelab.conf.j2" + dest: "{{ pg_conf_dir }}/conf.d/homelab.conf" + owner: postgres + group: postgres + mode: "0644" + register: pg_conf + +# pg_hba.conf has no include mechanism worth relying on across versions, so +# this one is rendered whole. 0640 rather than 0644: it enumerates which +# networks can reach the database, which is not something to publish to every +# account on the host. +- name: Configure host-based authentication + ansible.builtin.template: + src: "{{ app_local_dir }}/ansible/proxmox/templates/pg_hba.conf.j2" + dest: "{{ pg_conf_dir }}/pg_hba.conf" + owner: postgres + group: postgres + mode: "0640" + register: pg_hba + +# Restarted rather than reloaded, and done here rather than left to the role's +# start task: listen_addresses and shared_buffers both need a restart, and the +# superuser password below has to be set against a cluster already listening +# with the pg_hba above. The role's own "enable and start" then finds it +# running and does nothing. +- name: Restart PostgreSQL to apply configuration + ansible.builtin.systemd_service: + name: "{{ app_config.POSTGRES_SERVICE_NAME }}" + state: restarted + enabled: true + daemon_reload: true + when: pg_install.changed or pg_conf.changed or pg_hba.changed + +# Every task below runs as the `postgres` account, which means Ansible needs a +# temp directory under that account's home. Left to itself it creates one and +# warns, because a directory root made 0700 for another user is a plausible +# permissions trap; created here it is unambiguously postgres's own. +- name: Ensure the postgres account has an Ansible temp directory + ansible.builtin.file: + path: /var/lib/postgresql/.ansible/tmp + state: directory + owner: postgres + group: postgres + mode: "0700" + +- name: Wait for PostgreSQL to accept connections + ansible.builtin.command: >- + pg_isready --quiet + --host /var/run/postgresql + --port {{ app_config.POSTGRES_PORT }} + become: true + become_user: postgres + changed_when: false + register: pg_ready + until: pg_ready.rc == 0 + retries: 12 + delay: 5 + +# Over the Unix socket as the `postgres` OS account, which peer-authenticates +# — this is the one connection that cannot use the password, because setting +# the password is what it is here to do. Vault is authoritative: if the two +# ever disagree, this task makes the database match Vault rather than the +# other way round. +- name: Set the superuser password from Vault + community.postgresql.postgresql_user: + name: "{{ vault_secrets.POSTGRES_SUPERUSER }}" + password: "{{ vault_secrets.POSTGRES_SUPERUSER_PASSWORD }}" + role_attr_flags: SUPERUSER,LOGIN + login_unix_socket: /var/run/postgresql + login_port: "{{ app_config.POSTGRES_PORT }}" + state: present + become: true + become_user: postgres + no_log: true diff --git a/src/shared/postgres/ansible/proxmox/templates/homelab.conf.j2 b/src/shared/postgres/ansible/proxmox/templates/homelab.conf.j2 new file mode 100644 index 0000000..37cd57e --- /dev/null +++ b/src/shared/postgres/ansible/proxmox/templates/homelab.conf.j2 @@ -0,0 +1,50 @@ +{{ ansible_managed | comment }} +# +# Rendered by src/shared/postgres/ansible/proxmox/install.yml into +# {{ pg_conf_dir }}/conf.d/, which Debian's postgresql.conf includes from its +# last line. A drop-in rather than an edit to postgresql.conf itself: the +# packaging owns that file and rewrites it on a package upgrade, which would +# silently revert anything set there. +# +# Settings here win over postgresql.conf because the include comes last. + +# --- Connections -------------------------------------------------------- +listen_addresses = '{{ app_config.POSTGRES_LISTEN_ADDRESSES }}' +port = {{ app_config.POSTGRES_PORT }} +max_connections = {{ app_config.POSTGRES_MAX_CONNECTIONS }} + +# Passwords for new and changed roles are hashed with this. It has to agree +# with the method in pg_hba.conf — a role whose password was stored as md5 +# cannot authenticate against a scram-sha-256 line, which presents as a +# password failure with correct credentials. +password_encryption = '{{ app_config.POSTGRES_AUTH_METHOD }}' + +# --- Memory ------------------------------------------------------------- +shared_buffers = {{ app_config.POSTGRES_SHARED_BUFFERS }} +effective_cache_size = {{ app_config.POSTGRES_EFFECTIVE_CACHE_SIZE }} +maintenance_work_mem = {{ app_config.POSTGRES_MAINTENANCE_WORK_MEM }} +work_mem = {{ app_config.POSTGRES_WORK_MEM }} + +# --- Write-ahead log ---------------------------------------------------- +# +# See vars.yml for why full_page_writes is off here and must not be copied to +# a deployment on ext4 or xfs: it is safe because ZFS is copy-on-write and +# never overwrites a live block in place, so a page cannot be torn by a crash. +full_page_writes = {{ app_config.POSTGRES_FULL_PAGE_WRITES }} + +# --- Logging ------------------------------------------------------------ +# +# To stderr, which the packaging's systemd unit routes into the journal, so +# `journalctl -u postgresql@{{ app_config.POSTGRES_VERSION }}-{{ app_config.POSTGRES_CLUSTER }}` +# is the whole story and there is no second log directory to rotate or to +# remember to include in a backup. +log_destination = 'stderr' +logging_collector = off +log_line_prefix = '%m [%p] %q%u@%d ' + +# Enough to answer "who connected, and did anything fail", without logging +# every statement — this cluster serves app traffic, not an audit trail. +log_connections = on +log_disconnections = on +log_checkpoints = on +log_min_duration_statement = 1000 diff --git a/src/shared/postgres/ansible/proxmox/templates/pg_hba.conf.j2 b/src/shared/postgres/ansible/proxmox/templates/pg_hba.conf.j2 new file mode 100644 index 0000000..da775b3 --- /dev/null +++ b/src/shared/postgres/ansible/proxmox/templates/pg_hba.conf.j2 @@ -0,0 +1,32 @@ +{{ ansible_managed | comment }} +# +# Rendered whole by src/shared/postgres/ansible/proxmox/install.yml, replacing +# the file the packaging wrote. Unlike postgresql.conf there is no drop-in +# directory to use here, and order matters in this file — the first matching +# line wins and no later line can widen it — so it has to be owned outright +# rather than appended to. +# +# TYPE DATABASE USER ADDRESS METHOD + +# The postgres superuser over the Unix socket, peer-authenticated. This is the +# path install.yml uses to set the superuser password from Vault, and the one +# that still works if the password in Vault and the password in the database +# ever drift apart — remove it and a wrong password locks everyone out with no +# way back in short of single-user mode. +local all postgres peer + +# Local connections for everything else still need a password: `psql` from a +# shell on this container is not a stronger claim of identity than a +# connection from the LAN, and treating it as one would mean an app's database +# is readable by anyone who can open a shell here. +local all all {{ app_config.POSTGRES_AUTH_METHOD }} + +{% for cidr in app_config.POSTGRES_ALLOWED_CIDRS %} +# {{ cidr }} +host all all {{ cidr }} {{ app_config.POSTGRES_AUTH_METHOD }} +{% endfor %} + +# No `host all all 0.0.0.0/0` fallback, deliberately. A network not listed +# above is refused at connection time with a clear "no pg_hba.conf entry" +# error, which is a better failure than a database quietly reachable from +# somewhere nobody meant to allow. diff --git a/src/shared/postgres/ansible/proxmox/vars.yml b/src/shared/postgres/ansible/proxmox/vars.yml new file mode 100644 index 0000000..43ed020 --- /dev/null +++ b/src/shared/postgres/ansible/proxmox/vars.yml @@ -0,0 +1,75 @@ +--- +# Proxmox-specific overrides, merged over ../../common/vars.yml (these win). +# Reachable in install.yml and the templates as `app_config`. +env_defaults: + # 17, not the 13 the Unraid Compose stack was pinned to. That pin existed + # because the data directory predated this repo and an in-place major + # upgrade of a running container was not worth the risk; this is a fresh + # cluster loaded by dump/restore, so there is nothing to stay compatible + # with. Postgres comes from the PGDG archive rather than Debian's own + # precisely so this is a choice rather than whatever the distro ships. + # + # Changing this on an existing host does NOT upgrade it — the new major's + # package installs alongside the old one and pg_createcluster makes a + # second, empty cluster. Moving data between majors is pg_dumpall/psql or + # pg_upgradecluster, a run-once operation that belongs in a runbook. + POSTGRES_VERSION: "17" + + # Debian's postgresql-common lays a cluster out as /etc/postgresql// + # and /var/lib/postgresql//. `main` is the one the + # package creates on install; install.yml builds both paths from this and + # POSTGRES_VERSION rather than restating them here, so a version bump can't + # leave a stale path behind. + POSTGRES_CLUSTER: main + + # Listen on every interface. Unlike the Unraid deployment — where the + # container published 5432 and Docker did the forwarding — this LXC has an + # address of its own, and Postgres itself is what has to answer on it. What + # may connect is constrained by pg_hba below, not by the bind address. + POSTGRES_LISTEN_ADDRESSES: "*" + + # Networks allowed to authenticate, rendered into pg_hba.conf in order. + # A list of CIDRs rather than one LAN-wide entry, so that what can reach the + # database reads as a decision: + # - the LAN, covering the Ansible controller that provisions per-app + # databases and every Proxmox guest that connects to one + # - the k3s pod CIDR, for cluster apps pointed at this instance rather + # than the in-cluster CloudNativePG one + POSTGRES_ALLOWED_CIDRS: + - 192.168.50.0/24 + - 10.42.0.0/16 + + # scram-sha-256, not md5: the default since Postgres 14, md5 is deprecated + # upstream, and every client here is a current libpq or psycopg2 — there is + # no legacy driver to accommodate. + POSTGRES_AUTH_METHOD: scram-sha-256 + + # --- Tuning ------------------------------------------------------------ + # + # Sized against the LXC's 2048 MiB in ../../terraform/variables.tf. These do + # not track that value — raise the container's memory and these stay put + # until edited. + POSTGRES_MAX_CONNECTIONS: "100" + POSTGRES_SHARED_BUFFERS: 512MB + POSTGRES_EFFECTIVE_CACHE_SIZE: 1536MB + POSTGRES_MAINTENANCE_WORK_MEM: 128MB + POSTGRES_WORK_MEM: 8MB + + # ZFS-specific, and the reason this is not a copy of a generic tuning guide. + # + # full_page_writes exists to survive a torn page — half of an 8 KiB write + # landing before a crash. ZFS is copy-on-write and never overwrites a live + # block in place, so that failure mode does not arise and the full-page + # images are pure write amplification, which on this host is also + # replication traffic. Safe *only* on copy-on-write storage; the same + # setting on ext4 or xfs is a data-loss bug, which is why it lives in the + # Proxmox vars file and not in common/vars.yml. + POSTGRES_FULL_PAGE_WRITES: "off" + + # --- Service ----------------------------------------------------------- + # + # `lxc_app` enables and starts `app.service_name` once install.yml finishes. + # `postgresql` is the packaging's umbrella unit, which pulls in the + # per-cluster postgresql@-.service — targeting the umbrella + # means this needs no edit on a major bump. + POSTGRES_SERVICE_NAME: postgresql diff --git a/src/shared/postgres/ansible/unraid/.env.example b/src/shared/postgres/ansible/unraid/.env.example new file mode 100644 index 0000000..5be7262 --- /dev/null +++ b/src/shared/postgres/ansible/unraid/.env.example @@ -0,0 +1,18 @@ +# Reference only — real values are rendered by Ansible from vars.yml (non- +# secret) and Vault path homelab/shared/postgres (secret). Do not fill this in +# and deploy it directly. + +# Non-secret (see vars.yml) +POSTGRES_DATA_PATH=/mnt/user/appdata/shared-postgres/data +POSTGRES_PORT=5432 +POSTGRES_SUPERUSER=postgres + +# Non-secret, supplied by the compose_stack role rather than either vars.yml — +# the path it copies ansible/unraid/icon.png to on the host, for the +# net.unraid.docker.icon label. Empty when the stack ships no icon.png, which +# just leaves Unraid's question-mark placeholder. Override it in vars.yml to +# point at a hosted PNG instead. +STACK_ICON=/mnt/user/appdata/icons/postgres.png + +# Secret — Vault key: POSTGRES_SUPERUSER_PASSWORD +POSTGRES_SUPERUSER_PASSWORD= diff --git a/src/shared/postgres/ansible/unraid/docker-compose.yml b/src/shared/postgres/ansible/unraid/docker-compose.yml new file mode 100644 index 0000000..88e16e3 --- /dev/null +++ b/src/shared/postgres/ansible/unraid/docker-compose.yml @@ -0,0 +1,32 @@ +services: + postgres: + image: postgres:13 + container_name: shared-postgres + restart: unless-stopped + # Unraid's Docker page normally reads icon, WebUI link and console shell + # from the dockerMan template that created the container. A Compose stack + # has no template, so 6.10+ falls back to these labels; without them the + # container renders as a question mark on the Docker page. + # + # No `net.unraid.docker.webui` — Postgres has no web interface, and the + # label's only effect would be a WebUI entry that goes nowhere. Console + # still needs a shell, and this image is Debian-based, so bash. + labels: + net.unraid.docker.icon: ${STACK_ICON} + net.unraid.docker.shell: bash + environment: + POSTGRES_USER: ${POSTGRES_SUPERUSER} + POSTGRES_PASSWORD: ${POSTGRES_SUPERUSER_PASSWORD} + PGDATA: /var/lib/postgresql/data/pgdata + volumes: + - ${POSTGRES_DATA_PATH}:/var/lib/postgresql/data + ports: + # Exposed on the host so the Ansible controller can provision + # per-app databases/roles directly (see ansible/roles/compose_stack). + - "${POSTGRES_PORT}:5432" + networks: + - shared + +networks: + shared: + name: unraid_shared diff --git a/src/shared/postgres/ansible/unraid/vars.yml b/src/shared/postgres/ansible/unraid/vars.yml new file mode 100644 index 0000000..e1fd4a6 --- /dev/null +++ b/src/shared/postgres/ansible/unraid/vars.yml @@ -0,0 +1,16 @@ +--- +# Unraid-specific overrides, merged over ../../common/vars.yml (these win). +env_defaults: + POSTGRES_DATA_PATH: /mnt/user/appdata/shared-postgres/data + + # --- Unraid Docker page presentation (net.unraid.docker.icon label) ------ + # + # STACK_ICON is normally left to the compose_stack role, which points it at + # /mnt/user/appdata/icons/.png when this directory contains an + # icon.png. Uncomment to use a hosted icon instead of committing one: + # + # STACK_ICON: https://example.org/postgres.png + # + # PNG only — Unraid renders its placeholder for SVG and nothing at all for + # WebP, and a URL it can't reach when the Docker page renders fails the same + # way, which is the argument for committing the file. diff --git a/src/shared/postgres/common/vars.yml b/src/shared/postgres/common/vars.yml new file mode 100644 index 0000000..c0f6b16 --- /dev/null +++ b/src/shared/postgres/common/vars.yml @@ -0,0 +1,8 @@ +--- +# Portable config — see ../ansible//vars.yml for the values that +# depend on where this runs. +env_defaults: + POSTGRES_PORT: "5432" + POSTGRES_SUPERUSER: postgres + # POSTGRES_SUPERUSER_PASSWORD is a secret — sourced from Vault + # (homelab/shared/postgres), never set here. diff --git a/src/shared/postgres/terraform/README.md b/src/shared/postgres/terraform/README.md new file mode 100644 index 0000000..1a00dac --- /dev/null +++ b/src/shared/postgres/terraform/README.md @@ -0,0 +1,103 @@ +# terraform/ — postgres + +The LXC the shared Postgres runs in on Proxmox, plus the ZFS replication job +that copies it to the second node. Called from +`build/config/terraform/main.tf`, which holds the provider and the +`backend "pg"` state config. + +Terraform's scope stops at the guest and its replication job — installing and +configuring Postgres inside it is `../ansible/proxmox/`. + +## Required inputs + +Everything else has a default (see `variables.tf`): + +| Variable | Example | +|---|---| +| `node_name` | `turtle-proxmox-01` | +| `template_file_id` | `local:vztmpl/debian-12-standard_12.7-1_amd64.tar.zst` | +| `ip_address` | `192.168.50.54/24` | +| `gateway` | `192.168.50.254` | +| `ssh_public_keys` | the public half of the key in Vault at `homelab/ci/ssh` | + +Two defaults are worth setting deliberately rather than inheriting: +`datastore_id` (`AppData`) and `replication_target_node` +(`turtle-proxmox-02`). See below. + +## The state circularity, and how it is avoided + +Terraform keeps its state in Postgres — so provisioning *the Postgres LXC* +with Terraform looks circular: the database has to exist before `terraform +init` works, and Terraform is what creates the container the database runs in. + +It is avoided by putting state somewhere this configuration has no hand in +building: the CloudNativePG cluster on the k3s Pis, at +`k3s_postgres_loadbalancer_ip` (192.168.50.81). That cluster is deployed by +Ansible against hardware Terraform never touches, so nothing here is a +prerequisite for it. + +The cost is a dependency the rest of the repo avoids — `terraform apply` now +needs the Pis up — and it is worth paying because the alternative is a +bootstrap dance (local state, apply, migrate) that has to be got right exactly +once and is unrecoverable if it is not. Full ordering in +`docs/postgres-proxmox.md`. + +## On the ZFS pool, on purpose + +`datastore_id` defaults to `AppData`, a ZFS pool, rather than `local-lvm`. +Proxmox replicates by sending ZFS snapshots, so a guest on LVM-thin has no +replication available to it at all — the option simply is not offered. + +The storage has to exist under this exact name on the target node too. `pvesr` +resolves the volume by storage ID on the far side, so a pool named differently +there fails at replication time rather than at apply time, which is a much +later and quieter place to find out. + +## Replication is a per-guest job, not a pool property + +This is the part that is easy to get wrong from the UI. A ZFS pool is not +"replicated"; each *guest* on it has a replication job. So a container created +on `AppData` with no job is a single point of failure that looks identical to +a replicated one in the storage view. + +Hence `proxmox_replication` in `main.tf` rather than a manual `pvesr +create-local-job` — the job is part of what defines this guest, and leaving it +to a hand-run command means a rebuild silently produces an unreplicated +database. + +`replication_target_node = null` disables it. That is correct on a single-node +cluster, but set it explicitly, because null is also what "someone forgot" +looks like. + +## Everything on one volume + +There is no `mount_point` block splitting `PGDATA` or the WAL onto a second +volume, and unlike Forgejo's version of this decision it is a correctness +requirement rather than a tidiness preference: replication snapshots each +volume independently, so two volumes replicate as two snapshots taken at +different instants. See `../ansible/proxmox/README.md`. + +The practical consequence is that running low on space has exactly one fix: +raise `disk_size`. There is no second disk to add, and no way to move `PGDATA` +off the rootfs without giving up the atomic snapshot. That is a smaller +imposition than it sounds — an increase applies as a `pct resize` of the +rootfs, and on ZFS that is a refquota change rather than a partition +operation, so it takes effect while the cluster is running and needs nothing +done inside the guest afterwards. Size generously up front regardless: +shrinking is a replacement, which `prevent_destroy` refuses. + +## `prevent_destroy` + +`main.tf` sets `lifecycle { prevent_destroy = true }`. Shrinking `disk_size` +or changing `template_file_id` are replacements rather than in-place updates, +and replacing this container destroys every database on it — including any +whose only other copy is a replication target that would be recreated along +with it. To genuinely retire the guest, remove the block first; it cannot be +overridden from the command line. + +## Sizing + +`cores = 2`, `memory = 2048`, `disk_size = 256`. The tuning in +`../ansible/proxmox/vars.yml` is sized against that 2048 MiB and does **not** +track it — raise the container's memory and `shared_buffers` / +`effective_cache_size` stay where they are until edited. diff --git a/src/shared/postgres/terraform/main.tf b/src/shared/postgres/terraform/main.tf new file mode 100644 index 0000000..3f50c71 --- /dev/null +++ b/src/shared/postgres/terraform/main.tf @@ -0,0 +1,147 @@ +# The LXC the shared Postgres runs in on Proxmox. Terraform's scope stops at +# the guest and its replication job — installing and configuring Postgres +# inside it is ../ansible/proxmox/, run by the `lxc_app` role. +# +# Two things make this module different from src/forgejo/terraform, and both +# come from the same decision: the database lives on the `AppData` ZFS pool so +# that Proxmox can replicate it to the second node. +# +# 1. `datastore_id` defaults to a ZFS pool, not `local-lvm`. Only ZFS-backed +# volumes can be replicated — pvesr works by sending ZFS snapshots, so a +# guest on LVM-thin simply has no replication available to it. +# +# 2. Everything is on the rootfs. There is no `mount_point` block splitting +# PGDATA or the WAL onto a second volume, and that is a correctness +# requirement here rather than the tidiness preference it is for Forgejo. +# Replication snapshots each of a guest's volumes independently, so a +# cluster split across two volumes replicates as two snapshots taken at +# different instants — the far node can end up with a data directory and +# a WAL that disagree, which is not a state Postgres knows how to recover +# from. One volume means one atomic snapshot, which is exactly the +# crash-consistent image WAL replay is designed to handle. + +resource "proxmox_virtual_environment_container" "postgres" { + node_name = var.node_name + vm_id = var.vm_id + tags = var.tags + + # Unprivileged: Postgres binds 5432 and runs as its own service account. + # Nothing it does needs host privileges. + unprivileged = true + start_on_boot = true + + # systemd >= 252 in an unprivileged container needs a cgroup namespace of + # its own to set up delegation, which LXC only grants with nesting on. + # Without it systemd still boots, but units using the sandboxing directives + # (PrivateTmp, ProtectSystem, ProtectHome) fail to start — and the unit + # ../ansible/proxmox/ installs is exactly that shape. This is what the + # provider's "WARN: Systemd NNN detected. You may need to enable nesting." + # is pointing at, and it is the Proxmox UI's own default for unprivileged + # containers. + # + # Safe here precisely *because* the container is unprivileged: nesting on a + # privileged container widens a real escape surface, on an unprivileged one + # it grants a namespace the kernel already refuses to let root out of. + features { + nesting = true + } + + # Setting this on an *already existing* container takes root@pam. PVE + # hard-codes that for feature flags — it is not an ACL and no role grants + # it — and while its own error message exempts nesting, the provider sends + # the other flags alongside, so the exemption doesn't apply. A 403 reading + # "changing feature flags (except nesting) is only allowed for root@pam" + # means this container predates the block; fix it on the node with + # + # pct set --features nesting=1 && pct reboot + # + # after which the plan reads clean, because the provider only sends + # `features` when it differs. Whether a *fresh* create hits the same wall is + # untested — CT 161 was already up when this block was added. + + initialization { + hostname = var.hostname + + ip_config { + ipv4 { + address = var.ip_address + gateway = var.gateway + } + } + + dynamic "dns" { + for_each = var.dns_servers == null ? [] : [1] + + content { + servers = var.dns_servers + } + } + + user_account { + keys = var.ssh_public_keys + } + } + + cpu { + cores = var.cores + } + + memory { + dedicated = var.memory + swap = var.swap + } + + disk { + datastore_id = var.datastore_id + size = var.disk_size + + # Defaults to true, stated explicitly because the replication job below + # is silently pointless without it — a job on a guest whose only volume + # is excluded runs successfully and sends nothing. + replicate = true + } + + network_interface { + name = "eth0" + bridge = var.network_bridge + vlan_id = var.network_vlan_id + firewall = var.network_firewall + enabled = true + } + + operating_system { + template_file_id = var.template_file_id + type = var.operating_system_type + } + + # This container is the database. Several innocuous-looking edits (shrinking + # `disk_size`, changing `template_file_id`) are replacements rather than + # in-place updates, and a replacement here loses every database on the host. + lifecycle { + prevent_destroy = true + } +} + +# ZFS replication to the second node. Terraform-managed rather than created by +# hand in the UI, because replication is configured *per guest*, not per pool: +# the `AppData` pool being "replicated" is really every guest on it having a +# job like this one, and a container created without one is quietly a single +# point of failure that looks identical in the storage view. +# +# Note what this does and does not buy. The far copy is an inactive volume, +# not a running guest — recovery is migrating the container to the other node, +# which is a deliberate act unless a HA resource is added on top. And because +# replication is asynchronous, a failover loses every commit since the last +# successful run, up to one `schedule` interval of them. +resource "proxmox_replication" "postgres" { + count = var.replication_target_node == null ? 0 : 1 + + # -. Job 0 is this module's; a second target would be job 1. + id = "${var.vm_id}-0" + target = var.replication_target_node + type = "local" + schedule = var.replication_schedule + comment = "managed-by-homelab-iac — shared Postgres to ${var.replication_target_node}" + + depends_on = [proxmox_virtual_environment_container.postgres] +} diff --git a/src/shared/postgres/terraform/outputs.tf b/src/shared/postgres/terraform/outputs.tf new file mode 100644 index 0000000..aeee0e8 --- /dev/null +++ b/src/shared/postgres/terraform/outputs.tf @@ -0,0 +1,34 @@ +# Ansible finds the guest through the Proxmox API (inventory/proxmox.yml), not +# by having these wired into inventory — see the note at the top of +# build/config/terraform/main.tf. They exist so the backup job and anything +# else in the root module can refer to the guest without restating its +# identity. + +output "hostname" { + description = "Container hostname — the name this guest appears under in Ansible inventory." + value = proxmox_virtual_environment_container.postgres.initialization[0].hostname +} + +output "vm_id" { + description = "Proxmox VMID. Pass to the vzdump job if it is pinned to an explicit VMID list." + value = proxmox_virtual_environment_container.postgres.vm_id +} + +output "ip_address" { + description = <<-EOT + IPv4 address without the prefix length. This is the address every app's + `db.provision_host` and `DB_HOST` has to agree with — see the note on + `var.ip_address` for why keeping it in one place matters. + EOT + value = split("/", var.ip_address)[0] +} + +output "node_name" { + description = "Proxmox node the container was created on." + value = proxmox_virtual_environment_container.postgres.node_name +} + +output "replication_target_node" { + description = "Node this guest replicates to, or null if replication is disabled." + value = var.replication_target_node +} diff --git a/src/shared/postgres/terraform/providers.tf b/src/shared/postgres/terraform/providers.tf new file mode 100644 index 0000000..6714e16 --- /dev/null +++ b/src/shared/postgres/terraform/providers.tf @@ -0,0 +1,13 @@ +terraform { + required_version = ">= 1.6" + + required_providers { + proxmox = { + source = "bpg/proxmox" + # Deliberately unpinned until the first real `terraform init` — pin to + # whatever it resolves, and commit .terraform.lock.hcl (which is not + # gitignored) so everyone and CI get the same provider. + # version = "~> 0.0" + } + } +} \ No newline at end of file diff --git a/src/shared/postgres/terraform/variables.tf b/src/shared/postgres/terraform/variables.tf new file mode 100644 index 0000000..9c900fa --- /dev/null +++ b/src/shared/postgres/terraform/variables.tf @@ -0,0 +1,207 @@ +# Nothing environment-specific is hardcoded in the module — every value the +# caller could reasonably differ on is a variable, set from +# build/config/terraform/main.tf. + +variable "node_name" { + description = "Proxmox node to create the container on, e.g. `turtle-proxmox-01`." + type = string +} + +variable "vm_id" { + description = <<-EOT + Fixed VMID rather than letting Proxmox pick the next free one. Two things + key off it: the vzdump job in build/config/ansible/roles/pve_backup can be + pinned to an explicit VMID list, and the replication job ID in main.tf is + derived from it — a guest whose ID moves between rebuilds drops out of the + first and orphans the second. + EOT + type = number + default = 161 +} + +variable "hostname" { + description = <<-EOT + Container hostname, which is also the inventory hostname Ansible sees via + the community.proxmox dynamic inventory — so it has to match + build/config/ansible/inventory/host_vars/.yml, the file that declares + the `apps:` list installing Postgres into it. + EOT + type = string + default = "postgres" +} + +variable "cores" { + description = "CPU cores. Postgres at homelab query volumes is not CPU-bound; 2 is comfortable." + type = number + default = 2 +} + +variable "memory" { + description = <<-EOT + RAM in MiB. 2048 leaves room for `shared_buffers` (set to a quarter of this + in ../ansible/proxmox/vars.yml) plus per-connection work memory and the + kernel page cache Postgres leans on for everything not in shared_buffers. + Raise both together — the Postgres setting does not track this one. + EOT + type = number + default = 2048 +} + +variable "swap" { + description = <<-EOT + Swap in MiB. Deliberately small: a swapping database is a slow database, + and the useful behaviour when memory really runs out is a fast failure + rather than a cluster that stays up and misses every latency target. + EOT + type = number + default = 512 +} + +variable "disk_size" { + description = <<-EOT + Root disk in GiB, holding the OS *and* every database - nothing is split + onto a second volume, on purpose (see the header comment in main.tf: one + volume is what makes a replication snapshot atomic). Size it for the data + plus WAL plus headroom for any dump written locally. + + Because of that, this is the only place a "the database is running out of + space" problem gets solved: adding a second disk is not an option here, so + the number just goes up. An increase applies as a `pct resize` of the + rootfs, which on a ZFS-backed guest is a refquota change - it takes effect + online, with no data move and no filesystem grow step, so Postgres does + not need stopping. Going the other way is a replacement rather than an + in-place update, and `prevent_destroy` in main.tf blocks it, so pick a + size with room in it rather than raising this every few months. + + The real ceiling is free space on `datastore_id`, which Terraform will not + warn about - an apply that overcommits the pool succeeds, and the guest + hits ENOSPC later, when a write fails. + EOT + type = number + default = 256 +} + +variable "datastore_id" { + description = <<-EOT + Proxmox storage the root disk is created on. Defaults to the `AppData` ZFS + pool rather than `local-lvm`, because replication sends ZFS snapshots and + is therefore unavailable to a guest on any other storage type. + + The storage has to exist under this exact name on the replication target + node as well — pvesr resolves the volume by storage ID on the far side, so + a pool named differently there fails at replication time rather than at + apply time. + EOT + type = string + default = "AppData" +} + +variable "replication_target_node" { + description = <<-EOT + Proxmox node to replicate this guest to, e.g. `turtle-proxmox-02`. Null + disables the replication job entirely, which is the right setting for a + single-node cluster — but note it is also indistinguishable from "someone + forgot", so set it explicitly rather than relying on the default. + EOT + type = string + default = null +} + +variable "replication_schedule" { + description = <<-EOT + How often the ZFS snapshot is sent, as a systemd calendar event. This is + the data-loss window: an unplanned failover loses every transaction + committed since the last successful run, so the interval trades write + amplification against how much you are prepared to lose. + + `*/5` rather than Proxmox's own `*/15` default — sends are incremental, so + a shorter interval on a quiet database costs little beyond snapshot churn, + and 5 minutes of lost commits is a materially different conversation from + 15. + EOT + type = string + default = "*/5" +} + +variable "template_file_id" { + description = <<-EOT + Container template volume ID, e.g. + `local:vztmpl/debian-12-standard_12.7-1_amd64.tar.zst`. Must already be + downloaded on the node — see "Prerequisites → Proxmox" in the top-level + README.md. List what you have with `pveam list local`. + EOT + type = string +} + +variable "operating_system_type" { + description = "Template OS family, as Proxmox labels it. Match the template above." + type = string + default = "debian" +} + +variable "ip_address" { + description = <<-EOT + Static IPv4 in CIDR form, e.g. `192.168.50.54/24`. Static and not DHCP for + a stronger reason than usual: this address is written down in several + places that cannot discover it (the `provision_host` on every app that + declares a `db:`, and each app's own DB_HOST), so it moving is a + multi-file edit rather than a lease renewal. + EOT + type = string + + validation { + condition = can(cidrnetmask(var.ip_address)) + error_message = "ip_address must include a prefix length, e.g. 192.168.50.54/24." + } +} + +variable "gateway" { + description = "IPv4 default gateway." + type = string +} + +variable "dns_servers" { + description = "Resolvers for the container. Leave null to inherit the Proxmox node's." + type = list(string) + default = null +} + +variable "network_bridge" { + description = "Proxmox bridge to attach eth0 to." + type = string + default = "vmbr0" +} + +variable "network_vlan_id" { + description = "VLAN tag for eth0, or null for an untagged bridge." + type = number + default = null +} + +variable "network_firewall" { + description = "Enable the Proxmox firewall on this interface." + type = bool + default = false +} + +variable "ssh_public_keys" { + description = <<-EOT + Public keys installed for root in the container. Ansible authenticates + with the matching private half (`homelab/ci/ssh` in Vault); without at + least one key here the guest is unreachable and playbooks/proxmox.yml has + nothing it can do with it. + EOT + type = list(string) +} + +variable "tags" { + description = <<-EOT + Proxmox tags, shown in the UI and read by the Ansible dynamic inventory. + `terraform` is load-bearing: build/config/ansible/inventory/proxmox.yml + filters on it to tell guests this repo creates apart from ones made by + hand on the node. Drop it and this container is simply never deployed + to — silently, with no error anywhere. + EOT + type = list(string) + default = ["postgres", "shared", "terraform"] +}