russells/homelab
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Initial checkin of code for managing homelab
Some checks are pending
deploy / deploy (push) Waiting to run
Details

Browse source
This commit is contained in:
Russell Seymour 2026-08-24 20:52:36 +01:00
commit 1250c9cef6
Signed by: russells
GPG key ID: 78116D73A1E3EC41
131 changed files with 10107 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

116
.forgejo/workflows/deploy.yml Normal file
View file

@ -0,0 +1,116 @@
name: deploy
on:
push:
branches: [main]
paths:
# No build/config/terraform/** — this workflow only deploys to Unraid,
# and `terraform apply` is run by hand. Add it if that ever changes.
- "src/**"
- "build/config/ansible/**"
jobs:
deploy:
# Requires a self-hosted Forgejo Actions runner on your LAN (labelled
# `unraid-deploy`) with: network access to Vault and to the Unraid hosts'
# SSH + Postgres ports, and Python + pip available. See README.md
# "Bootstrapping" — this workflow can only run once that runner exists,
# which is why the first deploy has to happen by hand. The SSH key
# `ansible_user` (root) authenticates with is fetched from Vault below,
# not stored as a runner or repo secret — see README.md "SSH access".
runs-on: unraid-deploy
env:
VAULT_ADDR: ${{ secrets.VAULT_ADDR }}
VAULT_AUTH_METHOD: approle
VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID }}
VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # need history to diff — a shallow clone can't compute this
- name: Install Ansible and Python dependencies
run: |
python3 -m pip install --upgrade pip
python3 -m pip install ansible hvac psycopg2-binary
- name: Install Ansible collections
working-directory: build/config/ansible
run: ansible-galaxy collection install -r requirements.yml
- name: Fetch SSH deploy key from Vault
run: |
set -eu
python3 - <<'PY'
import os
import hvac
client = hvac.Client(url=os.environ["VAULT_ADDR"])
client.auth.approle.login(
role_id=os.environ["VAULT_ROLE_ID"],
secret_id=os.environ["VAULT_SECRET_ID"],
)
secret = client.secrets.kv.v2.read_secret_version(
path="homelab/ci/ssh", mount_point="kv"
)["data"]["data"]
key_path = os.path.join(os.environ["RUNNER_TEMP"], "unraid_ansible_key")
with open(key_path, "w") as f:
f.write(secret["PRIVATE_KEY"].rstrip() + "\n")
os.chmod(key_path, 0o600)
PY
echo "ANSIBLE_PRIVATE_KEY_FILE=$RUNNER_TEMP/unraid_ansible_key" >> "$GITHUB_ENV"
- name: Determine changed stacks
id: changed
run: |
set -eu
# Three possible outcomes, and they are not the same thing:
# all=true deploy every stack on every host
# only_stacks=a,b deploy just those
# skip=true nothing here affects the Unraid deployment
# An empty only_stacks must never be read as "deploy everything" —
# that is exactly what a Proxmox-only or Terraform-only push
# produces.
#
# A change under build/config/ansible/ can affect how every stack
# is deployed (the role, inventory, playbook), so don't try to
# narrow it. Same if the diff can't be computed at all (force-push,
# or a first push where `before` is all-zeros): fall back to a full
# deploy rather than silently deploying nothing.
if ! changed_files=$(git diff --name-only "${{ github.event.before }}" "${{ github.sha }}" 2>/dev/null); then
echo "all=true" >> "$GITHUB_OUTPUT"
exit 0
fi
if echo "$changed_files" | grep -q '^build/config/ansible/'; then
echo "all=true" >> "$GITHUB_OUTPUT"
exit 0
fi
# Only changes that affect the Unraid deployment count, since that
# is all this job runs. A stack's common/ and ansible/unraid/ do;
# its terraform/ and ansible/proxmox/ don't.
stacks=$(echo "$changed_files" \
| { grep '^src/' || true; } \
| { grep -Ev '/(terraform|ansible/proxmox)/' || true; } \
| awk -F/ '{ print ($2 == "shared") ? $3 : $2 }' \
| sort -u | paste -sd, -)
if [ -z "$stacks" ]; then
echo "skip=true" >> "$GITHUB_OUTPUT"
else
echo "only_stacks=$stacks" >> "$GITHUB_OUTPUT"
fi
# Unraid only. Deploying to Proxmox from CI would need the runner to
# reach the Proxmox API and the guests, and `terraform apply` is
# deliberately kept manual — creating and destroying LXCs on a push is
# a bigger blast radius than restarting a Compose stack. Run
# playbooks/proxmox.yml by hand for now.
- name: Run deploy playbook
if: steps.changed.outputs.skip != 'true'
working-directory: build/config/ansible
run: |
if [ "${{ steps.changed.outputs.all }}" = "true" ]; then
ansible-playbook playbooks/unraid.yml
else
ansible-playbook playbooks/unraid.yml -e only_stacks="${{ steps.changed.outputs.only_stacks }}"
fi

33
.gitignore vendored Normal file
View file

@ -0,0 +1,33 @@
# Ansible
*.retry
build/config/ansible/inventory/*.retry
# Rendered secrets — only .env.example is ever committed
**/.env
!**/.env.example
# Terraform
# .terraform/ holds the provider cache and, after
# `terraform init -backend-config=...`, the Postgres connection string with
# its password — never commit it. .terraform.lock.hcl is deliberately NOT
# ignored: it pins provider versions and belongs in git.
.terraform/
*.tfstate
*.tfstate.*
*.tfvars
!*.tfvars.example
# A saved plan (`terraform plan -out=…`) is not just a diff: it embeds a copy
# of the state it was made against *and* the values of every input variable,
# `sensitive` ones included — so build/config/terraform/deploy.plan contains
# the Proxmox API token in the clear. Ignored so that a `git add -A` can't
# publish it.
*.plan
# OS cruft
.DS_Store
Thumbs.db
# Scratch output that isn't state and isn't secret-free — e.g. the k3s
# kubeconfig playbooks/k3s.yml fetches to local/k3s/<cluster>.kubeconfig,
# which carries a client cert.
local/

597
CLAUDE.md Normal file
View file

@ -0,0 +1,597 @@
# Homelab IaC — Project Notes
Manages apps as infrastructure-as-code across three platforms — Unraid
server(s), Proxmox, and a bare-metal k3s cluster — with secrets sourced from
HashiCorp Vault and CI/CD via Forgejo Actions (self-hosted, on the Forgejo
instance this repo deploys).
Ansible is the deployment tool on **all three**; what differs is the
deployment primitive and who creates the host:
| | Unraid | Proxmox | k3s |
|---|---|---|---|
| Provision the host | already exists | Terraform (`bpg/proxmox`) creates an LXC | already exists (4 Raspberry Pis) |
| Install the app | Ansible + `compose_stack` → `docker compose up -d` | Ansible + `lxc_app` → native install + systemd | Ansible + `k3s_app` → manifests into k3s's auto-deploy dir |
| App config source | `common/vars.yml` + `ansible/unraid/vars.yml` | `common/vars.yml` + `ansible/proxmox/vars.yml` | `common/vars.yml` + `ansible/kubernetes/vars.yml` |
| App list lives in | `host_vars/<host>.yml` `stacks:` | `host_vars/<guest>.yml` `apps:` | `group_vars/k3s_cluster.yml` `k3s_apps:` |
The k3s column is the newest and reached that shape late: `homelab-utils`
started as cluster-bootstrap-only, with no `src/<app>/` entry, and the third
column was added when Authentik moved onto it (see
`docs/authentik-migration.md`). Two things about it don't reduce to the
table:
- **The cluster itself has no `src/` entry.** Terraform never provisions it
and it isn't an app, so it gets an inventory group (`k3s_cluster`) and
`roles/k3s_node`, the same way the Proxmox *node* gets `roles/pve_backup`
while its guests get `lxc_app`.
- **Cluster services aren't apps.** `k3s_metallb`, `k3s_monitoring`,
`k3s_postgres` and `k3s_cert_manager` are the cluster's own
infrastructure — one hard-coded role each, config in `group_vars`. Apps go
through the generic `k3s_app` and live in `src/`. Both end up as
`HelmChart` CRs, so they look alike on disk; the distinction is whether
anything would still need it if no app were deployed. See "K3s cluster
(homelab-utils)" below.
### Hosts
Two Unraid entries are gone from the original plan: `nas2` was retired, and its
address (192.168.50.2) was reassigned to `turtle-proxmox-01` — so a stale copy
of it fails by reaching the wrong machine rather than by timing out. What
remains: `nas01` (Unraid, 192.168.50.1), `turtle-proxmox-01`/`-02` (Proxmox,
.2/.3), the guests on them from the API inventory, and the four k3s Pis
(.60–.63, MetalLB pool .80–.99). Gateway is 192.168.50.254.
**Proxmox LXC guests live in 192.168.50.50–.59** — Forgejo .52, shared
Postgres .54. Addresses here are assigned by hand from that band, not derived
from anything, so a new guest's `ip_address` is a question for whoever owns
the network rather than a value to pick.
The shared Postgres for everything outside the cluster is now the Proxmox LXC
at 192.168.50.54 — see `docs/postgres-proxmox.md`. The Unraid Compose
deployment under `src/shared/postgres/ansible/unraid/` is kept for the
platform-shape reason every unused platform folder is kept, not because it is
running anywhere.
This repo supersedes the planning done in `../homelab/CLAUDE.md` — that file
has the original decision log if you want the "why" behind the architecture
in more detail. Multi-host support, the shared-Postgres pattern, and the
Forgejo-Actions-hosts-itself bootstrap problem were decided after that doc
was written; this file and `README.md` are the current source of truth.
## Layout
- `src/<app>/common/vars.yml` — `env_defaults:` that hold regardless of where
the app runs (version, ports, domain, DB name/user).
- `src/<app>/ansible/unraid/` — the Compose deployment: `docker-compose.yml`,
`.env.example`, a `vars.yml` of Unraid-specific overrides (appdata paths,
the `shared-postgres` Docker network alias), and optionally an `icon.png`
for the Unraid Docker page.
- `src/<app>/ansible/proxmox/` — the native install: `vars.yml` of
Proxmox-specific overrides, `install.yml` of app-specific install steps,
and templates for the app's own config file and systemd unit. Two apps
ship one: `src/forgejo/` (binary + systemd unit, the shape the role was
written for) and `src/shared/postgres/` (distro packaging, so the unit
comes from the package and `install.yml` only reconfigures the cluster).
- `src/<app>/ansible/kubernetes/` — the k3s deployment: `vars.yml` of
cluster-specific overrides, and one or more `*.yaml.j2` manifests (normally
a `HelmChart` CR, plus anything the chart won't create — see
`src/authentik/`). "kubernetes" rather than "k3s": the contents are plain
Kubernetes apart from the `HelmChart` CR.
- `src/<app>/terraform/` — a module defining this app's LXC (template, cores,
memory, disk, IP), called from `build/config/terraform/main.tf`.
- `src/shared/<service>/` — shared services (databases, etc.) used by
multiple apps, same shape.
- `build/config/ansible/` — inventory, playbooks, and the roles. Three of
them deploy *apps*, one per platform, and share the same layering and
`state: present|absent` contract: `compose_stack` (Unraid), `lxc_app`
(Proxmox guests), `k3s_app` (the cluster). `compose_stack` and `k3s_app`
each split into `deploy.yml`/`remove.yml`, dispatched by `main.yml` on the
entry's `state:`. The rest configure infrastructure rather than apps:
`k3s_node` (bootstraps a Pi as k3s server or agent, picked by inventory
group), `k3s_metallb`, `k3s_monitoring`, `k3s_postgres`,
`k3s_cert_manager` (cluster services), `k3s_maintenance` (drain/reboot/
uncordon), `unattended_upgrades`, and `pve_backup` (the Proxmox node's
backup storage and vzdump schedule).
- `build/config/terraform/` — Proxmox provider + `backend "pg"` state config,
and the module calls saying what infrastructure exists.
- `.forgejo/workflows/` — CI/CD, runs once Forgejo + a self-hosted runner
exist (see "Bootstrapping" in README.md — chicken-and-egg on the first
deploy).
## Key decisions
- Compose files are static and reference `${VAR}`; only the rendered `.env`
changes per deploy. Nothing secret is ever committed.
- Config is layered, not duplicated: `common/vars.yml` merged with
`ansible/<platform>/vars.yml` (platform wins), then Vault secrets on top
(secrets win). Both roles do this the same way, so moving an app between
platforms means writing a new platform vars file, not restating its config.
- One Vault KV path per app (`homelab/<app>`), fetched as a whole dict via
`community.hashi_vault.vault_kv2_get`. The prefix is `homelab/`, not
`unraid/`, because an app's secrets don't change with the platform it lands
on.
- Unraid draws a Compose deployment's stack row and its containers from
unrelated sources, so one committed `src/<app>/ansible/unraid/icon.png` is
copied to two places. The **stack row** is Compose Manager serving
`<project dir>/icon.png` off disk — a file or nothing, no label and no URL,
and only in the maintained fork (Compose Manager Plus); on the original
plugin the copy is inert. The **containers** get icon, WebUI link and
console shell from the dockerMan template that created them, which a Compose
stack doesn't have, so each service carries `net.unraid.docker.*` labels
(6.10+ falls back to them). The label path is a second copy under
`/mnt/user/appdata/icons/` rather than the project-folder one, because the
webgui resolves it on every Docker page load and that shouldn't hit the
flash drive.
- Of those labels only the icon is role-supplied: `compose_stack` injects
`STACK_ICON` *underneath* both vars.yml layers, so an app can override it
with a hosted URL. WebUI and shell stay literal in the compose file — the
WebUI port has to be the published one and the shell is a property of the
base image, so both belong next to `ports:` and `image:` rather than in a
variable. Details, including where SVG does and doesn't work, in README.md →
"Icons on the Unraid Docker page".
- Shared Postgres is provisioned once per host; per-app DB/user creation
is idempotent via `community.postgresql`, not `docker-entrypoint-initdb.d`
(which only runs once on an empty volume — wrong fit for a shared service
apps get added to over time). The Unraid instance was pinned to 13 because
its data directory predated this repo; the Proxmox one is 17 from the PGDG
archive, a fresh cluster with nothing to stay compatible with.
- The Proxmox shared Postgres lives on the `AppData` ZFS pool with a
`proxmox_replication` job to the second node, and **everything is on one
volume** — no `mount_point` splitting PGDATA from the WAL. Unlike Forgejo's
"no bind mounts", which is about what `vzdump` includes, this is a
correctness requirement: replication snapshots each of a guest's volumes
independently, so two volumes replicate as two snapshots taken at different
instants, and the far node can end up with a data directory and a WAL that
disagree. One volume means one atomic snapshot — exactly the
crash-consistent image WAL replay is designed for. It also means running
out of space has one fix and not two: raise `disk_size` on the module call
(256 GiB as of this writing, up from the original 32 once Forgejo's
database made the OS-sized rootfs look optimistic), never add a second
disk. Cheap to do — on ZFS the resize is a refquota change applied to a
running guest — but one-way, since shrinking is a replacement and
`prevent_destroy` blocks it. Runbook in `docs/postgres-proxmox.md`. Two
further consequences worth keeping straight: replication is configured *per guest*, not per pool, so a
container created on `AppData` without a job is a single point of failure
that looks identical in the storage view (hence Terraform owning the job
rather than a hand-run `pvesr`); and it's asynchronous, so a failover loses
up to `replication_schedule` — five minutes — of commits. It is not a
backup; `pve_backup`'s vzdump is.
- `full_page_writes = off` on the Proxmox instance, and only there. It guards
against a torn page, which copy-on-write storage cannot produce, so on ZFS
the full-page images are pure write amplification that also becomes
replication traffic. The same setting on ext4 or xfs risks unrecoverable
corruption — which is why it's in `ansible/proxmox/vars.yml` and must never
migrate to `common/vars.yml`.
- Terraform provisions the guest and stops there — it has no way to deploy an
app into an LXC, so Ansible picks up from the point the container exists.
- Proxmox guests keep all their state on their own rootfs; no bind mounts, no
NFS shares mounted into a container. Counter-intuitive if the goal is "back
the repos up to the NAS", but `vzdump` excludes bind mounts by design, so a
bind-mounted repo directory is the one thing *missing* from the archive.
Everything local means one archive is a complete filesystem restore point.
Getting a copy off-box is the node's job (`pve_backup` → NFS storage on the
NAS + a vzdump schedule), not the container's.
- State an app keeps *outside* its container is that app's problem. Forgejo's
database is on the shared Postgres LXC — a different guest, and so a
different vzdump archive — so a systemd timer in the
container `pg_dump`s it onto the container's own disk, and the vzdump archive
carries both halves. The dump schedule and the vzdump window are a pair —
move one, move the other.
- Proxmox hosts come from the `community.proxmox` dynamic inventory plugin
querying the Proxmox API, so `terraform apply` creating an LXC doesn't also
require an inventory edit.
- The `terraform` tag is what marks a Proxmox guest as this repo's to manage.
The dynamic inventory returns every LXC on the cluster, hand-made ones
included, so `inventory/proxmox.yml` filters on that tag — both Terraform
modules set it (`tags` in their `variables.tf`), nothing else does. It
filters rather than narrowing the `proxmox_guests` condition, so an
unmanaged guest is absent from `all` as well and no future play can reach
it by accident. The failure mode is silent in one direction: a new module
that forgets the tag creates a container Ansible simply never visits, with
no error anywhere — same shape as an omitted `stacks:` entry, and the first
thing to check if a freshly-applied guest is never deployed to.
- Terraform state lives in Postgres (`backend "pg"`), not a local file, so CI
and a workstation see the same state — specifically the **CloudNativePG
cluster on k3s**, not the shared Postgres this configuration provisions on
Proxmox. That would be circular: Terraform would need the database to exist
in order to create the container the database runs in. State goes somewhere
Terraform has no hand in building, which breaks the cycle outright at the
cost of a dependency the rest of the repo avoids — `terraform apply` now
needs the Pis up. Worth it over the alternative (local state, apply,
`init -migrate-state`), which has to be got right exactly once. Bootstrap
ordering is `docs/postgres-proxmox.md`.
- Every app keeps the same `common/` + `ansible/` + `terraform/` shape even
when only one platform is in use, so adding the other later doesn't require
restructuring.
- Removal on Unraid is explicit (`state: absent` on the stack entry), never
implied by deleting the entry. Ansible stores no record of the last run, so
an omitted stack isn't removed — it's just never visited again, and keeps
running unmanaged. The entry stays as a tombstone documenting that the app
was deliberately removed. Volumes, images and the app's database are
further opt-ins (`remove_volumes`, `remove_images`, `remove_database`)
rather than being implied by `state: absent`, and the Vault path is never
touched — a default teardown is meant to be reversible. `lxc_app` has no
counterpart yet; it would need a per-app `uninstall.yml`.
- The k3s cluster's join token is a fixed value from Vault
(`homelab/k3s-homelab-utils` → `K3S_TOKEN`), not the one `k3s server`
generates for itself on first install. Both the control-plane play and the
worker play look it up independently, so a full rebuild — wipe both SD
cards, reinstall — reproduces the same cluster identity instead of needing
a freshly-generated token fetched off the server and redistributed by hand.
Same "generate once, store it, never let a redeploy invent a new one"
reasoning as Forgejo's fixed secrets, applied to cluster identity instead
of app state.
- The SSH key the k3s plays connect with is fetched from Vault
(`homelab/ci/ssh-k3s`) by `playbooks/k3s_ssh_key.yml` and written to
`local/`, which both `k3s.yml` and `k3s_maintenance.yml` import as their
first play — rather than the operator loading it into `ssh-agent` first, as
the Unraid manual flow still expects. Same arrangement CI already uses for
the Unraid key, just moved into the playbook because these two are
manual-only and there's no workflow to hold the fetch step. It's a separate
playbook, not a `pre_tasks` block, because both entry points need it and
neither can assume the other ran. It targets `k3s_cluster` rather than
`localhost` for a non-obvious reason: the implicit localhost isn't a member
of `all`, so it wouldn't inherit `vault_addr`/`vault_kv_mount`/
`vault_auth_method` from `group_vars/all.yml`. Nothing in it connects to a
Pi — `gather_facts: false` plus `delegate_to: localhost`, necessarily, since
the key isn't on disk yet at that point. Falling back to a local key takes
*two* overrides (`-e k3s_ssh_key_fetch=false` alongside
`-e ansible_ssh_private_key_file=...`): pointing at another key alone
doesn't help if Vault is what's broken, because the fetch play fails before
any other play runs.
- `playbooks/k3s.yml` is deliberately **not** wired into `deploy.yml` or
`.forgejo/workflows/deploy.yml`. Converging 4 physical nodes on every push
is a bigger blast radius than restarting a Compose stack — the same
reasoning that keeps `terraform apply` manual for Proxmox. Run it by hand.
- `playbooks/k3s.yml`'s plays are tagged so one piece can be converged on its
own — `nodes`, `metallb`, `monitoring`, `postgres`, `services` (those three
cluster services together), `upgrades`, `kubeconfig`. Play-level tags, one
per play, not per-task tags inside the roles: the plays already are the
units, and tagging tasks would mean every role carrying tags for a
selectivity only this playbook needs. This is the k3s equivalent of
`only_stacks` for Unraid — a cluster service isn't a `stacks:` entry, so
`only_stacks` can't reach it. The SSH-key import play is tagged `always`
rather than getting a name of its own, since it's the prerequisite for
anything else connecting and has to survive every `--tags` filter. Note
what a tagged service run does *not* cover: the roles only drop manifests
for k3s's controllers to reconcile, so a green playbook means the files
landed, not that the workload came up.
- `k3s_node` is one role, not two, even though a server and an agent install
differently. Both need the same OS prep (cgroups, swap), and which install
path runs is a group var (`k3s_node_role: server`/`agent`, set per
inventory group) rather than a second role — so the shared prep can't drift
between the two nodes types the way copy-pasted tasks eventually would.
- Extra `INSTALL_K3S_EXEC` flags split across two vars, not one:
`k3s_extra_args` (both roles) and `k3s_server_extra_args` (control-plane
only). `k3s agent` doesn't understand server-only flags like `--disable`
and fails to start if handed one, so a flag like `--disable=servicelb`
(which `k3s_metallb` needs to avoid fighting k3s's bundled ServiceLB for
the same IPs) can't safely live in the shared list.
- `k3s_node`'s install tasks track the exec line they last installed a node
with (a small marker file next to k3s's own config) and reinstall —
restarting the `k3s`/`k3s-agent` service, not the node — whenever it
drifts from what `k3s_extra_args`/`k3s_server_extra_args`/`k3s_api_tls_san`
currently say, even if `k3s_version` hasn't changed. Deliberately folded
into the ordinary install task rather than a separate maintenance
playbook: `playbooks/k3s.yml` is already manual-only and documented as
safe to re-run, and unlike the reboots `k3s_maintenance` guards, bouncing
the k3s process doesn't take pods down — containerd keeps them running
underneath it. The worker play still runs with `serial: 1` so at most one
node's kubelet is ever mid-restart at once.
- `k3s server`/`k3s agent` are installed with an explicit `--node-name
{{ inventory_hostname }}`, not left to default to the OS hostname. This is
what lets `roles/k3s_maintenance` address a node by
`inventory_hostname` when draining/uncordoning — the k8s node object and
the Ansible host are guaranteed to be the same string.
- Patching a k3s Pi is split into two roles that don't know about each other
directly, only through a file: `unattended_upgrades` installs updates
hands-off but with `Automatic-Reboot "false"`, and `k3s_maintenance` is the
only thing that actually reboots a node — triggered by the presence of
`/var/run/reboot-required`, which is the OS's own signal, not something
either role invents. Splitting it this way means the risky half (taking a
node out of the cluster) is exactly one thing (`playbooks/k3s_maintenance.yml`,
`serial: 1`), not entangled with the routine half (installing packages),
which runs unattended every day on every node without anyone thinking about
it.
- `k3s_maintenance`'s kubectl calls (`drain`, `wait`, `uncordon`) are
delegated to the control-plane host and run as `k3s kubectl`, k3s's own
bundled client, rather than requiring a kubectl install or a kubeconfig
anywhere else — including on whatever eventually runs this on a schedule.
This also has to work when the node currently being processed *is* the
control plane: delegating a host to itself is just a normal SSH connection,
used right up until the moment that connection reboots out from under it.
- `k3s_monitoring` (a lean kube-prometheus-stack, for OpenLens and similar
tools) is installed as a `HelmChart` custom resource dropped into k3s's own
auto-deploying manifests directory, not run through a `helm` binary or the
`kubernetes.core.helm` collection — k3s ships a helm-controller that
reconciles anything found there, the same mechanism it uses to install its
own bundled Traefik and ServiceLB. That keeps the pattern this repo already
uses everywhere else (Ansible renders a file, something else converges on
it) instead of adding a second, unrelated way to reach the cluster from the
controller. Grafana and Alertmanager are left disabled — OpenLens brings
its own dashboards and this cluster doesn't page anyone — and the
control-plane scrape targets (`kubeControllerManager`, `kubeScheduler`,
`kubeProxy`, `kubeEtcd`) are disabled too, since k3s bundles those into one
static binary instead of exposing them the way the chart expects; leaving
them on just produces permanently-"down" targets, not a working scrape.
- `k3s_metallb` (MetalLB, L2 mode) follows the same `HelmChart` CR pattern as
`k3s_monitoring`, plus a second, plain manifest for its
`IPAddressPool`/`L2Advertisement` config dropped in the same
auto-deploying directory — k3s's deploy controller applies any manifest it
finds there, not only `HelmChart` CRs, and retries one referencing CRDs
that don't exist yet rather than failing outright, so the config doesn't
need to wait on the chart install finishing first. It replaces k3s's
bundled ServiceLB rather than running next to it — both would otherwise
hand out IPs for the same `LoadBalancer` Services — so
`k3s_server_extra_args` in `inventory/group_vars/k3s_cluster.yml`
carries `--disable=servicelb` (server-only — `k3s_extra_args`, applied to
both server and agent, can't carry a server-only flag like `--disable`
without breaking agent installs). The IP pool (`k3s_metallb_address_range`, same
file) has no built-in default; the role fails fast rather than silently
advertising an empty pool.
- `k3s_postgres` (CloudNativePG) is the k3s cluster's own shared Postgres —
a third platform for `src/shared/postgres/`, alongside Unraid and Proxmox,
deployed via Helm rather than Compose or a native install. Chosen over
Bitnami's postgresql-ha (repmgr+pgpool, and Bitnami's free chart/image
catalog was restructured into a "legacy" repo in 2025) and the Zalando
operator (Patroni-based, heavier) because it fits the same `HelmChart` CR
pattern already established: the operator installs as a chart CR, same
shape as `k3s_metallb`/`k3s_monitoring`, and the actual cluster is a plain
`Cluster` CR manifest — same "config manifest that outlives the CRDs it
references" trick as `k3s_metallb`'s `IPAddressPool`. It's a separate
physical instance from the Docker-based shared/postgres (pinned to 13),
but reuses the same Vault path (`homelab/shared/postgres`) rather than a
k3s-specific one — one superuser identity for the "shared postgres"
concept regardless of which platform it's running on, consistent with how
every host already draws from that same path. 1 primary + 1 replica,
scheduled on worker Pis only via `nodeAffinity` — the control plane stays
free of app pods, same boundary `k3s_metallb`/`k3s_monitoring` already
keep. HA replication only for now, no backups.
- The CNPG cluster is published on the LAN through MetalLB at a pinned
`k3s_postgres_loadbalancer_ip`, on top of the three `ClusterIP` Services
CNPG makes for every `Cluster`. Declared inside the `Cluster` CR as a
managed service (`.spec.managed.services.additional`, `selectorType: rw`)
rather than as a Service manifest of this repo's own, so the operator owns
the selector and the address follows a failover instead of needing to be
re-pointed by hand. Pinned rather than auto-assigned because the address is
meant to be written down in config elsewhere; it comes out of the bottom of
`k3s_metallb_address_range` and would need a second `autoAssign: false`
pool if anything else on this cluster ever wants a fixed IP. This is what
closes the reachability half of per-app DB provisioning — the
`community.postgresql` tasks `compose_stack`/`lxc_app` use can now reach
this instance the same way they reach the Unraid one. `roles/k3s_app` is
the other half, and uses exactly those tasks; note the asymmetry it
relies on, which is easy to misread as a mistake. The *provisioning* runs
from the Ansible controller and therefore uses the LoadBalancer address
(`k3s_postgres_loadbalancer_ip`), because `delegate_to: localhost` is off
the cluster and can't route to a ClusterIP. The *app* uses the in-cluster
`shared-postgres-rw` DNS name, because sending pod traffic out to the LAN
and back would put MetalLB's L2 speaker in the path of every query for no
benefit. Two addresses for one database, each correct for its caller.
- `roles/k3s_app` is the cluster's counterpart to `compose_stack` and
`lxc_app`: the generic role that deploys an *app*, driven by `k3s_apps:`
in `group_vars/k3s_cluster.yml`, as distinct from the `k3s_*` roles that
each deploy one cluster service. It renders whatever `*.yaml.j2` the app
ships under `src/<app>/ansible/kubernetes/` into k3s's auto-deploying
manifests directory, so adding a manifest to an app is dropping a file
next to the others rather than a role change. Secrets go into a separate
Kubernetes Secret rendered from Vault, never interpolated into the
manifests — the same committed-config/generated-secrets split as Unraid's
static compose file plus rendered `.env`, which is why the manifests can
stay 0644 on the node while the Secret is 0600.
- `k3s_apps:` lives in `group_vars/k3s_cluster.yml`, not `host_vars/`, unlike
both other platforms. An app is deployed to the cluster, not to a node:
the play runs against `k3s_control_plane` only because that's where the
manifests directory is, which is an implementation detail of how k3s is
reached rather than a statement about where the app runs.
- Removal on k3s is one pass, not two. `compose_stack` needs the compose file
still on disk to run `docker compose down`, so `state: absent` and deleting
`src/<app>/` have to be separate commits (see "Removing an app" below).
k3s's deploy controller tracks which resources each manifest file created,
so deleting the file *is* the teardown and `src/<app>/` can go in the same
commit. The tombstone convention still applies for the same reason as
everywhere else — an entry dropped from `k3s_apps:` is never visited again
and keeps running unmanaged.
- `k3s_cert_manager` is the fourth cluster service and the first that exists
purely for apps rather than for the cluster — nothing in the cluster needs
a certificate, `k3s_app`'s tenants do. It's still a cluster service rather
than an app, on the same reasoning as the shared Postgres: one
`ClusterIssuer` that every app's Ingress annotates itself against, rather
than each app carrying its own ACME account and DNS credentials. DNS-01,
not HTTP-01, because HTTP-01 needs Let's Encrypt to reach the cluster from
the internet on port 80 and this LAN deliberately isn't reachable — which
has the useful side effect that a certificate can be issued *before* DNS
points at the cluster, so a migration's TLS is settled before its cutover.
The solver stanza itself has no default and the role fails fast without
one: it depends on who runs the DNS, which the repo can't know. Same
fail-fast-on-unset treatment as `k3s_metallb_address_range`.
- Apps on k3s reach the outside through the Traefik k3s already bundles,
on the MetalLB address its Service holds, rather than each getting its own
LoadBalancer IP. One entrypoint and host-based routing means adding an app
costs a DNS record, not a pool address — the opposite of the choice made
for `k3s_postgres`, which is pinned to its own IP precisely because
Postgres isn't HTTP and can't be name-routed.
- `k3s_traefik` is the one cluster-service role that installs nothing: k3s
installs Traefik itself, so the role only adjusts it, via a
`HelmChartConfig` merged over k3s's own `HelmChart` rather than an edit to
the `traefik.yaml` k3s rewrites on every server start. Its scope is the
dashboard, which a stock k3s 404s — Traefik still builds it, but the chart
stopped shipping the router that reaches it in v28. The role restores that
router on Traefik's internal `traefik` entrypoint (port 9000, unpublished →
port-forward only), and optionally publishes the dashboard on a hostname
behind an Authentik forward-auth middleware when
`k3s_traefik_dashboard_host` is set. Three things worth keeping straight:
the internal route stays on even when the hostname one exists, because
authenticated access depends on Authentik → CNPG → a healthy cluster,
exactly what you'd open the dashboard to diagnose; the published route is
an `IngressRoute` with an explicit cert-manager `Certificate` rather than
an annotated `Ingress`, because the dashboard is `api@internal` and has no
Service to point an `Ingress` at; and the role fails fast if a hostname is
set without an auth address, since the failure mode is publishing every
router, service and middleware on the cluster to the LAN unauthenticated.
Authentik's provider must be **forward auth (domain level)** —
single-application mode needs `/outpost.goauthentik.io/` routed on the
dashboard's own host, a cross-namespace service reference Traefik rejects
unless `allowCrossNamespace` is on.
## Adding a new app
1. `src/<app>/common/vars.yml` — `env_defaults:` for the portable config.
2. Whichever platform(s) it targets:
- **Unraid:** `src/<app>/ansible/unraid/` with `docker-compose.yml`
(static, `${VAR}`-driven), `vars.yml` of overrides, `.env.example`
documenting every var.
- **Proxmox:** `src/<app>/ansible/proxmox/` with `vars.yml`,
`install.yml`, and config/systemd templates; plus
`src/<app>/terraform/` defining its LXC, wired into
`build/config/terraform/main.tf`.
- **k3s:** `src/<app>/ansible/kubernetes/` with `vars.yml` and at least
one `*.yaml.j2` manifest — normally a `HelmChart` CR, plus anything the
chart won't create for you. See `src/authentik/` for a worked example,
including the two things a chart usually leaves out: a PVC, and secrets
referenced by `secretKeyRef` rather than set as values.
3. Populate its Vault path (`homelab/<app>`) with real secret values.
4. Declare it on the target:
- Unraid → `host_vars/<host>.yml` `stacks:` list.
- Proxmox → `host_vars/<guest>.yml` `apps:` list.
- k3s → `group_vars/k3s_cluster.yml` `k3s_apps:` list.
All three take the same entry shape (name, src, vault_path, optional
`db:`, optional `state:`). Never declare the same app on more than one at
once — they share a Vault path and a database, so two live deployments
corrupt each other's state. Moving an app *between* platforms is
therefore a cutover with a verification step in the middle, not an edit;
`docs/authentik-migration.md` is the worked example.
5. Leave the unused platforms' folders as README placeholders rather than
deleting them — the shape is the point.
## Removing an app
**Unraid.** Set `state: absent` on its `stacks:` entry and run
`playbooks/unraid.yml` — don't delete the entry, which removes nothing. Then
delete `src/<app>/` in a *second* pass, never before: `docker compose down`
needs the compose file to know what it's tearing down. Full detail, including
the opt-in flags for volumes/images/database, is in README.md → "Removing an
app".
**k3s.** Same `state: absent` tombstone rule, but one pass — deleting the
manifests is the teardown, because k3s's deploy controller garbage-collects
what each file created, so `src/<app>/` can go in the same commit. The
database, PVCs and Vault path deliberately survive; there's no
`remove_volumes`/`remove_database` equivalent yet, and adding one is a
"when it's actually wanted" job rather than a guess at the shape.
**Proxmox.** No teardown path at all — `lxc_app` would need a per-app
`uninstall.yml`, and neither app that ships an `install.yml` has one. Note
this is worse for the shared Postgres than for an ordinary app: tearing it
down means every other app's database as well.
## K3s cluster (homelab-utils)
4 Raspberry Pis, `inventory/hosts.yml` → `k3s_cluster` (`k3s_control_plane`:
1 host, `k3s_workers`: 3). No Terraform — the Pis already exist. The cluster
itself has no `src/<app>/` entry (it isn't an app), but apps now land on it:
`k3s_apps:` in `group_vars/k3s_cluster.yml`, deployed by `roles/k3s_app` from
`src/<app>/ansible/kubernetes/`. `playbooks/k3s.yml` is manual-only (see "Key
decisions" above); run it with:
```sh
cd build/config/ansible
ansible-playbook playbooks/k3s.yml
# or one piece of it — see "Key decisions" above for the tag list
ansible-playbook playbooks/k3s.yml --tags postgres
# apps only, or one app
ansible-playbook playbooks/k3s.yml --tags apps
ansible-playbook playbooks/k3s.yml --tags apps -e only_apps=authentik
```
`--tags services` covers the cluster's own infrastructure and `--tags apps`
what runs on top of it; neither implies the other, so "everything except
reinstalling k3s" is `--tags services,apps`.
Prerequisites, once, before the first run: the `ansible` user exists on every
Pi with NOPASSWD sudo and the **public** half of this repo's k3s SSH key
installed, and `homelab/k3s-homelab-utils` → `K3S_TOKEN` plus
`homelab/ci/ssh-k3s` → `PRIVATE_KEY` are set in Vault. Nothing needs setting
up on the controller beyond `VAULT_ADDR`/`VAULT_TOKEN` — the private key comes
from Vault at run time (see "Key decisions" above). Full detail in README.md →
"K3s (Raspberry Pi)" and `docs/vault-secrets.md`.
**Rebuilding**: reimage the SD card(s), redo the `ansible` user + SSH key
step, then re-run the playbook — the fixed Vault token means the rebuilt
node(s) rejoin the same cluster identity rather than needing a token hunted
down from a live server. **Adding a fifth Pi**: add it under
`k3s_control_plane` or `k3s_workers` in `hosts.yml`; no `host_vars/` entry
needed, `k3s_node_role` comes from the group. This repo's `k3s_node/server.yml`
assumes a single control-plane node — adding a second one for HA needs
changes there, not just an inventory edit.
**Patching**: `playbooks/k3s.yml` also applies `roles/unattended_upgrades` to
every node, so updates install themselves daily with no login required — but
never reboot themselves (`Automatic-Reboot "false"`). Run
`playbooks/k3s_maintenance.yml` to actually apply a pending reboot: it checks
`/var/run/reboot-required` per node and, only where set, drains → reboots →
waits for `Ready` → uncordons, `serial: 1` so only one node is ever down.
Manual-only for now, same as `playbooks/k3s.yml` — meant to eventually run on
a schedule from a self-hosted Forgejo Actions runner rather than by hand.
**Metrics**: `playbooks/k3s.yml` also applies `roles/k3s_monitoring`, a lean
kube-prometheus-stack (no Grafana/Alertmanager) so OpenLens and similar tools
can show node/pod metrics — see "Key decisions" above for why it's a
`HelmChart` CR rather than a `helm` install, and README.md → "Metrics
(Prometheus)" for pointing OpenLens at it.
**LoadBalancer IPs**: `playbooks/k3s.yml` also applies `roles/k3s_metallb`,
MetalLB in L2 mode, so `type: LoadBalancer` Services get real LAN IPs instead
of staying `ClusterIP`-only. Set the pool it hands out from —
`k3s_metallb_address_range` in `inventory/group_vars/k3s_cluster.yml` — to a
range your DHCP scope and every static assignment in `hosts.yml` steer clear
of before the first run; see "Key decisions" above for why it replaces
ServiceLB rather than running alongside it, and README.md → "LoadBalancer
IPs (MetalLB)" for more.
There's no *node* teardown path, unlike `state: absent` for apps — removing a
Pi today means wiping its SD card and deleting its inventory entry by hand.
**Shared Postgres**: `playbooks/k3s.yml` also applies `roles/k3s_postgres`,
CloudNativePG — the cluster's own shared Postgres, a third platform for
`src/shared/postgres/` alongside Unraid and Proxmox. See "Key decisions"
above for why CloudNativePG and the topology, and README.md → "Postgres
(CloudNativePG)" for connecting to it. Per-app database provisioning works
now (`roles/k3s_app`, `db:` on a `k3s_apps:` entry) — note the two-address
asymmetry described in "Key decisions". Still **no backups**: HA replication
only, which protects against a dead SD card and not against a dropped table.
That gap matters more now that Authentik's database lives here rather than
inside whatever covers Unraid's `/mnt/user/appdata`; a CNPG `ScheduledBackup`
is the obvious next piece of work.
**TLS**: `playbooks/k3s.yml` also applies `roles/k3s_cert_manager` —
cert-manager plus one `ClusterIssuer` that every app's Ingress annotates
itself against. Two values have no default and the role refuses to run
without them: `k3s_cert_manager_acme_email` and `k3s_cert_manager_solver`
(the DNS-01 stanza, which depends on your DNS provider — a commented
Cloudflare example is in `group_vars/k3s_cluster.yml`). Use the Let's Encrypt
staging directory while working a solver out; production allows five failed
validations per hostname per hour.
**Traefik dashboard**: `playbooks/k3s.yml` also applies `roles/k3s_traefik` —
the only cluster-service role that installs nothing, since k3s installs
Traefik itself. Out of the box the dashboard is reachable only by
port-forward (`kubectl -n kube-system port-forward deploy/traefik 9000:9000`,
then `http://127.0.0.1:9000/dashboard/` — trailing slash required; without
this role a stock k3s 404s there). Publishing it on a hostname behind
Authentik is one commented-out line, `k3s_traefik_dashboard_host` in
`group_vars/k3s_cluster.yml`, plus a domain-level forward-auth Proxy Provider
on the Authentik side. See "Key decisions" above and README.md → "Dashboard
(Traefik)".
**Apps**: `k3s_apps:` in `group_vars/k3s_cluster.yml`, deployed by
`roles/k3s_app`. Authentik is the first and currently only one — it moved off
a hand-made Unraid container rather than being deployed fresh, so
`src/authentik/ansible/unraid/` is a README explaining why there's no compose
file there. The cutover procedure is `docs/authentik-migration.md`; the part
worth knowing without reading it is that the deployed `AUTHENTIK_VERSION`
must match the version a restored database was dumped from, because
Authentik's migrations run on startup and are one-way.

966
README.md Normal file
View file

@ -0,0 +1,966 @@
# Homelab IaC
Deploy apps as code across two platforms: Unraid server(s), where apps run as
Docker Compose stacks, and Proxmox, where Terraform creates an LXC and the
app is installed into it natively. Ansible is the deployment tool on both.
Secrets are pulled from HashiCorp Vault at deploy time; CI/CD is Forgejo
Actions once Forgejo itself is running.
There's also a third piece of infrastructure alongside those two: a bare-metal
k3s cluster (`homelab-utils`) across 4 Raspberry Pis, bootstrapped by Ansible
too. It doesn't fit the `src/<app>/` table above the way Unraid/Proxmox apps
do — Terraform never provisions it, and no `stacks:`/`apps:` list drives
what's on it — so it gets its own inventory group and role instead. It does
run shared services deployed via Helm, though: `k3s_monitoring`, `k3s_metallb`,
and the cluster's own shared Postgres (`k3s_postgres`, a third platform for
`src/shared/postgres/`). See `playbooks/k3s.yml` and "K3s (Raspberry Pi)"
below.
See `CLAUDE.md` for the short version of the architecture decisions.
## Layout
```
src/
<app>/
common/vars.yml config that holds on either platform
ansible/
unraid/ docker-compose.yml + .env.example + platform overrides
(+ optional icon.png for the Unraid Docker page)
proxmox/ install.yml + config/systemd templates + platform overrides
kubernetes/ vars.yml + *.yaml.j2 manifests (normally a HelmChart CR)
terraform/ this app's LXC definition (Proxmox only)
shared/<service>/ same shape, for services multiple apps use
build/config/
ansible/
inventory/
hosts.yml static: unraid_servers, proxmox_nodes, k3s_cluster
proxmox.yml dynamic: proxmox_guests, from the Proxmox API
group_vars/all.yml Vault connection, Compose Manager project path
group_vars/k3s_*.yml k3s cluster name/version, per-role vars (server/agent)
host_vars/<host>.yml which stacks/apps deploy there
(k3s apps are in group_vars/k3s_cluster.yml instead)
roles/compose_stack/ Unraid: sync compose + render .env + provision DB + up
(deploy.yml / remove.yml, picked by the entry's state:)
roles/lxc_app/ Proxmox: install natively + render config + systemd
roles/k3s_app/ k3s: render app manifests + Secret + provision DB
(deploy.yml / remove.yml, picked by the entry's state:)
roles/pve_backup/ Proxmox node: NAS backup storage + vzdump schedule
roles/k3s_node/ k3s Pi: cgroups/swap prep + install server or agent
roles/k3s_monitoring/ k3s control plane: lean kube-prometheus-stack via HelmChart CR
roles/k3s_metallb/ k3s control plane: MetalLB (LoadBalancer IPs) via HelmChart CR
roles/k3s_postgres/ k3s control plane: shared Postgres (CloudNativePG) via HelmChart CR
roles/k3s_cert_manager/ k3s control plane: cert-manager + ClusterIssuer via HelmChart CR
roles/unattended_upgrades/ any apt host: hands-off patching, no auto-reboot
roles/k3s_maintenance/ k3s Pi: drain + reboot-if-required + uncordon, one at a time
playbooks/
deploy.yml everything (Unraid + Proxmox — not k3s, see below)
unraid.yml just Unraid
proxmox.yml just the Proxmox guests
pve_host.yml just the Proxmox node (backups)
k3s.yml bootstraps the k3s cluster (manual-only, not in deploy.yml)
k3s_maintenance.yml rolls pending reboots across it, serial: 1 (manual-only)
terraform/ Proxmox provider, pg state backend, LXC module calls
.forgejo/workflows/ CI/CD (needs a self-hosted runner, see below)
docs/vault-secrets.md Vault path/key reference
docs/authentik-migration.md hand-run cutover: Authentik from Unraid to k3s
```
Terraform's scope stops at the guest — it creates the LXC and nothing more,
because the Proxmox provider has no way to install an application. Ansible
picks up from there, which is why it's the common tool and Terraform is
Proxmox-only.
## How config is layered
An app's settings are split by whether they survive a change of platform:
- `src/<app>/common/vars.yml` — version, ports, domain, database name/user.
- `src/<app>/ansible/<platform>/vars.yml` — the rest. Data paths differ
(`/mnt/user/appdata/forgejo` vs `/var/lib/forgejo`), and so does how the
database is reached (the `shared-postgres` Docker network alias only exists
on Unraid).
Both roles merge them the same way: common, then platform overrides, then
Vault secrets last. Nothing is restated between platforms.
## How a deploy works
**Unraid.** For each stack in a host's `stacks:` list, `compose_stack`:
1. Copies `src/<stack>/ansible/unraid/docker-compose.yml` to the host's
Compose Manager project folder, unmodified — it's static and only
references `${VAR}`.
2. Merges the two `vars.yml` layers with the stack's Vault KV path and
renders the project's `.env`. Nothing secret ever touches git.
3. If the stack declares a `db:` block, ensures its database/role exist on
the shared Postgres instance (idempotent — safe to add new apps to the
shared service over time without touching existing data).
4. Copies `icon.png`, if the stack ships one, to both the project folder and
`/mnt/user/appdata/icons/` (see "Icons on the Unraid Docker page" below).
5. Runs `docker compose up -d` for the stack.
**Proxmox.** `terraform apply` creates the LXC first. Then, for each app in a
guest's `apps:` list, `lxc_app` does the same config merge and database
provisioning, runs that app's `ansible/proxmox/install.yml`, and manages its
systemd unit instead of a Compose project.
**k3s.** For each app in `k3s_apps:` (`group_vars/k3s_cluster.yml`, not
host_vars — an app belongs to the cluster, not a node), `k3s_app`:
1. Does the same two-layer config merge, using
`src/<app>/ansible/kubernetes/vars.yml` as the platform layer.
2. Provisions the app's database/role on the CloudNativePG cluster if it
declares a `db:` block — the same `community.postgresql` tasks the other
two platforms use, pointed at MetalLB's LoadBalancer address for the CNPG
primary.
3. Renders the app's Vault path into a Kubernetes `Secret` (plus the
`Namespace`), keys passed through verbatim.
4. Renders every `*.yaml.j2` the app ships into
`/var/lib/rancher/k3s/server/manifests/`, where k3s's own deploy and helm
controllers pick them up.
Nothing here talks to the Kubernetes API — no `helm`, no kubeconfig, no
`kubernetes.core`. Ansible writes files; k3s reconciles them. The consequence
worth remembering: **a green run means the manifests landed, not that the
workload came up.** Check with `kubectl -n <namespace> get pods`.
Secrets land on disk in the rendered `.env` (root-readable, on the Unraid
box) or, on k3s, in a 0600 manifest and then in etcd as an ordinary
Kubernetes Secret — this is "fetch from Vault at deploy time," not a
zero-secrets-at-rest model. Fine for a homelab; revisit if that changes.
## Icons on the Unraid Docker page
There are two icons per stack, and they come from completely different places.
Commit **one** `src/<app>/ansible/unraid/icon.png` and the deploy feeds both.
A stack running more than one container can commit
`icon-<service>.png` alongside it for the extra services — see "More than one
container" below.
### The stack row
Compose Manager draws the collapsible header row for the whole project, and
serves its icon straight off disk from `icon.png` in the project folder —
`/boot/config/plugins/compose.manager/projects/<stack>/icon.png`, the same
directory the compose file and rendered `.env` land in. No label, no
template, no URL: the file is there or the row has no icon. `.jpg`, `.gif`,
`.svg` and an extensionless `icon` also work, since the plugin serves the file
itself rather than handing a path to the webgui — but stick to PNG so the same
file can serve the labels below.
This is a feature of **Compose Manager Plus** (`mstrhakr/compose_plugin`), the
maintained fork. The original Docker Compose Manager plugin (`dcflachs`,
archived April 2026) has no icon support at all — on that one the copied file
is simply ignored, so the deploy is safe either way, you just won't see it.
### The containers under it
Unraid's Docker page gets each *container's* icon, WebUI link and console
shell out of the dockerMan template that created it, under
`/boot/config/plugins/dockerMan/templates-user/`. A Compose stack never
creates one, which is why compose-deployed containers show a question mark and
no WebUI entry. Unraid 6.10+ falls back to Docker labels when there's no
template, so each service sets them itself:
```yaml
labels:
net.unraid.docker.icon: ${STACK_ICON}
net.unraid.docker.webui: "http://[IP]:${FORGEJO_HTTP_PORT}/"
net.unraid.docker.shell: sh
```
They're per **service**, not per stack — a stack with two containers labels
each one separately, and can point them at different icons.
- **icon** — `${STACK_ICON}` is set by the role, not by hand. The same
committed `icon.png` is copied a second time to
`/mnt/user/appdata/icons/<stack>.png` and the label points there; ship no
`icon.png` and the variable renders empty, leaving the placeholder. To use a
hosted image instead, set `STACK_ICON` in the app's
`ansible/unraid/vars.yml` — it's merged over the role's value.
A second copy rather than reusing the project-folder one because this label
is a path the *webgui* resolves on every Docker page load, and that path
shouldn't run through the flash drive.
**PNG only here** — unlike the stack row, this goes through the webgui's own
icon handling, where SVG renders as the question-mark fallback and WebP
renders nothing at all. A remote URL is fetched when the page renders, so an
unreachable host breaks the icon — the reason a committed file is the
default.
- **webui** — `[IP]` is substituted with the host's address. The port must be
the *published* one, so interpolate the stack's own port variable rather
than the container port. Omit the label entirely for something with no web
interface (as `shared/postgres` does) — an empty value still draws a WebUI
entry that goes nowhere.
- **shell** — `sh` or `bash`, whichever the image actually has. Alpine-based
images (Forgejo) need `sh`; Debian-based ones (Postgres, Shelfarr) can take
`bash`.
One caveat: editing a compose container's labels through the Unraid UI writes
a template for it, and the template then wins over the labels until you clear
it. Change these in the compose file and redeploy, not in the webgui.
### More than one container
`${STACK_ICON}` is a single value — the stack's `icon.png` — so a stack with
two services would otherwise have to give both the same image. Since the
labels are per-service, commit `icon-<service>.png` next to `icon.png` for
each additional container, named for the **compose service**:
```
src/arr/ansible/unraid/
icon.png → ${STACK_ICON} → /mnt/user/appdata/icons/arr.png
icon-prowlarr.png → ${STACK_ICON_PROWLARR} → /mnt/user/appdata/icons/arr-prowlarr.png
```
The variable is the service name uppercased with `-` folded to `_`
(`icon-shelfarr-libation.png` → `${STACK_ICON_SHELFARR_LIBATION}`). These get
the appdata copy only — there is one Compose Manager stack row and `icon.png`
already has it — and, like `STACK_ICON`, each can be overridden with a hosted
URL by setting the same name in the app's `ansible/unraid/vars.yml`.
The variable only exists when the file does. Interpolating
`${STACK_ICON_FOO}` with no `icon-foo.png` committed gets Compose's
unset-variable warning and the question-mark placeholder — so if the icon is
meant to be a URL, set it in `vars.yml` rather than relying on the file.
## Backups (Proxmox)
Proxmox guests keep **all** their state on their own rootfs. Nothing is
bind-mounted in from the NAS, which is the opposite of the obvious instinct —
the reason is that `vzdump` deliberately excludes bind mounts, so a
bind-mounted repository directory would be the one thing missing from the
backup that was made to protect it. Keeping everything local means one archive
is a complete copy of the guest's filesystem.
`playbooks/pve_host.yml` (role: `pve_backup`) configures the off-box copy on
the node: an NFS storage pointing at the NAS, and a scheduled `vzdump` job
writing to it. Retention is the storage's `prune-backups` settings. Point it at
a Proxmox Backup Server instead by setting `pve_backup_storage_type: pbs` in
`host_vars/<node>.yml`.
**Data that lives outside a container is that app's problem, not vzdump's.**
Forgejo's database is on the shared Postgres over on Unraid, so restoring its
container alone would give back every repository with no issues, pull requests,
users or permissions. The fix is a `pg_dump` on a systemd timer *inside* the
container, writing into its own filesystem — which puts the dump inside the
same archive as the repositories it belongs to. That only works if it finishes
before the backup window, so the two schedules are a pair:
| | Set in | Default |
|---|---|---|
| Forgejo database dump | `src/forgejo/ansible/proxmox/vars.yml` → `FORGEJO_DB_DUMP_ONCALENDAR` | 01:30 |
| vzdump job | `inventory/host_vars/pve.yml` → `pve_backup_schedule` | 02:00 |
Any future app with external state should follow the same pattern. Restore
steps are in `src/forgejo/ansible/proxmox/README.md`.
None of this covers the Unraid side, which has its own backup arrangements
outside this repo.
## Prerequisites
### Both platforms
- Ansible control node needs to be Linux — **on Windows, run it from WSL2**,
not natively.
- If this repo is checked out on a Windows mount (e.g. under `/mnt/c/...`,
which is where a Seafile-synced folder ends up) rather than the native WSL
filesystem, Ansible will warn that `ansible.cfg` is being ignored because
the directory looks world-writable — that's DrvFs reporting `777` on
everything, not a real permissions problem. Either `export
ANSIBLE_CONFIG="$(pwd)/ansible.cfg"` before running (from the
`build/config/ansible/` dir), or fix it permanently by adding to
`/etc/wsl.conf` inside WSL:
```ini
[automount]
options = "metadata,umask=22,fmask=11"
```
then `wsl --shutdown` from PowerShell and reopen the shell.
- `pip install ansible hvac psycopg2-binary`
- `ansible-galaxy collection install -r build/config/ansible/requirements.yml`
- A reachable Vault instance, with secrets populated per
`docs/vault-secrets.md`.
### Unraid
- Root SSH enabled on the Unraid box(es) (Settings → Management Access).
- Compose Manager (Docker Compose plugin) installed on Unraid — already
done.
- **Python3 on each Unraid host itself**, not just the controller: the
`compose_stack` role's tasks (`file`, `copy`, `template`,
`docker_compose_v2`) run on the target, and Ansible modules — unlike
`raw`/ad hoc shell — need a Python interpreter there to execute at all.
Stock Unraid doesn't ship one. Installing it by hand into the running
system doesn't survive a reboot, same reason as the SSH-key gotcha above —
Unraid rebuilds its root filesystem from the flash drive on every boot.
Install the **NerdTools** plugin (Community Applications → search
"NerdTools", formerly "NerdPack") and enable `python3` in its package
list — it reinstalls whatever you've selected on every boot itself, same
mechanism Compose Manager already relies on. `ansible.cfg` already sets
`interpreter_python = auto_silent`, so Ansible finds it automatically
wherever NerdTools puts it; no path to hardcode. Verify after installing:
```sh
ansible unraid_servers -m ping
```
### Proxmox
- Terraform CLI (>= 1.6) on whatever runs `terraform apply`.
- A Proxmox API token with rights to create containers, stored in Vault at
`homelab/ci/proxmox` (see `docs/vault-secrets.md`). Both the Terraform
provider and Ansible's dynamic inventory authenticate with it.
- A `terraform_state` database and `terraform` role on the **CloudNativePG
cluster on k3s** (192.168.50.81), for `backend "pg"` — not on either shared
Postgres this repo deploys, which would be circular. See "Bootstrapping"
below.
- Container templates downloaded on the Proxmox node for whatever OS the
LXCs are built from.
- An NFS export on the NAS for backup archives, reachable from the Proxmox
node — set it in `inventory/host_vars/pve.yml`. See "Backups (Proxmox)".
- Root SSH from the Ansible controller to the Proxmox node itself, not just
the guests: `playbooks/pve_host.yml` configures the node over SSH.
- Unlike Unraid, nothing special is needed *inside* the guests: they're
ordinary Linux containers with Python already present, so Ansible works
without the NerdTools workaround above.
### K3s (Raspberry Pi)
- 4 Raspberry Pis running Ubuntu Server (64-bit), already imaged and on the
network at the addresses in `inventory/hosts.yml` → `k3s_cluster`.
- An `ansible` user on each Pi with **NOPASSWD sudo** and this repo's k3s SSH
key installed as an authorized key — see "SSH access" below. Not automated
by this repo; create it by hand (or via cloud-init at image time) before the
first run of `playbooks/k3s.yml`.
- `homelab/k3s-homelab-utils` → `K3S_TOKEN` set in Vault *before* the first
run — see `docs/vault-secrets.md`. Both the server and every agent read
this same fixed value rather than one generating it and handing it to the
other, which is what makes a rebuild reproducible.
- Nothing else special: `python3` ships with Ubuntu Server, and the k3s
install script (`get.k3s.io`) handles containerd, the systemd unit, and
everything else that isn't Pi-specific. The one Pi-specific thing —
ensuring the memory cgroup controller is on — is handled by
`roles/k3s_node` itself, not a prerequisite here.
#### Keeping it patched
`playbooks/k3s.yml` also applies `roles/unattended_upgrades` to every node —
apt updates install themselves on their own daily schedule, no login
required. It sets `Unattended-Upgrade::Automatic-Reboot "false"` though, so a
kernel or containerd update that needs a reboot to take effect just sits
applied-but-inactive until one happens; blindly auto-rebooting a k3s node
takes its pods down with no warning.
`playbooks/k3s_maintenance.yml` is the other half: it checks
`/var/run/reboot-required` on each node and, only where it's set, cordons and
drains the node, reboots it, waits for it to report `Ready` again, then
uncordons it — one node at a time (`serial: 1`), so patching the cluster never
means dropping every workload at once. Manual-only for now, same as
`playbooks/k3s.yml`:
```sh
ansible-playbook playbooks/k3s_maintenance.yml
```
Eventually this is meant to run on a schedule from a self-hosted Forgejo
Actions runner rather than by hand — not wired up yet, see the header comment
in the playbook.
#### Metrics (Prometheus)
`playbooks/k3s.yml` also deploys `roles/k3s_monitoring`: a lean
kube-prometheus-stack (Prometheus + prometheus-operator + node-exporter +
kube-state-metrics — no Grafana, no Alertmanager) so tools like OpenLens can
show node/pod metrics. It's installed as a `HelmChart` custom resource
dropped into k3s's own auto-deploying manifests directory
(`/var/lib/rancher/k3s/server/manifests/`) rather than run through a `helm`
binary — k3s's bundled helm-controller reconciles it the same way it
installs its own Traefik and ServiceLB, so this needs no extra Ansible
collection or kubeconfig on the controller. See
`roles/k3s_monitoring/defaults/main.yml` for the resource sizing (tuned for
a Raspberry Pi 4, not a real node) and why the control-plane component
scrapers (`kubeControllerManager`, `kubeScheduler`, `kubeProxy`, `kubeEtcd`)
are disabled — k3s doesn't expose those the way kube-prometheus-stack
expects, so leaving them on just produces permanently-"down" targets.
To point OpenLens at it: open the cluster's Settings → Metrics, set
Prometheus to "Auto detect" or explicitly to the `Operator` provider — it
should find the `kube-prometheus-stack-prometheus` service in the
`monitoring` namespace via the API server proxy, the same path OpenLens
already uses to reach the cluster, so nothing needs to be exposed outside
it.
#### LoadBalancer IPs (MetalLB)
`playbooks/k3s.yml` also deploys `roles/k3s_metallb`: MetalLB in L2 mode,
handing out real LAN IPs to `type: LoadBalancer` Services instead of the
`ClusterIP`-only world k3s would otherwise leave homelab-utils apps in. It's
installed the same way as monitoring — a `HelmChart` CR for k3s's
helm-controller to reconcile — plus a plain `IPAddressPool`/`L2Advertisement`
manifest dropped in the same directory; k3s's deploy controller applies both
kinds of file and retries the config manifest until the HelmChart's CRDs
exist.
MetalLB replaces k3s's bundled ServiceLB (Klipper), it doesn't run alongside
it — both would otherwise compete to satisfy the same Services. That's why
`inventory/group_vars/k3s_cluster.yml` sets `k3s_server_extra_args:
['--disable=servicelb']` — a separate, server-only var from `k3s_extra_args`,
since `k3s agent` doesn't understand `--disable` and would fail to start if
it were passed there too. The IP pool itself is
`k3s_metallb_address_range` in the same file — a range on the cluster's LAN
(`192.168.50.0/24`) that DHCP and every static assignment in
`inventory/hosts.yml` steer clear of. There's no built-in default: the role
fails fast if it's still empty.
Enabling `--disable=servicelb` on an already-running cluster doesn't need a
separate step: `roles/k3s_node` notices its exec line has changed and
reinstalls (restarting just the `k3s` service, not the node) the next time
`playbooks/k3s.yml` runs — see `CLAUDE.md` → "Key decisions".
#### Postgres (CloudNativePG)
`playbooks/k3s.yml` also deploys `roles/k3s_postgres`: CloudNativePG (CNPG),
a Postgres operator, installed the same way as monitoring and MetalLB — a
`HelmChart` CR for k3s's helm-controller to reconcile — plus a plain
`Cluster` CR (the actual database) and a `Secret` (superuser credentials),
dropped in the same auto-deploying manifests directory. This is a third
platform for `src/shared/postgres/`, alongside the Unraid Compose stack and
the Proxmox placeholder: `roles/k3s_postgres` layers
`src/shared/postgres/common/vars.yml` with
`src/shared/postgres/ansible/kubernetes/vars.yml` the same way
`compose_stack`/`lxc_app` layer an app's config, and pulls the superuser
password from the same `homelab/shared/postgres` Vault path those platforms
already use (see `docs/vault-secrets.md`) — a separate physical instance,
not a shared login across platforms, just the same identity.
1 primary + 1 replica, kept off the control-plane Pi via `nodeAffinity` (see
`roles/k3s_postgres/templates/postgres-cluster.yaml.j2`) — CNPG's own pod
anti-affinity then spreads the two across the 3 worker Pis. Storage is k3s's
default `local-path` StorageClass; that's node-local, but resilience here
comes from CNPG's own streaming replication between instances, not from
shared storage, so losing one instance's disk doesn't lose the data as long
as the other instance is up. HA replication only for now — no
`ScheduledBackup` to the NAS or anywhere else yet.
To connect: `kubectl -n shared-postgres get svc` lists four Services. Three
are CNPG's own, all `ClusterIP` — `shared-postgres-rw` (the current primary),
`-ro` (replicas only) and `-r` (any instance). An app running on the cluster
uses the first of those by DNS and needs nothing else:
```
shared-postgres-rw.shared-postgres.svc.cluster.local:5432
```
The fourth, `shared-postgres-lb`, is a `LoadBalancer` on
`k3s_postgres_loadbalancer_ip` (`inventory/group_vars/k3s_cluster.yml` —
inside the MetalLB pool above), which is how the cluster is reached from
outside it:
```sh
psql -h 192.168.50.81 -U postgres # password: homelab/shared/postgres
```
It's declared in the `Cluster` CR as a CNPG *managed service* rather than as
a Service manifest of our own, so the operator keeps its selector pointed at
whichever instance is currently primary — a hand-written Service would need
re-pointing by hand after a failover. Credentials are the
`shared-postgres-superuser-vault` Secret Ansible renders; CNPG separately
auto-creates `shared-postgres-app` for the default `app` database it
bootstraps, which nothing uses yet.
The `-vault` suffix keeps that Secret clear of `<cluster>-superuser`, the
name CNPG uses for the superuser secret it generates itself. Name our own
object that and the operator treats it as one it already authored, so a
rotated password reaches the Secret and never reaches the database — the
Secret reads correctly, `psql` from the LAN keeps failing authentication, and
only `select rolpassword is null from pg_authid where rolname='postgres'`
inside the pod shows the disagreement.
That LoadBalancer is what makes the controller-reachable `host:port` the
`community.postgresql` tasks need — the same ones `compose_stack` and
`lxc_app` use to create an app's database and role — available here too, and
`roles/k3s_app` now uses exactly those tasks for any app declaring a `db:`
block.
Note the deliberate asymmetry, which reads like a mistake until you know why:
provisioning uses `192.168.50.81` (the LoadBalancer) because it runs on the
Ansible controller, which is off-cluster and can't route to a ClusterIP;
apps use `shared-postgres-rw.shared-postgres.svc.cluster.local` because
sending pod traffic out to the LAN and back would put MetalLB's L2 speaker in
the path of every query for nothing. Two addresses, one database, each
correct for its caller.
Without `kubectl` to hand, or if MetalLB is having a bad day, the in-cluster
routes still work:
```sh
kubectl -n shared-postgres port-forward svc/shared-postgres-rw 5432:5432
kubectl -n shared-postgres exec -it shared-postgres-1 -- psql -U postgres
```
**No backups.** HA replication only — a streaming replica on another Pi,
which covers a dead SD card and not a dropped table, a bad migration, or a
mistyped `DROP`. This mattered less when the cluster held nothing; it matters
now that Authentik's database lives here rather than on Unraid. A CNPG
`ScheduledBackup` to the NAS is the obvious next piece of work and isn't
built yet.
#### TLS (cert-manager)
`playbooks/k3s.yml` also applies `roles/k3s_cert_manager`: cert-manager plus
one `ClusterIssuer` named `letsencrypt`, both via the usual `HelmChart` CR +
plain manifest pair. An app's Ingress opts in with a single annotation —
```yaml
annotations:
cert-manager.io/cluster-issuer: letsencrypt
```
— and cert-manager creates and renews the certificate on its own.
Two values have no default and the role refuses to run without them, both in
`inventory/group_vars/k3s_cluster.yml`:
- `k3s_cert_manager_acme_email` — an ACME account is registered against it.
- `k3s_cert_manager_solver` — the DNS-01 stanza for your DNS provider,
rendered into the issuer as-is. A commented Cloudflare example is in that
file; for anything else take the stanza from
[cert-manager's docs](https://cert-manager.io/docs/configuration/acme/dns01/).
Its API token goes in Vault at `homelab/k3s-cert-manager`.
DNS-01 rather than HTTP-01 because HTTP-01 needs Let's Encrypt to reach this
cluster from the internet on port 80, which it can't. That has a useful
consequence: a certificate can be issued **before** DNS points at the
cluster, so a migration's TLS is settled before its cutover rather than
after.
While working out a solver, point `k3s_cert_manager_acme_server` at Let's
Encrypt staging — production allows five failed validations per hostname per
hour, and exhausting it means waiting, not retrying. Check it registered:
```sh
kubectl get clusterissuer letsencrypt -o jsonpath='{.status.conditions[*].message}'
```
#### Dashboard (Traefik)
Traefik is the one chart here this repo doesn't install — k3s installs it
itself. `roles/k3s_traefik` only adjusts it, through a `HelmChartConfig`
merged over k3s's own `HelmChart`. Editing
`/var/lib/rancher/k3s/server/manifests/traefik.yaml` on the node instead
works until the next server restart rewrites it.
A stock k3s answers **404** on the dashboard, which looks like a broken
install and isn't: Traefik still builds the dashboard, but the Traefik chart
stopped creating the router that reaches it in v28. The role puts that router
back on Traefik's internal `traefik` entrypoint, which isn't published on the
Service — so it's reachable by port-forward and nothing else:
```sh
kubectl -n kube-system port-forward deploy/traefik 9000:9000
# then http://127.0.0.1:9000/dashboard/ — the trailing slash is required
```
Two things to check if that still 404s. `curl -i http://127.0.0.1:9000/ping`
returning 200 means the port-forward is fine and only the router is missing
(so the role hasn't run, or its `HelmChartConfig` hasn't reconciled yet —
`kubectl -n kube-system get ingressroute` should list `traefik-dashboard`).
And forward to `deploy/traefik`, not `svc/traefik`: the Service only publishes
80/443, so going through it lands you on the `web` entrypoint, where an
unmatched request also returns 404.
**Publishing it on a hostname** is one commented line in
`inventory/group_vars/k3s_cluster.yml`:
```yaml
k3s_traefik_dashboard_host: traefik.turtlesystems.uk
```
That renders an `IngressRoute` on `websecure` with a cert-manager certificate
and an Authentik forward-auth middleware in front. `k3s_traefik_dashboard_auth_address`
must be set too — the role refuses to run otherwise rather than publishing an
unauthenticated view of every router, service and middleware on the cluster.
It's already set in that file, so in practice this is the single line above
plus a DNS record pointing at Traefik's MetalLB address
(`kubectl -n kube-system get svc traefik`).
On the Authentik side, first: a Proxy Provider in **forward auth (domain
level)** mode, assigned to an application, added to the embedded outpost.
Domain level rather than single-application because the latter needs
`/outpost.goauthentik.io/` routed to Authentik on the dashboard's own
hostname, which from `kube-system` is a cross-namespace service reference
Traefik rejects by default.
An `IngressRoute` rather than an `Ingress`, unlike every app here, because
the dashboard is served by Traefik's internal `api@internal` and has no
Kubernetes Service for an `Ingress` to point at. That's also why its
certificate is an explicit `Certificate` resource — cert-manager watches
`Ingress`, not `IngressRoute`.
Clearing the hostname again is a real teardown: the role deletes the
manifest, and k3s's deploy controller garbage-collects the route, middleware
and certificate it created.
The port-forward stays enabled alongside the published route on purpose. Once
the dashboard is behind Authentik it depends on Authentik, which depends on
CNPG, which depends on a healthy cluster — the things you'd open the
dashboard to diagnose. Port-forward talks to the pod and traverses none of
it, so it's the break-glass path, not a leftover.
#### Apps on the cluster
Apps go in `k3s_apps:` in `inventory/group_vars/k3s_cluster.yml` — the k3s
equivalent of a host's `stacks:`/`apps:` list, in group_vars because an app
is deployed to the cluster rather than to a node. Same entry shape as the
other platforms (name, src, vault_path, optional `db:`, optional `state:`).
```sh
ansible-playbook playbooks/k3s.yml --tags apps
ansible-playbook playbooks/k3s.yml --tags apps -e only_apps=authentik
```
Ingress goes through the Traefik k3s already bundles, on the MetalLB address
its Service holds (`kubectl -n kube-system get svc traefik`) — so adding an
app costs a DNS record pointed at that one address, not a pool IP each.
Removal is `state: absent` on the entry, same tombstone rule as Unraid: keep
the entry, don't delete it, or nothing is torn down and the workload keeps
running unmanaged. Unlike Unraid it's a single pass — deleting the manifests
*is* the teardown, because k3s's deploy controller garbage-collects what each
file created — so `src/<app>/` can go in the same commit. The database, PVCs
and Vault path deliberately survive.
**Authentik** is currently the only app, and it got here by migrating off a
hand-made Unraid container rather than being deployed fresh. If you're
repeating that for something else, `docs/authentik-migration.md` is the
worked procedure; the part worth knowing up front is that the deployed
version must match the version a restored database was dumped from, because
Authentik runs its migrations on startup and they don't go backwards.
## SSH access
Ansible connects as `ansible_user: root` (set in
`build/config/ansible/inventory/hosts.yml`) over SSH. Password auth would mean either an interactive prompt every run —
which doesn't work from CI at all — or a plaintext password somewhere, so
use a dedicated key pair instead:
```sh
ssh-keygen -t ed25519 -f ~/.ssh/unraid_ansible -C "ansible@unraid-iac" -N ""
```
Install the public half as an authorized key on **every** Unraid host this
repo targets. Don't just `ssh-copy-id` it into `~/.ssh/authorized_keys` —
Unraid boots from the flash drive and `/root` lives on a RAM-backed overlay,
so anything written there is gone on the next reboot. Persist it through
`/boot/config/ssh/root.authorized_keys` instead, which Unraid copies into
place at every boot:
```sh
cat ~/.ssh/unraid_ansible.pub | ssh root@<unraid-host> \
'cat >> /boot/config/ssh/root.authorized_keys'
```
For manual/bootstrap runs, load the key into `ssh-agent` and Ansible picks it
up automatically — no config file changes needed:
```sh
eval "$(ssh-agent -s)"
ssh-add ~/.ssh/unraid_ansible
```
For CI, rather than adding yet another place secrets live, the private key
is stored in Vault (`homelab/ci/ssh`, see `docs/vault-secrets.md`) and the
workflow fetches it at the start of each run:
```sh
vault kv put kv/homelab/ci/ssh PRIVATE_KEY=@~/.ssh/unraid_ansible
```
**K3s Pis** work the same way but as a non-root `ansible` user, not root, and
with their own key pair (`~/.ssh/k3s_ansible`, Vault path
`homelab/ci/ssh-k3s`) — see `docs/vault-secrets.md` for the full commands.
Three differences from the Unraid steps above. The public key goes in the
ordinary `~/.ssh/authorized_keys` for the `ansible` user (Ubuntu's root
filesystem isn't rebuilt from flash on every boot, so there's no persistence
quirk to work around), and that user needs NOPASSWD sudo configured in
`/etc/sudoers.d/` — `playbooks/k3s.yml` installs k3s via `become: true`.
And the private half doesn't need loading into `ssh-agent` at all, even for a
manual run: `playbooks/k3s_ssh_key.yml` fetches it from Vault and writes it to
`local/k3s/homelab-utils.key`, and both `playbooks/k3s.yml` and
`playbooks/k3s_maintenance.yml` import that as their first play. So a k3s run
needs `VAULT_ADDR`/`VAULT_TOKEN` in the environment and nothing else — the
same "Vault is the only place secrets live" arrangement CI already uses for
the Unraid key, rather than a second setup step to remember. To fall back to a
local copy (Vault down, say), skip the fetch **and** point at the key —
overriding the path alone isn't enough, since the fetch play would still fail
before anything else ran:
```sh
ansible-playbook playbooks/k3s.yml \
-e k3s_ssh_key_fetch=false \
-e ansible_ssh_private_key_file=~/.ssh/k3s_ansible
```
## Bootstrapping (chicken-and-egg on the very first deploy)
CI/CD runs on Forgejo Actions — but Forgejo doesn't exist yet on the first
run, and Actions needs a self-hosted runner besides. So the first deploy is
manual:
1. Fill in `build/config/ansible/inventory/hosts.yml` with nas2's real
address.
2. Set up the SSH key pair per "SSH access" above and install the public key
on nas2 (and nas1, once you migrate anything to it).
3. `vault kv put` the secrets in `docs/vault-secrets.md` for
`homelab/shared/postgres`, `homelab/forgejo`, and `homelab/ci/ssh`.
4. From WSL, with `VAULT_ADDR` / `VAULT_TOKEN` exported (`vault login` first)
and the key loaded in `ssh-agent`:
```sh
cd build/config/ansible
ansible-playbook playbooks/unraid.yml --limit nas2
```
This brings up shared Postgres, provisions the `forgejo` database/role,
and brings up Forgejo. Use `unraid.yml` rather than `deploy.yml` at this
point — no Proxmox guests exist yet, so the Proxmox play has nothing to
target.
5. Log into Forgejo, create this repo there, push it.
6. Register a self-hosted Forgejo Actions runner reachable to Vault and to
both Unraid hosts (labelled `unraid-deploy` — see
`.forgejo/workflows/deploy.yml`). Give it an AppRole (`VAULT_ROLE_ID` /
`VAULT_SECRET_ID`) scoped to read the `kv/homelab/*` paths it needs
(including `homelab/ci/ssh`), stored as Forgejo Actions secrets alongside
`VAULT_ADDR`.
7. From then on, pushes to `main` touching `src/**` or `build/config/ansible/**`
deploy automatically — and only the stack(s) whose `src/<stack>/` folder
actually changed (see "Deploying one platform, one host, one stack"
below); a change under `build/config/ansible/` still triggers a full
deploy of every stack on every host, since that's a change to how *all* of
them get deployed.
### Bringing Proxmox online later
The shared Postgres and its LXC are the first thing to stand up here, and the
ordering is fiddly enough to have its own runbook: **`docs/postgres-proxmox.md`**.
The short version, and the reason it isn't just "run Terraform":
Terraform stores its state in Postgres, so provisioning the Postgres LXC with
Terraform would be circular. It's broken by keeping state on the
CloudNativePG cluster on the k3s Pis instead — something this configuration
has no hand in building — which means `terraform apply` now depends on the
cluster being up. That's a cross-platform dependency the rest of the repo
avoids, and it's worth it because the alternative is a local-state-then-migrate
dance that has to be got right exactly once.
After that, `docs/postgres-proxmox.md` walks through the template download,
`terraform apply -target=module.postgres`, checking the ZFS replication job
actually exists, installing Postgres with
`ansible-playbook playbooks/proxmox.yml -e only_stacks=postgres`, and
`playbooks/pve_host.yml` for node backups. It also covers failing over between
the two nodes and repointing clients afterwards.
Forgejo comes after all of that, on **turtle-proxmox-02** — its database is
the shared Postgres the runbook above creates, so none of it works until that
one is finished. `docs/forgejo-proxmox.md` walks through the extra Vault key
the native install needs, `terraform apply -target=module.forgejo`,
`ansible-playbook playbooks/proxmox.yml -e only_stacks=forgejo`, and the parts
Ansible deliberately does not do — moving repository data, and the DNS/SSH-port
cutover.
## Deploying one platform, one host, one stack
`playbooks/deploy.yml` runs everything; `playbooks/unraid.yml`,
`playbooks/proxmox.yml` and `playbooks/pve_host.yml` do one slice each
(Unraid stacks, Proxmox guests, and the Proxmox node's backup config
respectively). Within any of them, `--limit <host>`
narrows by host and `-e only_stacks=<name>[,<name>...]` narrows by stack or
app (name = the `name:` in the host's `stacks:`/`apps:` list, e.g. `forgejo`
or `postgres` — not the `src:` path):
```sh
# everything, both platforms
ansible-playbook playbooks/deploy.yml
# everything declared for nas2
ansible-playbook playbooks/unraid.yml --limit nas2
# just forgejo on nas2
ansible-playbook playbooks/unraid.yml --limit nas2 -e only_stacks=forgejo
# forgejo and its database, skipping any other stack on nas2
ansible-playbook playbooks/unraid.yml --limit nas2 -e only_stacks=forgejo,postgres
```
Note `only_stacks` doesn't resolve dependencies for you — if you limit to
`forgejo` before `postgres` has ever been deployed, the DB provisioning step
will fail because there's nothing to connect to. CI doesn't hit this: it
computes `only_stacks` from which `src/**` paths actually changed in the
push, so an unrelated app's push never touches Postgres or Forgejo.
`deploy.yml` runs Unraid before Proxmox by convention rather than necessity —
the two platforms no longer depend on each other. Proxmox-side apps provision
their databases against the Proxmox shared Postgres (192.168.50.54), and
Terraform keeps its state on the CloudNativePG cluster on k3s; neither goes
through Unraid. The node's backup config goes last, since a vzdump job pinned
to specific VMIDs needs those guests to exist. Note that `only_stacks` has no
meaning for `pve_host.yml` — it configures a node, not an app, and ignores it.
`playbooks/k3s.yml` is separate from all of the above — it isn't included in
`deploy.yml` and doesn't take `only_stacks`, because a cluster service there
isn't a `stacks:` entry to filter down to. It has two selectors of its own
instead: `--limit` narrows *which nodes*, and `--tags` narrows *which piece*.
```sh
# the whole cluster: nodes, then the cluster services, then the kubeconfig
ansible-playbook playbooks/k3s.yml
# re-converge one worker, e.g. after reimaging it
ansible-playbook playbooks/k3s.yml --limit k3s-node-02
# just one cluster service, no node install across 4 Pis
ansible-playbook playbooks/k3s.yml --tags postgres
# every cluster service, still no node install
ansible-playbook playbooks/k3s.yml --tags services
```
The tags, one per play (see the playbook's header comment): `nodes` (both
node plays), `metallb`, `monitoring`, `postgres`, `cert-manager`, `traefik`,
`services` (all five of those together), `apps`, `upgrades`, `kubeconfig`.
`services` and `apps` don't imply each other, so "everything except
reinstalling k3s" is `--tags services,apps`. The SSH-key fetch play is tagged
`always` rather than named, so it survives every filter — it's what any of
the others connect with, not something you'd pick.
Deploying a single service this way skips the `kubeconfig` play too, so
`local/`'s copy isn't refreshed. That only matters on a cluster rebuild;
add `--tags postgres,kubeconfig` if you want both.
Worth knowing what a tagged run does and doesn't tell you: every service role
just templates manifests into `/var/lib/rancher/k3s/server/manifests/`
for k3s's own controllers to reconcile, so the playbook finishing means the
files landed, not that the workload is up. Watch that separately, e.g.
`kubectl -n shared-postgres get cluster,pods -w`.
`playbooks/k3s_maintenance.yml` is separate again — see "Keeping it patched"
above. It ignores `--limit` grouping in one sense worth knowing: `serial: 1`
still applies to whatever `--limit` narrows the run to, so limiting to two
nodes still patches them one at a time, not together.
## Adding a new app
1. `src/<app>/common/vars.yml` — `env_defaults:` for the config that doesn't
depend on where it runs.
2. Build the platform side(s) you need:
**Unraid** — `src/<app>/ansible/unraid/` containing
`docker-compose.yml` (static, `${VAR}`-driven, joining `unraid_shared`
with `external: true` if it needs the shared Postgres), `vars.yml` of
overrides, and `.env.example` documenting every var with secrets blank.
Optionally an `icon.png` and the `net.unraid.docker.*` labels, so the
Docker page shows something other than a question mark.
**Proxmox** — `src/<app>/ansible/proxmox/` containing `vars.yml`,
`install.yml`, and templates for the app's config file and systemd unit;
plus `src/<app>/terraform/` defining its LXC, called from
`build/config/terraform/main.tf`.
3. Add its `homelab/<app>` path to Vault (`docs/vault-secrets.md`).
4. Declare it on the target host — `stacks:` in an Unraid host's
`host_vars/<host>.yml`, or `apps:` in a Proxmox guest's. Same entry shape
either way (name, src, vault_path, optional `db:` block).
5. Push — or run the relevant playbook manually before CI exists.
6. Leave the platform you didn't build as a README placeholder rather than
deleting the folder.
## Removing an app (Unraid)
**Deleting the entry from `stacks:` does not remove anything.** Ansible keeps
no record of what it deployed last run, so an app that disappears from the
list is simply never visited again — its containers keep running on the host,
now unmanaged and invisible to the playbook. This is the one place the Unraid
side doesn't behave like the Terraform side, where deleting a module call does
destroy the LXC.
Removal is therefore an instruction, not an absence. Set `state: absent` on
the entry and leave it in place:
```yaml
stacks:
- name: arr
src: arr
vault_path: homelab/arr
state: absent
```
The next run of `playbooks/unraid.yml` runs `docker compose down`, then
deletes the project folder (and with it the rendered `.env`). Because a
`host_vars/` edit is a change under `build/config/ansible/`, CI treats it as a
full converge — so pushing that change is enough to action the teardown.
Keep the entry as a tombstone until you're sure. It's the only record that
the app was deliberately removed rather than never deployed, and flipping
`state:` back to `present` redeploys it.
### What survives, and how to remove the rest
Bind mounts are untouched, so `/mnt/user/appdata/<app>` — which for a
SQLite-backed app like Shelfarr *is* the application — survives a teardown and
makes it reversible. External networks (`caddy-net`, `unraid_shared`) are left
alone too; they belong to Unraid or to another stack. Both copies of the
stack's icon *are* deleted without needing to be asked — the project-folder
one goes with the folder, and the one under `/mnt/user/appdata/icons/` is
removed explicitly. Neither is state; they're copies of a file in the repo,
and a redeploy puts them back.
Three things are destructive enough to stay opt-in, per entry:
| Key | Effect |
|---|---|
| `remove_volumes: true` | Also delete the project's named volumes. Bind mounts are unaffected either way. |
| `remove_images: local` | Also delete its images (`local` or `all`, as `docker compose down --rmi`). |
| `remove_database: true` | Drop the Postgres database **and** its role. Only for stacks with a `db:` block. |
Nothing removes the app's `homelab/<app>` path from Vault — that has its own
lifecycle, and destroying it would make the teardown irreversible. Delete it
by hand with `vault kv metadata delete` once you're finished with the app.
### Order of operations
`docker compose down` reads the compose file to know what it's removing, so
the teardown has to run **before** you delete `src/<app>/` or the host's
project folder. Delete those first and there's nothing left to tell Docker
what belonged to the project — you're cleaning up by hand with `docker rm` and
`docker network rm` instead. So: set `state: absent`, run the playbook, then
delete files in a second pass.
There's no equivalent on the Proxmox side yet. `lxc_app` only ever installs,
and a symmetric teardown needs a per-app `uninstall.yml` convention that no
app implements — the role is still a skeleton. For now, removing a Proxmox app
means stopping and disabling its systemd unit by hand, or destroying the
container via Terraform.
## Adding a host
**Unraid:** add it under `unraid_servers` in
`build/config/ansible/inventory/hosts.yml`, then create
`build/config/ansible/inventory/host_vars/<name>.yml` with its own `stacks:`
list — each host only runs what it's assigned.
**Proxmox:** define the LXC as a module in `src/<app>/terraform/`, call it
from `build/config/terraform/main.tf`, and `terraform apply`. The guest
appears in inventory automatically via the API — no `hosts.yml` edit — but
it still needs a `host_vars/<guest>.yml` declaring its `apps:` list.
**K3s:** add it as a new host under `k3s_control_plane` or `k3s_workers` (per
`inventory/hosts.yml` → `k3s_cluster`) — no `terraform apply`, the Pi has to
physically exist and have SSH access set up first (see "K3s (Raspberry Pi)"
above). No `host_vars/` entry needed beyond that: `roles/k3s_node` reads
`k3s_node_role` off the group, not the host, so which group a new Pi joins is
the only thing that decides whether it becomes another agent or a second
control-plane node — and this repo's role doesn't support the latter (single
server, no HA etcd) without changes to `k3s_node/tasks/server.yml`.

18
build/config/ansible/ansible.cfg Normal file
View file

@ -0,0 +1,18 @@
[defaults]
# A directory, not a single file: static Unraid hosts (hosts.yml) and the
# Proxmox API-backed dynamic inventory (proxmox.yml) are both sources.
inventory = inventory/
roles_path = roles
host_key_checking = False
retry_files_enabled = False
interpreter_python = auto_silent
[inventory]
# `auto` must come before `yaml`: it reads the `plugin:` key in
# inventory/proxmox.yml and dispatches to the named plugin. With `yaml`
# first, that file gets parsed as a static host list instead, and `plugin:`
# is read as a group name.
enable_plugins = auto, yaml, ini, community.proxmox.proxmox
[ssh_connection]
pipelining = True

18
build/config/ansible/inventory/group_vars/all.yml Normal file
View file

@ -0,0 +1,18 @@
---
# HashiCorp Vault connection. VAULT_ADDR/VAULT_TOKEN (or VAULT_ROLE_ID /
# VAULT_SECRET_ID for AppRole) are read from the environment of whatever is
# running ansible-playbook — a local shell during bootstrap, or Forgejo
# Actions secrets once CI is wired up. Never hardcode a token here.
vault_addr: "{{ lookup('env', 'VAULT_ADDR') }}"
vault_kv_mount: kv
# AppRole auth (used by CI). Leave unset for local token-based auth
# (`vault login` + VAULT_TOKEN env var) during bootstrap.
vault_auth_method: "{{ lookup('env', 'VAULT_AUTH_METHOD') | default('token', true) }}"
vault_role_id: "{{ lookup('env', 'VAULT_ROLE_ID') | default(omit) }}"
vault_secret_id: "{{ lookup('env', 'VAULT_SECRET_ID') | default(omit) }}"
# Where Compose Manager (the Unraid Docker Compose plugin) expects project
# folders. Verify this against your plugin version — override per-host in
# host_vars if yours differs.
compose_projects_root: /boot/config/plugins/compose.manager/projects

205
build/config/ansible/inventory/group_vars/k3s_cluster.yml Normal file
View file

@ -0,0 +1,205 @@
---
# Shared config for the whole k3s_cluster group (both k3s_control_plane and
# k3s_workers) — see roles/k3s_node and playbooks/k3s.yml.
# Used as the kubeconfig context/cluster name and as the Vault path suffix
# below, so the cluster has one name in every place it shows up.
k3s_cluster_name: homelab-utils
# Pinned, not "latest" — a rebuild months from now should produce the same
# cluster it does today, same reasoning as pinning an app's image tag in
# common/vars.yml. Bump deliberately; check the current stable release at
# https://github.com/k3s-io/k3s/releases first.
k3s_version: v1.31.4+k3s1
# One Vault path for the whole cluster, same "one path per stack" convention
# as everything under homelab/<app> — see docs/vault-secrets.md.
k3s_vault_path: "homelab/k3s-{{ k3s_cluster_name }}"
# The SSH key this group is reached with. A different path from k3s_vault_path
# above because it's a different kind of secret: that one is cluster identity
# (the join token), this one is access to the hosts, and it lives alongside
# the Unraid key under homelab/ci/ rather than with the cluster's own data.
# Fetched by playbooks/k3s_ssh_key.yml, which both k3s playbooks import.
k3s_ssh_key_vault_path: homelab/ci/ssh-k3s
# Where that play writes it, and what every play here then connects with — so
# a manual run needs Vault credentials in the environment and nothing else,
# rather than the key pre-loaded into ssh-agent. Override on the command line
# (-e ansible_ssh_private_key_file=~/.ssh/k3s_ansible) to use a local copy
# instead, e.g. if Vault itself is down.
#
# Under local/, which .gitignore already excludes — same as the kubeconfig
# below, and for the same reason: generated credential, never committed. Built
# from playbook_dir rather than repo_root, unlike k3s_kubeconfig_local_path:
# repo_root is a *play* var set in k3s.yml, so it only resolves inside the two
# plays that set it, whereas this is a connection var and has to resolve in
# every play in the file.
k3s_ssh_key_local_path: "{{ playbook_dir }}/../../../../local/k3s/{{ k3s_cluster_name }}.key"
ansible_ssh_private_key_file: "{{ k3s_ssh_key_local_path }}"
# Extra INSTALL_K3S_EXEC args appended on every node, both roles. Empty by
# default. Only flags valid on *both* `k3s server` and `k3s agent` belong
# here — see k3s_server_extra_args below for server-only flags like
# `--disable`, which `k3s agent` doesn't understand and will fail to start
# with.
k3s_extra_args: []
# Extra INSTALL_K3S_EXEC args appended on the control-plane node only.
# --disable=servicelb turns off k3s's bundled LoadBalancer controller
# (Klipper) in favour of roles/k3s_metallb — the two would otherwise both
# try to satisfy the same LoadBalancer Services. Add e.g. '--disable=traefik'
# here too if homelab-utils apps end up wanting their own ingress instead.
#
# Changing this (or k3s_extra_args) on an already-installed cluster is picked
# up on the next `ansible-playbook playbooks/k3s.yml` run — roles/k3s_node
# tracks the exec line it last installed with and reinstalls (restarting the
# k3s/k3s-agent service, not the node) whenever it drifts from what's
# configured here. See tasks/server.yml and tasks/agent.yml.
k3s_server_extra_args:
- "--disable=servicelb"
# MetalLB's LoadBalancer IP pool — see roles/k3s_metallb. Must be addresses
# on the LAN (192.168.50.0/24 — see inventory/hosts.yml) that nothing else,
# DHCP included, will ever hand out. Adjust to a range your DHCP scope
# doesn't cover before the first run.
k3s_metallb_address_range: "192.168.50.80-192.168.50.99"
# The LAN address the shared Postgres primary answers on — see
# roles/k3s_postgres, which declares a LoadBalancer Service for it alongside
# CNPG's built-in ClusterIP ones. Pinned rather than auto-assigned because
# the address gets written down elsewhere (an app's vars.yml, a connection
# string in Vault) instead of only being looked up at runtime, so it mustn't
# move when the Service is recreated.
#
# Must be inside the pool above — MetalLB only assigns from its own pools,
# and a request for an address outside them leaves the Service pending
# forever rather than failing loudly. The caveat of pinning out of the same
# pool MetalLB auto-assigns from: it hands unpinned Services the lowest free
# address, so if some other LoadBalancer Service is created before this one
# and takes .80, this Service is the one left pending. Nothing else here
# requests an IP today. If that changes, the fix is a second
# IPAddressPool with autoAssign: false in roles/k3s_metallb reserved for
# pinned addresses, not a different number here.
k3s_postgres_loadbalancer_ip: "192.168.50.81"
# Where playbooks/k3s.yml fetches the kubeconfig to, on the Ansible
# controller. Under local/, which .gitignore already excludes — it's a
# generated credential, never committed.
k3s_kubeconfig_local_path: "{{ repo_root }}/local/k3s/{{ k3s_cluster_name }}.kubeconfig"
# --- cert-manager (roles/k3s_cert_manager) ----------------------------------
#
# TLS for anything this cluster serves over Traefik. See that role's
# defaults/main.yml for why DNS-01 rather than HTTP-01, and what each of
# these does.
# Where Let's Encrypt sends expiry warnings. An ACME account is registered
# against it.
k3s_cert_manager_acme_email: russell.seymour@turtlesystems.co.uk
# >>> SET BEFORE THE FIRST RUN <<<
#
# How cert-manager proves control of the domain, rendered into the
# ClusterIssuer's solvers list as-is. Left empty deliberately — it depends on
# who runs DNS for turtlesystems.uk, which this repo has no way to know.
# roles/k3s_cert_manager fails fast while it's empty, rather than creating an
# issuer that can never satisfy an order.
#
# Cloudflare, as the most common case — the token needs Zone:DNS:Edit on the
# zone, and is stored in Vault (see k3s_cert_manager_vault_path below and
# docs/vault-secrets.md) rather than written here:
#
# k3s_cert_manager_solver:
# dns01:
# cloudflare:
# apiTokenSecretRef:
# name: cert-manager-dns-credentials
# key: CLOUDFLARE_API_TOKEN
# selector:
# dnsZones:
# - turtlesystems.uk
#
# For another provider, take the stanza from
# https://cert-manager.io/docs/configuration/acme/dns01/ and point its secret
# ref at whichever key you stored at that Vault path.
#
# Use the Let's Encrypt staging directory while working this out — override
# k3s_cert_manager_acme_server. Production allows 5 failed validations per
# hostname per hour, and exhausting it means waiting, not retrying.
k3s_cert_manager_solver:
dns01:
cloudflare:
apiTokenSecretRef:
name: cert-manager-dns-credentials
key: CLOUDFLARE_API_TOKEN
selector:
dnsZones:
- turtlesystems.uk
# --- Traefik (roles/k3s_traefik) --------------------------------------------
#
# The ingress controller k3s installs for itself; that role installs nothing,
# it only adjusts what's already there. Its defaults are enough for the
# dashboard over `kubectl port-forward` — see that role's defaults/main.yml
# and README.md → "Dashboard (Traefik)".
# >>> SET WHEN AUTHENTIK IS READY TO PROTECT IT <<<
#
# Uncomment to publish the dashboard on this hostname, behind the forward-auth
# middleware below. Until then the port-forward is the only way in, which is
# the safe default rather than a limitation. Needs a DNS record pointing at
# Traefik's MetalLB address (`kubectl -n kube-system get svc traefik`).
#
# Before uncommenting, in Authentik: create a Proxy Provider in **forward auth
# (domain level)** mode covering turtlesystems.uk, assign it to an
# application, and add that application to the embedded outpost. Domain level
# rather than single-application for a Kubernetes-specific reason — see
# roles/k3s_traefik/defaults/main.yml.
#
# k3s_traefik_dashboard_host: traefik.turtlesystems.uk
# Authentik's embedded outpost, by in-cluster DNS — the `authentik-server`
# Service in the `authentik` namespace, i.e. the chart's own service, not a
# separately deployed outpost. Unused while the host above is commented out;
# set here anyway so enabling the dashboard is one line rather than two.
# roles/k3s_traefik refuses to run if the host is set and this isn't.
k3s_traefik_dashboard_auth_address: "http://authentik-server.authentik.svc.cluster.local/outpost.goauthentik.io/auth/traefik"
# --- Apps on the cluster (roles/k3s_app) ------------------------------------
#
# The k3s equivalent of `stacks:` in host_vars/nas01.yml and `apps:` in a
# Proxmox guest's host_vars — the list of *applications* on this cluster, as
# opposed to the k3s_* cluster services above, which are the cluster's own
# infrastructure.
#
# In group_vars rather than host_vars because an app is deployed to the
# cluster, not to a node: playbooks/k3s.yml runs roles/k3s_app against
# k3s_control_plane only (that's where the manifests directory is), but
# nothing about an app belongs to that Pi specifically.
#
# Same entry shape as the other two platforms — name, src, vault_path, an
# optional `db:`, and an optional `state:`. And the same removal rule as
# Unraid: set `state: absent` to tear an app down and leave the entry here as
# a tombstone; deleting the entry removes nothing, it just stops Ansible
# visiting it. See roles/k3s_app/tasks/remove.yml.
k3s_apps:
- name: authentik
src: authentik
vault_path: homelab/authentik
db:
name: authentik
user: authentik
# The key at homelab/authentik holding this role's password. Named for
# the environment variable Authentik itself reads, because
# roles/k3s_app passes Vault keys through to the app's Secret verbatim
# — so one value serves both the provisioning step and the running app.
password_vault_key: AUTHENTIK_POSTGRESQL__PASSWORD
admin_vault_path: homelab/shared/postgres
# The CNPG LoadBalancer, not the in-cluster -rw Service: these tasks
# run on the Ansible controller (delegate_to: localhost), which is off
# the cluster and can't resolve or route to a ClusterIP. Authentik
# itself uses the ClusterIP — see
# src/authentik/ansible/kubernetes/vars.yml.
provision_host: "{{ k3s_postgres_loadbalancer_ip }}"
provision_port: "5432"

9
build/config/ansible/inventory/group_vars/k3s_control_plane.yml Normal file
View file

@ -0,0 +1,9 @@
---
# Selects the server branch of roles/k3s_node/tasks/main.yml.
k3s_node_role: server
# The API server's cert needs this IP in its SAN list so kubectl can reach it
# at the same address agents join through. Defaults to the control-plane
# host's own address; override here if you later put a DNS name or VIP in
# front of it.
k3s_api_tls_san: "{{ ansible_host }}"

3
build/config/ansible/inventory/group_vars/k3s_workers.yml Normal file
View file

@ -0,0 +1,3 @@
---
# Selects the agent branch of roles/k3s_node/tasks/main.yml.
k3s_node_role: agent

19
build/config/ansible/inventory/group_vars/proxmox_guests.yml Normal file
View file

@ -0,0 +1,19 @@
---
# Connection settings for the LXCs on the Proxmox nodes — the group
# inventory/proxmox.yml composes from the API, targeted by
# playbooks/proxmox.yml. As distinct from proxmox_nodes.yml, which is the
# hosts those guests run on.
#
# In group_vars rather than repeated per guest because it is a property of
# how every container this repo creates is built, not of any one app: the
# Terraform modules install `ssh_public_keys` for root (see
# build/config/terraform/variables.tf) and create no other account, so root
# is the only user Ansible can authenticate as.
#
# Needed explicitly because the other three groups set this in hosts.yml
# (`root` for unraid_servers and proxmox_nodes, `ansible` for k3s_cluster)
# and the guests have no static entry to set it in. Without it Ansible falls
# back to the *controller's* login name, which no container has an account
# for — a "Permission denied (publickey)" that names a user nothing in the
# repo mentions, so there is no obvious thread to pull.
ansible_user: root

46
build/config/ansible/inventory/group_vars/proxmox_nodes.yml Normal file
View file

@ -0,0 +1,46 @@
---
# Configuration for the Proxmox nodes themselves — as distinct from the guests
# on them, which come from the API-backed dynamic inventory and are configured
# by playbooks/proxmox.yml. Applied by playbooks/pve_host.yml.
#
# In group_vars rather than a host_vars file per node: both nodes want the
# same backup storage and the same schedule, and each backs up whichever
# guests are running on it, so there is nothing here that differs between
# them. A node that ever does need to differ can still override in
# host_vars/<node>.yml.
# Where archives go. An export on the NAS, added to Proxmox as a storage so
# vzdump can write to it — the node mounts it, no guest ever sees it.
#
# nas01 (192.168.50.1). This used to point at nas2, which no longer exists —
# and note the address it used, 192.168.50.2, now belongs to
# turtle-proxmox-01, so a stale copy of this value would have had each node
# quietly trying to write its backups to itself.
pve_backup_storage: nas-backup
pve_backup_storage_type: nfs
pve_backup_nfs_server: 192.168.50.1
pve_backup_nfs_export: /mnt/user/backups/proxmox
# Every guest on the node, so anything added later is covered without an edit
# here. Set false and list VMIDs (postgres is 161, forgejo 160 — see the
# `backup_vmids` output in build/config/terraform/main.tf) to be selective.
#
# Applied per node, so with two nodes each one archives whatever is currently
# running on it. That matters for a replicated guest: it is a real guest on
# one node and an inactive replica volume on the other, so it is backed up
# once, by whichever node currently owns it — and after a failover, by the
# other one, without an edit here.
pve_backup_all: true
# 02:00, half an hour after Forgejo's database dump at 01:30
# (src/forgejo/ansible/proxmox/vars.yml). The archive is only a complete
# restore point if the dump inside it is from the same night — move one and
# move the other.
pve_backup_schedule: "02:00"
# Archive retention, applied by Proxmox. Distinct from the retention inside a
# guest: Proxmox prunes archives, not their contents.
pve_backup_prune:
keep-daily: 7
keep-weekly: 4
keep-monthly: 6

52
build/config/ansible/inventory/host_vars/forgejo.yml Normal file
View file

@ -0,0 +1,52 @@
---
# The Proxmox LXC created by src/forgejo/terraform (VMID 160), on
# turtle-proxmox-02. Named for the container's hostname, which is how it
# appears in the API-backed dynamic inventory.
#
# This was empty for a long time while Forgejo also ran as a Compose stack on
# nas2 — two deployments sharing one Vault path and one database means the
# second to start wins and the other quietly serves stale state. nas2 has
# since been retired, so there is no longer a second copy to collide with and
# the `apps:` list below is live.
#
# WHAT THIS DOES NOT DO: nothing here moves repository data. Ansible
# provisions the database and installs the app; a Forgejo with no
# /var/lib/forgejo/data/forgejo-repositories is a working, empty forge. If the
# nas2 repositories still exist somewhere, restore them into
# `FORGEJO_DATA_PATH` and `chown -R git:git /var/lib/forgejo/data` before
# pointing DNS at this host — the original instructions here were an rsync
# off nas2, which is no longer a machine that can be read from.
#
# Also note SSH clone URLs move from port 2222 to 22 (see FORGEJO_SSH_PORT in
# src/forgejo/ansible/proxmox/vars.yml): existing remotes need editing, and
# git.turtlesystems.uk has to resolve to this container.
# The address Terraform assigns this guest (build/config/terraform/main.tf).
# Pinned here rather than left to DNS: the API-backed dynamic inventory
# supplies no `ansible_host`, so without this Ansible SSHes to the bare
# inventory name `forgejo`, which the LAN's resolver has no record for at
# all. These addresses are hand-assigned from the 192.168.50.50-.59 band
# anyway, so restating one here duplicates nothing that was ever derived.
ansible_host: 192.168.50.52
apps:
- name: forgejo
src: forgejo
vault_path: homelab/forgejo
db:
# The shared Postgres LXC, reached over the LAN. Unlike the Unraid
# host_vars this can't be `{{ ansible_host }}` — that would be this
# container's own address, not the database's. Keep in step with
# DB_HOST in src/forgejo/ansible/proxmox/vars.yml; the authoritative
# copy of both is the `postgres_address` output in
# build/config/terraform/main.tf.
#
# Provisioning runs from the Ansible controller (`delegate_to:
# localhost` in roles/lxc_app), so this address has to be reachable
# from wherever you run the playbook, not just from this container.
provision_host: 192.168.50.54
provision_port: 5432
name: forgejo
user: forgejo
password_vault_key: DB_PASSWORD
admin_vault_path: homelab/shared/postgres

19
build/config/ansible/inventory/host_vars/nas01.yml Normal file
View file

@ -0,0 +1,19 @@
---
# Existing Unraid server. Stacks are applied in list order.
#
# Each entry takes an optional `state:` — `present` (the default) deploys it,
# `absent` tears it down. To remove a stack, set `state: absent` and leave the
# entry here; deleting it instead removes nothing, it just stops Ansible
# managing the containers. See README.md → "Removing an app".
stacks:
# The arr ecosystem as one stack: Shelfarr (library) plus Prowlarr (indexer
# manager) in a single Compose project, so they come up together and share a
# network, a rendered .env and a Vault path.
- name: arr
src: arr
vault_path: homelab/arr
# No `db:` block — neither container uses an external database. Shelfarr
# is SQLite-only (config/database.yml pins production to storage/*.sqlite3
# with no DATABASE_URL support) and Prowlarr keeps its own SQLite under
# /config, so there is nothing to provision on the shared Postgres. Both
# databases live in the appdata bind mounts along with everything else.

28
build/config/ansible/inventory/host_vars/postgres.yml Normal file
View file

@ -0,0 +1,28 @@
---
# The shared Postgres LXC created by src/shared/postgres/terraform (VMID 161,
# 192.168.50.54). Named for the container's hostname, which is how it appears
# in the API-backed dynamic inventory.
#
# This is the one app with no `db:` block, because it *is* the database — the
# per-app databases are created against it by whichever compose_stack /
# lxc_app / k3s_app run declares them, using homelab/shared/postgres as
# `admin_vault_path`.
# The address Terraform assigns this guest (build/config/terraform/main.tf).
# Pinned here rather than left to DNS: the API-backed dynamic inventory
# supplies no `ansible_host`, so without this Ansible SSHes to the bare
# inventory name and whatever the LAN's resolver happens to return. For
# `postgres` that resolver answers 192.168.50.53 — the *Unraid* Postgres
# container, a different instance of the same service — which fails as a
# connection refused rather than as anything that names the real problem.
# These addresses are hand-assigned from the 192.168.50.50-.59 band anyway,
# so restating one here duplicates nothing that was ever derived.
ansible_host: 192.168.50.54
apps:
- name: postgres
src: shared/postgres
vault_path: homelab/shared/postgres
# The packaging's umbrella unit, not postgresql@17-main — see
# POSTGRES_SERVICE_NAME in src/shared/postgres/ansible/proxmox/vars.yml.
service_name: postgresql

58
build/config/ansible/inventory/hosts.yml Normal file
View file

@ -0,0 +1,58 @@
---
all:
children:
unraid_servers:
hosts:
nas01:
ansible_host: 192.168.50.1
vars:
ansible_user: root
# The Proxmox host itself, not the guests on it — those come from the
# API-backed dynamic inventory (proxmox.yml) as `proxmox_guests`. Node-level
# configuration (backup storage, the vzdump schedule) has to be applied over
# SSH to the node, which the API inventory gives no way to address.
#
# VERIFY: depending on its configuration, the community.proxmox inventory
# plugin can also emit the node as a host. If it does, this static entry and
# the dynamic one merge under the same name — check with
# `ansible-inventory --graph` before assuming which vars won.
proxmox_nodes:
hosts:
turtle-proxmox-01:
ansible_host: 192.168.50.2
turtle-proxmox-02:
ansible_host: 192.168.50.3
vars:
ansible_user: root
# Bare-metal K3s cluster (4 Raspberry Pis) — a third platform alongside
# Unraid and Proxmox, not an app deployed onto either of them. No
# Terraform: the hosts already exist. See playbooks/k3s.yml and
# roles/k3s_node. Split into two groups, not one, so the playbook can
# bootstrap the control plane before any agent tries to join it.
#
# ansible_user is `ansible`, not root: create this user on every Pi with
# NOPASSWD sudo and install the public half of `homelab/ci/ssh-k3s`
# (docs/vault-secrets.md) as its authorized key before the first run —
# see README.md "SSH access". The private half doesn't need setting up on
# the controller: group_vars/k3s_cluster.yml points
# ansible_ssh_private_key_file at a path playbooks/k3s_ssh_key.yml fetches
# from that same Vault entry.
k3s_cluster:
vars:
ansible_user: ansible
ansible_become: true
children:
k3s_control_plane:
hosts:
k3s-ctrl-01:
ansible_host: 192.168.50.60
k3s_workers:
hosts:
k3s-wkr-01:
ansible_host: 192.168.50.61
k3s-wkr-02:
ansible_host: 192.168.50.62
k3s-wkr-03:
ansible_host: 192.168.50.63

56
build/config/ansible/inventory/proxmox.yml Normal file
View file

@ -0,0 +1,56 @@
---
# Dynamic inventory: asks the Proxmox API which guests exist rather than
# listing them by hand, so LXCs created by `terraform apply` show up without
# a second edit here.
#
# VERIFY BEFORE FIRST USE — two things this config asserts that depend on
# your Proxmox and collection versions:
#
# 1. Collection name. Proxmox content was split out of `community.general`
# into its own `community.proxmox` collection. If your installed
# Ansible predates that split, this is `community.general.proxmox`
# instead — here, in requirements.yml, and in ansible.cfg's
# `enable_plugins`. Check with:
# ansible-doc -t inventory community.proxmox.proxmox
# 2. Auto-generated group names. The plugin builds groups from guest type
# and state; `proxmox_all_lxc` is the conventional one but confirm
# against your own output before relying on it:
# ansible-inventory -i inventory/proxmox.yml --graph
#
plugin: community.proxmox.proxmox
url: "{{ lookup('env', 'PROXMOX_URL') }}"
user: "{{ lookup('env', 'PROXMOX_USER') }}"
# API token, not a password — same reasoning as the Unraid SSH key: a
# password would mean an interactive prompt (impossible from CI) or a
# plaintext secret on disk. Store these in Vault under `homelab/ci/proxmox`
# and export them before running; see docs/vault-secrets.md.
token_id: "{{ lookup('env', 'PROXMOX_TOKEN_ID') }}"
token_secret: "{{ lookup('env', 'PROXMOX_TOKEN_SECRET') }}"
# Homelab Proxmox typically has a self-signed cert. Flip to true once you
# put a real one on it.
validate_certs: false
want_facts: true
# Manually-created guests are not this repo's to touch. Both Terraform
# modules tag the LXCs they create with `terraform` (see
# src/*/terraform/variables.tf), so that tag is the line between "managed
# here" and "made by hand on the node". Filtering rather than narrowing
# the group below keeps hand-made guests out of the inventory entirely,
# so no future play can target them by accident either.
#
# `filters` needs a reasonably recent plugin version. If yours rejects the
# option, delete this block and put the same condition on the group:
# proxmox_guests: >-
# proxmox_vmtype == 'lxc'
# and 'terraform' in (proxmox_tags_parsed | default([]))
filters:
- "'terraform' in (proxmox_tags_parsed | default([]))"
groups:
# The group playbooks/proxmox.yml targets. Containers only — VMs, if you
# ever add any, are not what `lxc_app` knows how to install into.
proxmox_guests: "proxmox_vmtype == 'lxc'"

21
build/config/ansible/playbooks/deploy.yml Normal file
View file

@ -0,0 +1,21 @@
---
# Everything, both platforms. Unraid first only by convention now — the two
# platforms no longer depend on each other. Proxmox-side apps provision their
# databases against the Proxmox shared Postgres (192.168.50.54), and Terraform
# keeps its state on the CloudNativePG cluster on k3s, neither of which is on
# Unraid.
#
# Run a single platform with `ansible-playbook playbooks/unraid.yml` or
# `playbooks/proxmox.yml` instead. `--limit <host>` and `-e only_stacks=...`
# work the same way against any of the three.
- name: Deploy Unraid stacks
ansible.builtin.import_playbook: unraid.yml
- name: Deploy Proxmox apps
ansible.builtin.import_playbook: proxmox.yml
# Last, because the vzdump job can be pinned to specific VMIDs and those guests
# have to exist first. Harmless to re-run; skipped entirely if no node is
# defined in the `proxmox_nodes` group.
- name: Configure Proxmox node backups
ansible.builtin.import_playbook: pve_host.yml

241
build/config/ansible/playbooks/k3s.yml Normal file
View file

@ -0,0 +1,241 @@
---
# Bootstraps the homelab-utils k3s cluster from bare Pis: no Terraform, the
# hosts already exist (inventory/hosts.yml → k3s_cluster). Manual-only for
# now — not wired into .forgejo/workflows/deploy.yml, since converging 4
# physical nodes on every push is a bigger blast radius than restarting a
# Compose stack (same reasoning the repo already applies to `terraform
# apply`). Run by hand:
#
# ansible-playbook playbooks/k3s.yml
#
# Every play below the SSH-key import is tagged so a single piece can be
# converged without touching the rest — deploying one cluster service
# shouldn't mean re-running the node install across 4 Pis:
#
# --tags nodes k3s install/join + exec-line drift (both node plays)
# --tags metallb MetalLB chart + IPAddressPool
# --tags monitoring kube-prometheus-stack chart
# --tags postgres CloudNativePG operator + Cluster
# --tags cert-manager cert-manager chart + ClusterIssuer
# --tags traefik dashboard route on k3s's bundled Traefik
# --tags services all five of the above, no node install
# --tags apps the k3s_apps: list (roles/k3s_app) — apps, not
# cluster services; narrow further with -e only_apps=…
# --tags upgrades unattended-upgrades config
# --tags kubeconfig re-fetch the admin kubeconfig
#
# e.g. `ansible-playbook playbooks/k3s.yml --tags postgres`. The SSH-key
# import is tagged `always` rather than given a name of its own: it's not a
# thing you'd select, it's the prerequisite for any of these connecting at
# all, so it has to survive every --tags filter.
#
# `services` covers the cluster's own infrastructure and `apps` covers what
# runs on top of it; neither implies the other, so a run meaning "everything
# except reinstalling k3s" is `--tags services,apps`.
#
# Prerequisites (see README.md "K3s (Raspberry Pi)"): the `ansible` user
# exists on every Pi with NOPASSWD sudo and this repo's SSH key installed,
# and homelab/k3s-homelab-utils/K3S_TOKEN is already set in Vault — see
# docs/vault-secrets.md. Both plays fetch that same fixed token independently
# rather than one generating it and handing it to the other, which is what
# makes a full rebuild (wipe both SD cards, reinstall) reproduce the same
# cluster identity.
# First, before anything tries to connect: pull the SSH key the plays below
# authenticate with out of Vault and onto the controller. See
# k3s_ssh_key.yml — it's a separate file because k3s_maintenance.yml imports
# it too.
- ansible.builtin.import_playbook: k3s_ssh_key.yml
tags: always
- name: Bootstrap the k3s control plane
hosts: k3s_control_plane
gather_facts: true
tags: nodes
roles:
- k3s_node
# serial: 1 not for joining itself (idempotent, safe in parallel) but for
# roles/k3s_node's exec-line drift detection: if k3s_extra_args changes and
# every worker's k3s-agent restarts to pick it up, one at a time keeps more
# than one node's kubelet from bouncing simultaneously. See tasks/agent.yml.
- name: Join k3s worker nodes
hosts: k3s_workers
gather_facts: true
serial: 1
tags: nodes
roles:
- k3s_node
# Against the control plane only — it just drops manifests for k3s's own
# helm-controller and deploy controller to reconcile, so it doesn't need
# every node like the plays above. Runs after workers have joined so the
# resulting speaker DaemonSet schedules across the whole cluster on first
# reconcile, though this doesn't strictly matter: the controller picks up
# new nodes on its own regardless of ordering. Before monitoring so a
# LoadBalancer Service (e.g. exposing Prometheus/Grafana later) has
# somewhere to get an IP from as soon as it's requested.
- name: Deploy MetalLB (LoadBalancer IPs)
hosts: k3s_control_plane
gather_facts: false
tags:
- services
- metallb
roles:
- k3s_metallb
# Against the control plane only — it just drops a HelmChart manifest for
# k3s's own helm-controller to reconcile, so it doesn't need every node like
# the plays above. Runs after workers have joined so the resulting
# node-exporter DaemonSet schedules across the whole cluster on first
# reconcile, though this doesn't strictly matter: the controller picks up
# new nodes on its own regardless of ordering.
- name: Deploy cluster monitoring (Prometheus)
hosts: k3s_control_plane
gather_facts: false
tags:
- services
- monitoring
roles:
- k3s_monitoring
# Against the control plane only, same reasoning as monitoring/MetalLB above
# — it just drops manifests for k3s's own controllers to reconcile. Needs
# repo_root (unlike the other k3s roles) because it reads config from
# src/shared/postgres/, the same shared-service config Unraid/Proxmox
# already deploy from — see roles/k3s_postgres/tasks/main.yml.
- name: Deploy shared Postgres (CloudNativePG)
hosts: k3s_control_plane
gather_facts: false
tags:
- services
- postgres
vars:
repo_root: "{{ playbook_dir }}/../../../.."
roles:
- k3s_postgres
# Against the control plane only, same reasoning as the services above.
# Before the apps play because an app's Ingress annotates itself against the
# ClusterIssuer this creates — not that ordering is load-bearing (k3s's
# deploy controller retries, and cert-manager picks up an Ingress whenever it
# appears), but an app deployed first would sit without a certificate until
# this ran, which reads as a broken deploy rather than a pending one.
- name: Deploy cert-manager (TLS certificates)
hosts: k3s_control_plane
gather_facts: false
tags:
- services
- cert-manager
roles:
- k3s_cert_manager
# Against the control plane only, same reasoning as the services above. The
# odd one out among them: it installs nothing, because k3s installs Traefik
# itself — it only adjusts what k3s already put there, via a HelmChartConfig.
# After cert-manager because publishing the dashboard on a hostname asks for a
# Certificate from the ClusterIssuer that play creates; ordering isn't
# load-bearing (the deploy controller retries), it just avoids a route sitting
# without a certificate in between.
- name: Configure Traefik (ingress + dashboard)
hosts: k3s_control_plane
gather_facts: false
tags:
- services
- traefik
roles:
- k3s_traefik
# Apps, as opposed to the cluster services above — see roles/k3s_app for the
# distinction and inventory/group_vars/k3s_cluster.yml for the list. Against
# the control plane because that's where the manifests directory is; nothing
# about an app belongs to that Pi in particular.
#
# Note what a green run here does and doesn't mean, the same caveat every
# service play above carries: the role renders manifests for k3s's
# controllers to reconcile, so success means the files landed and any
# database was provisioned — not that the workload came up. Check with
# `kubectl -n <namespace> get pods`.
- name: Converge apps on the cluster
hosts: k3s_control_plane
gather_facts: false
tags: apps
vars:
repo_root: "{{ playbook_dir }}/../../../.."
# Comma-separated app names to restrict this run to, e.g.
# `-e only_apps=authentik`. Empty (the default) converges every app in
# k3s_apps. Same knob as only_stacks in playbooks/unraid.yml — --tags
# can select the apps play as a whole, but not one app within it.
only_apps: ""
tasks:
# "Converge", not "deploy": an entry carrying `state: absent` is torn
# down rather than brought up. Removals have to stay in the list to be
# acted on — see roles/k3s_app/tasks/main.yml.
- name: Converge each declared app
ansible.builtin.include_role:
name: k3s_app
loop: >-
{{ k3s_apps | default([]) if only_apps == ''
else k3s_apps | default([])
| selectattr('name', 'in', only_apps.split(',')) | list }}
loop_control:
loop_var: app
label: "{{ app.name }}"
# Every node, both roles — hands-off patching so the Pis don't need logging
# into just to stay updated. Reboots are deliberately not part of this: see
# roles/unattended_upgrades and playbooks/k3s_maintenance.yml.
- name: Configure unattended upgrades
hosts: k3s_cluster
gather_facts: true
tags: upgrades
roles:
- unattended_upgrades
# Last, and against the control plane specifically (there's only one) — pulls
# the admin kubeconfig k3s wrote for itself back to the controller so
# `kubectl` works from your workstation.
- name: Fetch the cluster kubeconfig
hosts: k3s_control_plane
gather_facts: false
tags: kubeconfig
vars:
repo_root: "{{ playbook_dir }}/../../../.."
tasks:
- name: Read the cluster's kubeconfig
ansible.builtin.slurp:
src: /etc/rancher/k3s/k3s.yaml
register: k3s_kubeconfig_raw
- name: Ensure the local kubeconfig directory exists
ansible.builtin.file:
path: "{{ k3s_kubeconfig_local_path | dirname }}"
state: directory
mode: "0700"
delegate_to: localhost
become: false
# k3s.yaml points at 127.0.0.1 and names everything "default" — both
# correct only on the node itself. Rewritten so the file is usable
# straight off the controller: the server address becomes reachable from
# off-box, and the cluster/context/user names become this cluster's own
# rather than colliding with every other "default" in ~/.kube/config.
- name: Write the rewritten kubeconfig to the controller
ansible.builtin.copy:
dest: "{{ k3s_kubeconfig_local_path }}"
content: >-
{{ (k3s_kubeconfig_raw.content | b64decode)
| replace('127.0.0.1', ansible_host)
| regex_replace('\\bdefault\\b', k3s_cluster_name) }}
mode: "0600"
delegate_to: localhost
become: false
- name: Show how to use the fetched kubeconfig
ansible.builtin.debug:
msg: >-
Kubeconfig written to {{ k3s_kubeconfig_local_path }}. Use it with
`export KUBECONFIG={{ k3s_kubeconfig_local_path }}`, or merge it
into ~/.kube/config by hand.

37
build/config/ansible/playbooks/k3s_maintenance.yml Normal file
View file

@ -0,0 +1,37 @@
---
# Rolls a "reboot required" flag left by unattended-upgrades
# (roles/unattended_upgrades, Automatic-Reboot disabled) into an actual
# reboot, one node at a time — cordon, drain, reboot, wait for Ready,
# uncordon — so patching the OS never means dropping every workload on the
# cluster at once. A no-op on any node that isn't carrying a pending reboot.
#
# serial: 1 is the point of this playbook: exactly one node is ever down for
# maintenance at a time, whatever `--limit` narrows the run to.
#
# Host order matters here and isn't left to inventory.yml's own group
# order (control plane, then workers) — reversed below so workers are
# rebooted while the API server is still up on the control plane, and the
# control plane's own reboot (which drops the API for everyone, single-server
# cluster, no HA) happens last rather than first.
#
# Manual-only for now, like playbooks/k3s.yml — run by hand:
#
# ansible-playbook playbooks/k3s_maintenance.yml
#
# Intended to eventually run on a schedule from a self-hosted Forgejo Actions
# runner (a `schedule:`-triggered workflow, same runner as
# .forgejo/workflows/deploy.yml) rather than by hand — not wired up yet, but
# the SSH key import below is what makes that possible without also handing
# the runner an ssh-agent.
# Same first play as k3s.yml — the key comes from Vault, and this playbook is
# routinely run on its own, so it can't rely on a k3s.yml run having fetched
# it. See k3s_ssh_key.yml.
- ansible.builtin.import_playbook: k3s_ssh_key.yml
- name: Roll pending reboots across the k3s cluster
hosts: k3s_workers:k3s_control_plane
gather_facts: true
serial: 1
roles:
- k3s_maintenance

82
build/config/ansible/playbooks/k3s_ssh_key.yml Normal file
View file

@ -0,0 +1,82 @@
---
# Materializes the SSH key the k3s plays authenticate with, from Vault
# (homelab/ci/ssh-k3s) onto the controller. Imported as the first play of both
# playbooks/k3s.yml and playbooks/k3s_maintenance.yml — a separate file rather
# than copied into each, since both target k3s_cluster and both would
# otherwise depend on a file only the other one creates.
#
# This is the same shape CI already uses for the Unraid key:
# .forgejo/workflows/deploy.yml fetches homelab/ci/ssh, writes it to disk and
# points ANSIBLE_PRIVATE_KEY_FILE at it. Vault is the one place secrets live,
# so a manual k3s run shouldn't need the key hand-loaded into ssh-agent first
# — which was the only way it worked before.
#
# hosts: k3s_cluster, not localhost, and deliberately so: the implicit
# localhost is not a member of `all`, so it doesn't inherit group_vars/all.yml
# — where vault_addr, vault_kv_mount and vault_auth_method live. Targeting the
# group picks those up along with k3s_ssh_key_vault_path/_local_path from
# group_vars/k3s_cluster.yml. Nothing here connects to a Pi (gather_facts is
# off and every task is delegated or connectionless), which is the whole
# point: at this stage the key isn't on disk yet.
#
# become: false on the delegated tasks because the k3s_cluster group sets
# ansible_become: true for the Pis (inventory/hosts.yml) — without it these
# would try to sudo on the controller.
- name: Fetch the k3s SSH deploy key from Vault
hosts: k3s_cluster
gather_facts: false
tasks:
# The escape hatch, and the reason it's a flag rather than just pointing
# ansible_ssh_private_key_file somewhere else: if Vault is unreachable,
# overriding the key path alone doesn't help — this play would still fail
# before the first real one runs. Skipping it is the only thing that lets
# a run proceed on a local key. Both overrides together:
#
# ansible-playbook playbooks/k3s.yml \
# -e k3s_ssh_key_fetch=false \
# -e ansible_ssh_private_key_file=~/.ssh/k3s_ansible
- name: Fetch and write the key
when: k3s_ssh_key_fetch | default(true) | bool
block:
# run_once because the key is per-cluster, not per-host: one Vault
# read for the whole group. Facts set by a run_once task apply to
# every host in the play, but nothing outside this play needs it.
- name: Look up the k3s SSH deploy key from Vault
ansible.builtin.set_fact:
k3s_ssh_key_secret: >-
{{ lookup('community.hashi_vault.vault_kv2_get',
k3s_ssh_key_vault_path,
engine_mount_point=vault_kv_mount,
url=vault_addr,
auth_method=vault_auth_method,
role_id=vault_role_id | default(omit),
secret_id=vault_secret_id | default(omit)).secret }}
run_once: true
no_log: true
- name: Ensure the local key directory exists
ansible.builtin.file:
path: "{{ k3s_ssh_key_local_path | dirname }}"
state: directory
mode: "0700"
delegate_to: localhost
become: false
run_once: true
# trim + an explicit trailing newline: OpenSSH rejects a key file
# whose final line isn't terminated, and `vault kv put
# PRIVATE_KEY=@file` is as likely to have stored one with trailing
# whitespace as not. 0600 for the same reason — ssh refuses a key file
# that's group- or world-readable. Both fail at connection time rather
# than here, which is a much less obvious error to read.
- name: Write the private key to the controller
ansible.builtin.copy:
dest: "{{ k3s_ssh_key_local_path }}"
content: "{{ k3s_ssh_key_secret.PRIVATE_KEY | trim }}\n"
mode: "0600"
delegate_to: localhost
become: false
run_once: true
no_log: true

29
build/config/ansible/playbooks/proxmox.yml Normal file
View file

@ -0,0 +1,29 @@
---
# Installs apps natively into Proxmox LXCs. The LXCs themselves are created
# by Terraform (build/config/terraform) — run that first; this playbook
# assumes the guests already exist and are reachable.
#
# Hosts come from the community.proxmox inventory plugin
# (inventory/proxmox.yml), which queries the Proxmox API rather than listing
# guests by hand. `apps` is declared per guest in host_vars/<guest>.yml,
# mirroring how `stacks` works for Unraid hosts.
- name: Install apps into Proxmox LXCs
hosts: proxmox_guests
gather_facts: true
vars:
repo_root: "{{ playbook_dir }}/../../../.."
# Comma-separated app names to restrict this run to. Shares the
# `only_stacks` name with playbooks/unraid.yml so a single `-e` narrows
# both platforms in one invocation of deploy.yml.
only_stacks: ""
tasks:
- name: Install each declared app
ansible.builtin.include_role:
name: lxc_app
loop: >-
{{ apps if only_stacks == ''
else apps | selectattr('name', 'in', only_stacks.split(',')) | list }}
loop_control:
loop_var: app
label: "{{ app.name }}"

14
build/config/ansible/playbooks/pve_host.yml Normal file
View file

@ -0,0 +1,14 @@
---
# Configures the Proxmox node itself — as distinct from the guests on it,
# which playbooks/proxmox.yml handles. Right now that means backups: the NAS
# storage archives are written to, and the vzdump schedule that fills it.
#
# Targets the static `proxmox_nodes` group in inventory/hosts.yml rather than
# the API-backed dynamic inventory, which enumerates guests. If no node is
# defined there yet, this play matches nothing and is skipped.
- name: Configure Proxmox nodes
hosts: proxmox_nodes
gather_facts: true
roles:
- pve_backup

24
build/config/ansible/playbooks/unraid.yml Normal file
View file

@ -0,0 +1,24 @@
---
- name: Converge Unraid Docker Compose stacks
hosts: unraid_servers
gather_facts: true
vars:
repo_root: "{{ playbook_dir }}/../../../.."
# Comma-separated stack names to restrict this run to, e.g.
# `-e only_stacks=forgejo` or `-e only_stacks=forgejo,postgres`.
# Empty (the default) converges every stack declared for the host.
only_stacks: ""
tasks:
# "Converge", not "deploy": an entry carrying `state: absent` is torn down
# rather than brought up. Removals have to stay in the list to be acted
# on — see the header comment in roles/compose_stack/tasks/main.yml.
- name: Converge each declared stack
ansible.builtin.include_role:
name: compose_stack
loop: >-
{{ stacks if only_stacks == ''
else stacks | selectattr('name', 'in', only_stacks.split(',')) | list }}
loop_control:
loop_var: stack
label: "{{ stack.name }}"

15
build/config/ansible/requirements.yml Normal file
View file

@ -0,0 +1,15 @@
---
collections:
- name: community.docker
version: ">=3.10.0"
- name: community.hashi_vault
version: ">=6.2.0"
- name: community.postgresql
version: ">=3.0.0"
- name: ansible.posix
version: ">=1.5.0"
# Proxmox dynamic inventory (inventory/proxmox.yml). If your Ansible
# predates the split of Proxmox content out of community.general, drop
# this and use community.general instead — see the note in
# inventory/proxmox.yml.
- name: community.proxmox

9
build/config/ansible/roles/compose_stack/defaults/main.yml Normal file
View file

@ -0,0 +1,9 @@
---
compose_projects_root: /boot/config/plugins/compose.manager/projects
vault_kv_mount: kv
# Where a stack's committed icon.png is copied to on the Unraid host, for the
# `net.unraid.docker.icon` label to point at. Under appdata rather than
# /boot — it's regenerated content, not configuration worth putting on the
# flash drive's write budget.
unraid_icons_root: /mnt/user/appdata/icons

183
build/config/ansible/roles/compose_stack/tasks/deploy.yml Normal file
View file

@ -0,0 +1,183 @@
---
# Deploys one stack to the current Unraid host: sync the compose file, render
# its .env from Vault, provision its database if it declares one, then
# `docker compose up -d`. Compose files stay static/generic — only the
# rendered .env differs per run, which is what keeps re-runs idempotent.
#
# Included from main.yml when the stack's `state:` is `present`; the facts it
# relies on (stack_local_dir, stack_remote_dir) are set there.
- name: Load portable stack variables
ansible.builtin.include_vars:
file: "{{ stack_local_dir }}/common/vars.yml"
name: stack_common_vars
- name: Load Unraid-specific stack variables
ansible.builtin.include_vars:
file: "{{ stack_local_dir }}/ansible/unraid/vars.yml"
name: stack_platform_vars
- name: Ensure remote project directory exists
ansible.builtin.file:
path: "{{ stack_remote_dir }}"
state: directory
mode: "0750"
- name: Copy docker-compose.yml
ansible.builtin.copy:
src: "{{ stack_local_dir }}/ansible/unraid/docker-compose.yml"
dest: "{{ stack_remote_dir }}/docker-compose.yml"
mode: "0640"
# --- Unraid Docker page presentation ---------------------------------------
#
# The committed icon.png has two destinations, because Unraid draws the stack
# row and the containers under it from entirely different sources:
#
# stack row — Compose Manager serves <project dir>/icon.png directly
# (icon.php, which also accepts .jpg/.gif/.svg or a bare
# `icon`). No label, no template, no URL: a file or nothing.
# Present only in the maintained fork, Compose Manager Plus;
# the original plugin has no icon support at all, and there
# the copy is inert rather than harmful.
# containers — the Docker page reads a container's icon, WebUI link and
# console shell from the dockerMan template that created it.
# A Compose stack has no template, so 6.10+ falls back to the
# `net.unraid.docker.*` labels the compose files set, and
# STACK_ICON below is what the icon label resolves to.
#
# Hence the appdata copy as well: the label is a path the webgui resolves at
# page-render time, and pointing it into /boot to reuse the project-dir copy
# would put the flash drive in the path of every Docker page load.
#
# Only the second of those is per-container, so a stack running more than one
# service can also commit icon-<service>.png files. Those get the appdata copy
# only — there is exactly one stack row and it already has its icon. See
# main.yml, which is where the names are resolved.
- name: Sync stack icons to the Unraid host
when: stack_icon_dest | length > 0 or stack_service_icon_names | length > 0
block:
- name: Ensure the icon directory exists
ansible.builtin.file:
path: "{{ unraid_icons_root }}"
state: directory
mode: "0755"
- name: Copy stack icon for the container labels
ansible.builtin.copy:
src: "{{ stack_local_dir }}/ansible/unraid/icon.png"
dest: "{{ stack_icon_dest }}"
mode: "0644"
when: stack_icon_dest | length > 0
# Into the project folder on the flash drive, next to the compose file.
# `copy` is checksum-based, so a redeploy that hasn't changed the icon
# writes nothing — this costs one flash write per icon, not one per run.
- name: Copy stack icon for the Compose Manager stack row
ansible.builtin.copy:
src: "{{ stack_local_dir }}/ansible/unraid/icon.png"
dest: "{{ stack_remote_dir }}/icon.png"
mode: "0644"
when: stack_icon_dest | length > 0
# Appdata only, and no flash-drive copy: these label individual containers,
# which is the one thing the Compose Manager stack row isn't.
- name: Copy per-service icons for the container labels
ansible.builtin.copy:
src: "{{ stack_local_dir }}/ansible/unraid/icon-{{ item }}.png"
dest: "{{ unraid_icons_root }}/{{ stack.name }}-{{ item }}.png"
mode: "0644"
loop: "{{ stack_service_icon_names }}"
- name: Look up stack secrets from Vault
ansible.builtin.set_fact:
vault_secrets: >-
{{ lookup('community.hashi_vault.vault_kv2_get',
stack.vault_path,
engine_mount_point=vault_kv_mount,
url=vault_addr,
auth_method=vault_auth_method,
role_id=vault_role_id | default(omit),
secret_id=vault_secret_id | default(omit)).secret }}
no_log: true
- name: Render .env file
ansible.builtin.template:
src: env.j2
dest: "{{ stack_remote_dir }}/.env"
mode: "0600"
vars:
# The role-computed icon paths first, then portable values, then
# Unraid-specific overrides on top; Vault secrets win over all three (see
# env.j2). STACK_ICON and the per-service STACK_ICON_<SERVICE> keys sit at
# the bottom so either vars.yml layer can replace one with a hosted URL,
# and STACK_ICON is always defined so Compose never warns about an unset
# variable in the label block.
#
# STACK_ICON_<SERVICE> only exists when the matching icon-<service>.png is
# committed. A compose file that interpolates one without the file present
# gets Compose's empty-value warning and the placeholder icon — set it in
# vars.yml instead if the icon is meant to be a URL.
env_defaults: >-
{{ {'STACK_ICON': stack_icon_dest}
| combine(stack_service_icons)
| combine(stack_common_vars.env_defaults | default({}))
| combine(stack_platform_vars.env_defaults | default({})) }}
no_log: true
- name: Provision application database
when: stack.db is defined
no_log: true
block:
- name: Look up Postgres superuser credentials from Vault
ansible.builtin.set_fact:
pg_admin_secrets: >-
{{ lookup('community.hashi_vault.vault_kv2_get',
stack.db.admin_vault_path,
engine_mount_point=vault_kv_mount,
url=vault_addr,
auth_method=vault_auth_method,
role_id=vault_role_id | default(omit),
secret_id=vault_secret_id | default(omit)).secret }}
- name: Ensure application database role exists
community.postgresql.postgresql_user:
name: "{{ stack.db.user }}"
password: "{{ vault_secrets[stack.db.password_vault_key] }}"
login_host: "{{ stack.db.provision_host }}"
login_port: "{{ stack.db.provision_port }}"
login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}"
login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}"
state: present
delegate_to: localhost
become: false
# Role first, then the database with `owner:` — see the same pair of
# tasks in roles/lxc_app for the full reasoning. The short version is
# that `priv: ALL` on postgresql_user was removed from
# community.postgresql in 4.0.0, and ownership is what the replacement
# should have been anyway.
#
# This instance is pinned to Postgres 13, so unlike the Proxmox one it
# was never *also* broken by the Postgres 15 `public` schema change —
# the removal of `priv` is the only thing that forces the edit here.
# Kept identical to the other two roles regardless: the point of the
# three app roles sharing this block is that an app moving between
# platforms gets the same database either way.
- name: Ensure application database exists
community.postgresql.postgresql_db:
name: "{{ stack.db.name }}"
owner: "{{ stack.db.user }}"
login_host: "{{ stack.db.provision_host }}"
login_port: "{{ stack.db.provision_port }}"
login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}"
login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}"
state: present
delegate_to: localhost
become: false
- name: Deploy stack with Docker Compose
community.docker.docker_compose_v2:
project_src: "{{ stack_remote_dir }}"
state: present
pull: policy

75
build/config/ansible/roles/compose_stack/tasks/main.yml Normal file
View file

@ -0,0 +1,75 @@
---
# Converges one stack (one loop iteration of `stack` from playbooks/unraid.yml)
# to the state its host_vars entry asks for.
#
# `state: present` (the default) deploys; `state: absent` tears down. Removal
# has to be asked for explicitly because Ansible keeps no record of what it
# deployed last run — deleting a stack from a host's `stacks:` list only stops
# the loop visiting it, leaving the containers running and unmanaged on the
# host forever. So the entry stays put as a tombstone, with `state: absent`,
# until the teardown has actually been applied.
#
# Unraid-only by design. Proxmox guests install apps natively rather than as
# Compose stacks — see the `lxc_app` role.
- name: Set stack facts
ansible.builtin.set_fact:
stack_local_dir: "{{ repo_root }}/src/{{ stack.src }}"
stack_remote_dir: "{{ compose_projects_root }}/{{ stack.name }}"
stack_state: "{{ stack.state | default('present') }}"
- name: Validate requested stack state
ansible.builtin.assert:
that: stack_state in ['present', 'absent']
fail_msg: >-
Stack '{{ stack.name }}' has state '{{ stack_state }}'; expected
'present' or 'absent'.
quiet: true
# Which icons this stack ships, worked out from the repo checkout on the
# controller rather than from anything on the host — `is exists` and the
# `fileglob` lookup both evaluate locally. Computed here rather than in
# deploy.yml because both branches need it: deploy copies these files out,
# remove deletes them again.
#
# Two naming conventions, because Unraid has two icon consumers (see
# deploy.yml for the full explanation):
#
# icon.png the stack — Compose Manager's project row, plus the
# `net.unraid.docker.icon` label of whichever service
# interpolates ${STACK_ICON}.
# icon-<service>.png one container — becomes ${STACK_ICON_<SERVICE>}, so a
# multi-container stack can label each of its services
# with its own image instead of sharing the stack's.
#
# `<service>` is the compose service name; the variable is it uppercased with
# `-` folded to `_` (icon-shelfarr-libation.png → STACK_ICON_SHELFARR_LIBATION).
- name: Set stack icon facts
ansible.builtin.set_fact:
# Empty when the stack ships no icon.png — an empty label is what the
# webgui already assumes, so it falls back to the placeholder as before.
stack_icon_dest: >-
{{ ((stack_local_dir ~ '/ansible/unraid/icon.png') is exists)
| ternary(unraid_icons_root ~ '/' ~ stack.name ~ '.png', '') }}
stack_service_icon_names: "{{ _service_icon_names }}"
stack_service_icons: >-
{{ dict(_service_icon_names
| map('upper') | map('replace', '-', '_')
| map('regex_replace', '^(.+)$', 'STACK_ICON_\1')
| zip(_service_icon_names
| map('regex_replace', '^(.+)$',
unraid_icons_root ~ '/' ~ stack.name ~ '-\1.png'))) }}
vars:
_service_icon_names: >-
{{ query('fileglob', stack_local_dir ~ '/ansible/unraid/icon-*.png')
| map('basename')
| map('regex_replace', '^icon-(.+)\.png$', '\1')
| map('lower') | list }}
- name: Deploy stack
ansible.builtin.include_tasks: deploy.yml
when: stack_state == 'present'
- name: Remove stack
ansible.builtin.include_tasks: remove.yml
when: stack_state == 'absent'

106
build/config/ansible/roles/compose_stack/tasks/remove.yml Normal file
View file

@ -0,0 +1,106 @@
---
# Tears one stack down off the current Unraid host: `docker compose down`,
# then delete the project folder, then optionally drop its database.
#
# Included from main.yml when the stack's `state:` is `absent`.
#
# Ordering matters and is not obvious: `docker compose down` reads the compose
# file to know what it is removing, so the project folder has to survive until
# after that step. Delete the folder (or the stack's `src/` tree) first and
# there is nothing left to tell Docker what belonged to the project — the
# containers and networks have to be cleaned up by hand instead.
#
# What `down` takes with it, and what it deliberately doesn't:
# - removed: the containers, and any network the project itself created
# - kept: bind mounts. Appdata under /mnt/user/appdata/<app> survives, so
# a removal is reversible by flipping `state:` back to `present`.
# - kept: external networks (caddy-net, unraid_shared) — they belong to
# another stack or to Unraid, not to this project.
# - opt-in: named volumes (`remove_volumes: true`) and images
# (`remove_images: local` or `all`).
#
# Not touched at all: the stack's `homelab/<app>` path in Vault. That has its
# own lifecycle and no reason to be destroyed by a redeployable teardown.
- name: Check whether the stack's project directory is still present
ansible.builtin.stat:
path: "{{ stack_remote_dir }}/docker-compose.yml"
register: stack_compose_file
- name: Tear down stack with Docker Compose
community.docker.docker_compose_v2:
project_src: "{{ stack_remote_dir }}"
state: absent
# Named volumes and images are destructive beyond "stop running this
# here", so they stay opt-in per stack rather than being implied by
# `state: absent`.
remove_volumes: "{{ stack.remove_volumes | default(false) }}"
remove_images: "{{ stack.remove_images | default(omit) }}"
remove_orphans: true
# Absent already — a re-run after a successful removal, or a stack that was
# never deployed to this host. Both are the desired end state, not an error.
when: stack_compose_file.stat.exists
- name: Remove stack project directory
ansible.builtin.file:
path: "{{ stack_remote_dir }}"
state: absent
# Unlike appdata, these are not state — they're copies of PNGs committed in
# the repo, put there purely so the Docker page had something to render.
# Nothing is lost by deleting them and a redeploy puts them back, so they
# aren't opt-in the way volumes and images are.
#
# Listed from the repo checkout (main.yml) rather than globbed on the host:
# `{{ stack.name }}-*.png` would also match the icons of any stack whose name
# starts with this one's.
- name: Remove stack icons
ansible.builtin.file:
path: "{{ item }}"
state: absent
loop: "{{ [unraid_icons_root ~ '/' ~ stack.name ~ '.png']
+ stack_service_icons.values() | list }}"
- name: Drop application database
when:
- stack.db is defined
- stack.remove_database | default(false)
no_log: true
block:
# Only the Postgres superuser credentials are needed here. The stack's own
# Vault secrets are not read, so a teardown still works after its
# `homelab/<app>` path has been deleted.
- name: Look up Postgres superuser credentials from Vault
ansible.builtin.set_fact:
pg_admin_secrets: >-
{{ lookup('community.hashi_vault.vault_kv2_get',
stack.db.admin_vault_path,
engine_mount_point=vault_kv_mount,
url=vault_addr,
auth_method=vault_auth_method,
role_id=vault_role_id | default(omit),
secret_id=vault_secret_id | default(omit)).secret }}
# Database before role: Postgres refuses to drop a role that still owns
# objects, and the role owns this database.
- name: Drop application database
community.postgresql.postgresql_db:
name: "{{ stack.db.name }}"
login_host: "{{ stack.db.provision_host }}"
login_port: "{{ stack.db.provision_port }}"
login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}"
login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}"
state: absent
delegate_to: localhost
become: false
- name: Drop application database role
community.postgresql.postgresql_user:
name: "{{ stack.db.user }}"
login_host: "{{ stack.db.provision_host }}"
login_port: "{{ stack.db.provision_port }}"
login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}"
login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}"
state: absent
delegate_to: localhost
become: false

7
build/config/ansible/roles/compose_stack/templates/env.j2 Normal file
View file

@ -0,0 +1,7 @@
{# Non-secret defaults first, then Vault secrets — secrets win on key clashes. #}
{% for key, value in (env_defaults | default({})).items() %}
{{ key }}="{{ value }}"
{% endfor %}
{% for key, value in (vault_secrets | default({})).items() %}
{{ key }}="{{ value }}"
{% endfor %}

34
build/config/ansible/roles/k3s_app/defaults/main.yml Normal file
View file

@ -0,0 +1,34 @@
---
# Ansible-side knobs for roles/k3s_app — the k3s equivalent of
# compose_stack (Unraid) and lxc_app (Proxmox): the generic role that
# deploys one *application* onto the homelab-utils cluster, as opposed to
# the k3s_* roles that deploy one *cluster service*.
#
# The distinction is worth stating, because the file layout looks similar.
# roles/k3s_metallb, k3s_monitoring, k3s_postgres and k3s_cert_manager each
# hard-code a single thing the cluster itself needs, with their config in
# defaults/ and group_vars. This role hard-codes nothing: it's driven by the
# `k3s_apps:` list in inventory/group_vars/k3s_cluster.yml and reads each
# app's config out of src/<app>/, the same src/ tree Unraid and Proxmox
# already deploy from. An app that moves between platforms gets a new
# platform vars file, not a rewrite — see CLAUDE.md → "How config is
# layered".
#
# Everything reaches the cluster the same way every other k3s role does:
# rendering a file into k3s's auto-deploying manifests directory and letting
# k3s's own controllers reconcile it. No helm binary, no kubeconfig, no
# kubernetes.core collection on the controller.
# Where k3s watches for manifests to apply. Anything written here is applied
# by k3s's deploy controller, and — importantly for remove.yml — anything
# *deleted* from here has its resources torn down again, because the
# controller tracks what each file created via an Addon CR.
k3s_manifests_dir: /var/lib/rancher/k3s/server/manifests
vault_kv_mount: kv
# Which directory under src/<app>/ansible/ holds the Kubernetes deployment.
# "kubernetes" rather than "k3s": what's in there is plain Kubernetes plus
# k3s's HelmChart CR, and an app moved to a different distribution would
# keep most of it.
k3s_app_platform_dir: kubernetes

170
build/config/ansible/roles/k3s_app/tasks/deploy.yml Normal file
View file

@ -0,0 +1,170 @@
---
# Deploys one app to the k3s cluster: layer its config, provision its
# database if it declares one, render its secrets into a Secret, then render
# whatever manifests the app ships into k3s's auto-deploying directory.
#
# The shape deliberately mirrors compose_stack/deploy.yml, because the
# separation it enforces is the same one: config that's committed, secrets
# that never are. On Unraid the split is "static docker-compose.yml
# referencing ${VAR}" + "rendered .env"; here it's "manifests carrying only
# non-secret config" + "a rendered Secret the manifests reference by name".
# In both cases the committed half is safe to read and the generated half
# never lands in git.
#
# Included from main.yml when the app's `state:` is `present`; the facts it
# relies on (app_local_dir) are set there.
- name: Load portable app variables
ansible.builtin.include_vars:
file: "{{ app_local_dir }}/common/vars.yml"
name: app_common_vars
- name: Load Kubernetes-specific app variables
ansible.builtin.include_vars:
file: "{{ app_local_dir }}/ansible/{{ k3s_app_platform_dir }}/vars.yml"
name: app_platform_vars
# Portable values first, platform overrides on top — same precedence as
# compose_stack's env_defaults merge, and the same reason: an app's ports and
# database name don't change with the platform, its storage class and
# ingress class do. Vault secrets are *not* merged in here; they go to the
# Secret in a separate task below, so a manifest template can never
# accidentally interpolate one into a world-readable file.
- name: Merge app configuration
ansible.builtin.set_fact:
app_config: >-
{{ (app_common_vars.env_defaults | default({}))
| combine(app_platform_vars.env_defaults | default({})) }}
- name: Look up app secrets from Vault
ansible.builtin.set_fact:
app_vault_secrets: >-
{{ lookup('community.hashi_vault.vault_kv2_get',
app.vault_path,
engine_mount_point=vault_kv_mount,
url=vault_addr,
auth_method=vault_auth_method,
role_id=vault_role_id | default(omit),
secret_id=vault_secret_id | default(omit)).secret }}
no_log: true
# --- Database ---------------------------------------------------------------
#
# Identical to compose_stack's block, pointed at a different Postgres. That
# it *can* be identical is the whole point of roles/k3s_postgres publishing
# the CNPG primary on a MetalLB LoadBalancer: `community.postgresql` runs on
# the controller (delegate_to: localhost) and needs a real host:port, which
# an in-cluster ClusterIP Service isn't. See CLAUDE.md → "Key decisions" —
# this is the deploy path that entry says is missing.
#
# provision_host is the LoadBalancer address, not the -rw ClusterIP; the app
# itself still talks to the ClusterIP by DNS (see the app's vars.yml), so
# the LAN-facing address is only ever used by the controller at deploy time.
- name: Provision application database
when: app.db is defined
no_log: true
block:
- name: Look up Postgres superuser credentials from Vault
ansible.builtin.set_fact:
pg_admin_secrets: >-
{{ lookup('community.hashi_vault.vault_kv2_get',
app.db.admin_vault_path,
engine_mount_point=vault_kv_mount,
url=vault_addr,
auth_method=vault_auth_method,
role_id=vault_role_id | default(omit),
secret_id=vault_secret_id | default(omit)).secret }}
- name: Ensure application database role exists
community.postgresql.postgresql_user:
name: "{{ app.db.user }}"
password: "{{ app_vault_secrets[app.db.password_vault_key] }}"
login_host: "{{ app.db.provision_host }}"
login_port: "{{ app.db.provision_port }}"
login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}"
login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}"
state: present
delegate_to: localhost
become: false
# After the role, not before, and with owner: — unlike compose_stack,
# which creates the database first and then grants on it. The difference
# is that this database may be restored into from a pg_dump taken
# elsewhere (see docs/authentik-migration.md): a dump recreates objects
# with their original ownership, which only resolves if the owning role
# already exists and owns the database. Creating it owner-less and
# granting after works for an empty database and quietly leaves a
# restored one owned by postgres.
- name: Ensure application database exists
community.postgresql.postgresql_db:
name: "{{ app.db.name }}"
owner: "{{ app.db.user }}"
login_host: "{{ app.db.provision_host }}"
login_port: "{{ app.db.provision_port }}"
login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}"
login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}"
state: present
delegate_to: localhost
become: false
# --- Manifests --------------------------------------------------------------
# Namespace + Secret in one file, so the namespace an app's manifests target
# is guaranteed to be created by something even if the app ships only a
# HelmChart CR (whose createNamespace: true fires too late for a Secret the
# chart's pods mount). Ordering between files doesn't otherwise matter here:
# k3s's deploy controller retries a manifest whose namespace or CRDs don't
# exist yet rather than failing once and giving up — the same property
# roles/k3s_postgres and roles/k3s_metallb already rely on.
#
# 0600 and no_log because this one carries every value from the app's Vault
# path. Note this is on-disk protection on the node only: the Secret's
# contents are then base64 in etcd like any Kubernetes Secret, which is the
# same trust boundary the rest of this cluster already assumes.
- name: Render the app namespace and secrets manifest
ansible.builtin.template:
src: app-secrets.yaml.j2
dest: "{{ k3s_manifests_dir }}/{{ app.name }}-secrets.yaml"
owner: root
group: root
mode: "0600"
become: true
no_log: true
# Whatever the app ships — a HelmChart CR, an Ingress, a PVC, a Certificate.
# Enumerated from the repo checkout on the controller (fileglob evaluates
# locally), so adding a manifest to an app means dropping a .yaml.j2 next to
# the others, with no role change.
#
# Prefixed with the app name on the node, because every app's manifests share
# one flat directory there. Name the files for their content and not for the
# app (helmchart.yaml.j2, not authentik.helmchart.yaml.j2) — the prefix is
# added here, and an app-named file stutters into authentik-authentik.yaml.
- name: Find the app's Kubernetes manifest templates
ansible.builtin.set_fact:
app_manifest_templates: >-
{{ query('fileglob',
app_local_dir ~ '/ansible/' ~ k3s_app_platform_dir ~ '/*.yaml.j2')
| sort }}
- name: Fail fast if the app ships no manifests
ansible.builtin.assert:
that: app_manifest_templates | length > 0
fail_msg: >-
App '{{ app.name }}' has no *.yaml.j2 under
src/{{ app.src }}/ansible/{{ k3s_app_platform_dir }}/ — nothing to
deploy. An app on this platform needs at least one manifest (normally
a HelmChart CR).
quiet: true
- name: Render the app's Kubernetes manifests
ansible.builtin.template:
src: "{{ item }}"
dest: >-
{{ k3s_manifests_dir }}/{{ app.name }}-{{
item | basename | regex_replace('\.j2$', '') }}
owner: root
group: root
mode: "0644"
loop: "{{ app_manifest_templates }}"
become: true

37
build/config/ansible/roles/k3s_app/tasks/main.yml Normal file
View file

@ -0,0 +1,37 @@
---
# Converges one app (one loop iteration of `app` from playbooks/k3s.yml) to
# the state its k3s_apps: entry asks for.
#
# `state: present` (the default) deploys; `state: absent` tears down. Same
# explicit-removal rule as compose_stack, and for the same reason: Ansible
# keeps no record of what it deployed last run, so deleting an app from
# `k3s_apps:` only stops the loop visiting it — the workload keeps running,
# unmanaged, until something says `absent`. The entry stays as a tombstone.
#
# Unlike compose_stack, removal here does tear the workload down completely
# on the first pass, because k3s's deploy controller owns the resources a
# manifest created and garbage-collects them when the file goes away. What
# it does *not* touch, by the same "a default teardown should be
# reversible" reasoning compose_stack applies: the app's database, its
# PersistentVolumeClaims, and its Vault path.
- name: Set app facts
ansible.builtin.set_fact:
app_local_dir: "{{ repo_root }}/src/{{ app.src }}"
app_state: "{{ app.state | default('present') }}"
- name: Validate requested app state
ansible.builtin.assert:
that: app_state in ['present', 'absent']
fail_msg: >-
App '{{ app.name }}' has state '{{ app_state }}'; expected 'present'
or 'absent'.
quiet: true
- name: Deploy app
ansible.builtin.include_tasks: deploy.yml
when: app_state == 'present'
- name: Remove app
ansible.builtin.include_tasks: remove.yml
when: app_state == 'absent'

60
build/config/ansible/roles/k3s_app/tasks/remove.yml Normal file
View file

@ -0,0 +1,60 @@
---
# Tears one app off the cluster: delete the manifests it was deployed from
# and let k3s's deploy controller garbage-collect what they created.
#
# This is the one place k3s is *less* work than Compose. `compose_stack`
# has to run `docker compose down` and therefore needs the compose file to
# still be on disk to know what it's tearing down (which is why removing an
# app there is a two-pass job — see CLAUDE.md → "Removing an app"). Here the
# deploy controller already tracks which resources each manifest file
# created, via the Addon CR it writes alongside them, so deleting the file
# is the teardown. src/<app>/ can be deleted in the same commit.
#
# Deliberately *not* removed, same reasoning as compose_stack's opt-in
# flags — a default teardown should be reversible:
#
# - the app's database and role on the shared Postgres
# - its PersistentVolumeClaims (the app's manifests own those; if the
# chart's PVCs carry a Helm ownership annotation they go with the
# HelmChart CR, so check `kubectl -n <ns> get pvc` after)
# - its Vault path
# - the namespace, which the secrets manifest below creates but the
# controller will only remove if nothing else landed in it
#
# There's no `remove_database`/`remove_volumes` equivalent yet. Add one the
# day it's actually wanted rather than guessing at the shape now.
- name: Find the app's Kubernetes manifest templates
ansible.builtin.set_fact:
app_manifest_templates: >-
{{ query('fileglob',
app_local_dir ~ '/ansible/' ~ k3s_app_platform_dir ~ '/*.yaml.j2')
| sort }}
# The secrets manifest first, then the app's own. Order is cosmetic — the
# controller reconciles each file's removal independently — but removing the
# workload's namespace/Secret last would leave pods briefly running without
# the credentials they were started with, and losing them noisily in a log
# is worse than losing them quietly.
- name: Remove the app's Kubernetes manifests
ansible.builtin.file:
path: >-
{{ k3s_manifests_dir }}/{{ app.name }}-{{
item | basename | regex_replace('\.j2$', '') }}
state: absent
loop: "{{ app_manifest_templates }}"
become: true
- name: Remove the app namespace and secrets manifest
ansible.builtin.file:
path: "{{ k3s_manifests_dir }}/{{ app.name }}-secrets.yaml"
state: absent
become: true
- name: Report what removal left behind
ansible.builtin.debug:
msg: >-
App '{{ app.name }}' manifests removed; k3s will garbage-collect the
resources they created. Its database, PVCs and Vault path were left
alone on purpose — remove those by hand if the teardown is meant to be
permanent.

35
build/config/ansible/roles/k3s_app/templates/app-secrets.yaml.j2 Normal file
View file

@ -0,0 +1,35 @@
{#
Managed by Ansible (roles/k3s_app) — do not edit on the node.
The k3s counterpart of compose_stack's rendered .env: every key from the
app's Vault path, verbatim, in one Secret the app's manifests reference by
name. Keys are not renamed or filtered on the way through, which is what
keeps this role generic — an app decides what its environment looks like by
choosing its Vault keys, exactly as it does on Unraid, and the role stays
ignorant of any particular app's variables.
The intended consumer is an envFrom/secretRef in the app's own manifest,
so the values become environment variables without ever being named in a
committed file. See src/authentik/ansible/kubernetes/ for the worked
example.
Namespace lives here rather than in the app's manifests so it's guaranteed
to exist before anything mounts this Secret — a HelmChart CR's
createNamespace: true happens when the chart installs, which is after the
helm-controller job needs somewhere to put it.
-#}
apiVersion: v1
kind: Namespace
metadata:
name: {{ app_config.K8S_NAMESPACE }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ app.name }}-secrets
namespace: {{ app_config.K8S_NAMESPACE }}
type: Opaque
stringData:
{% for key, value in (app_vault_secrets | default({})) | dictsort %}
{{ key }}: {{ value | string | to_json }}
{% endfor %}

73
build/config/ansible/roles/k3s_cert_manager/defaults/main.yml Normal file
View file

@ -0,0 +1,73 @@
---
# Ansible-side knobs for roles/k3s_cert_manager — cert-manager, which issues
# and renews the TLS certificates Ingresses on this cluster serve. Installed
# via k3s's own bundled helm-controller, same mechanism as roles/k3s_metallb,
# k3s_monitoring and k3s_postgres (see tasks/main.yml) — no helm binary or
# extra Ansible collection needed on the controller.
#
# This is the fourth cluster service, and the first one that exists purely
# for apps rather than for the cluster: nothing here needs a certificate,
# roles/k3s_app's tenants do. It's still a cluster service rather than an
# app, because a certificate issuer is shared infrastructure in the same way
# the shared Postgres is — one ClusterIssuer, every app's Ingress annotates
# itself against it.
k3s_cert_manager_namespace: cert-manager
k3s_cert_manager_chart_repo: https://charts.jetstack.io
# Pinned, same reasoning as every other chart version here — a rebuild
# months from now should reproduce today's install, not whatever's newest.
# Bump deliberately; check the current release first at
# https://github.com/cert-manager/cert-manager/releases.
k3s_cert_manager_chart_version: "v1.21.1"
# The name every app's Ingress annotates itself with
# (cert-manager.io/cluster-issuer: <this>). A ClusterIssuer rather than a
# per-namespace Issuer, so an app doesn't need its own copy of the ACME
# account and DNS credentials in its own namespace.
k3s_cert_manager_issuer_name: letsencrypt
# Let's Encrypt's production directory. Swap for the staging URL
# (https://acme-staging-v02.api.letsencrypt.org/directory) while working out
# a solver configuration: production has a hard rate limit of 5 failed
# validations per account/hostname/hour, and burning it means waiting rather
# than retrying. Changing this changes which ACME account the issuer uses, so
# existing certificates are re-issued rather than renewed.
k3s_cert_manager_acme_server: https://acme-v02.api.letsencrypt.org/directory
# The address Let's Encrypt sends expiry warnings to. No default — an ACME
# account is registered against it, so a wrong value is worth failing on
# rather than guessing. Set in inventory/group_vars/k3s_cluster.yml.
k3s_cert_manager_acme_email: ""
# Where the ACME account's private key is kept. cert-manager generates it on
# first registration; it is *not* the certificate key, and losing it means
# re-registering rather than losing certificates.
k3s_cert_manager_acme_key_secret: letsencrypt-account-key
# --- Solver -----------------------------------------------------------------
#
# How cert-manager proves control of the domain. DNS-01 by default, not
# HTTP-01, because HTTP-01 requires Let's Encrypt to reach this cluster from
# the internet on port 80 — true only if the LAN is port-forwarded, and this
# cluster deliberately isn't. DNS-01 needs no inbound path at all, and is the
# only option that can issue wildcards.
#
# The provider block is left open rather than hard-coding Cloudflare, since
# it's the one part of this role that depends on who runs the DNS. Set
# k3s_cert_manager_solver in inventory/group_vars/k3s_cluster.yml to whatever
# your provider's stanza looks like in cert-manager's docs
# (https://cert-manager.io/docs/configuration/acme/dns01/); it's rendered
# into the ClusterIssuer's solvers list as-is. Any secret it references
# should name the Secret this role renders from Vault (below), so no token is
# ever committed. tasks/main.yml fails fast while this is empty.
k3s_cert_manager_solver: {}
# The Vault path holding the DNS provider's API credentials. Every key at
# this path becomes a key in a Secret named after it in the cert-manager
# namespace, the same "Vault keys pass through verbatim" rule roles/k3s_app
# uses — so the solver above references whichever key name you stored.
# Leave empty for a solver that needs no credentials (e.g. HTTP-01).
k3s_cert_manager_vault_path: homelab/k3s-cert-manager
k3s_cert_manager_credentials_secret: cert-manager-dns-credentials

75
build/config/ansible/roles/k3s_cert_manager/tasks/main.yml Normal file
View file

@ -0,0 +1,75 @@
---
# Drops a HelmChart CR for cert-manager, plus a ClusterIssuer and the DNS
# credentials it solves with, into k3s's auto-deploying manifests directory;
# k3s's bundled helm-controller and deploy controller reconcile them — same
# mechanism and same two-manifest shape as roles/k3s_metallb (chart CR +
# plain config manifest referencing CRDs the chart hasn't installed yet).
#
# The ClusterIssuer is exactly that case: cert-manager.io/v1 doesn't exist
# until the chart has installed, so this manifest is unappliable at the
# moment it's written. That's fine and deliberate — k3s's deploy controller
# retries a manifest referencing not-yet-existing CRDs until they show up,
# rather than failing once and giving up.
- name: Fail fast if cert-manager is not configured
ansible.builtin.assert:
that:
- k3s_cert_manager_acme_email | length > 0
- k3s_cert_manager_solver | length > 0
fail_msg: >-
k3s_cert_manager_acme_email and/or k3s_cert_manager_solver are unset —
set both in inventory/group_vars/k3s_cluster.yml before running
playbooks/k3s.yml. An ACME account is registered against the email, and
without a solver the ClusterIssuer would be created but never able to
prove domain control, leaving every Certificate pending indefinitely
rather than failing loudly.
quiet: true
run_once: true
- name: Deploy the cert-manager HelmChart manifest
ansible.builtin.template:
src: cert-manager.helmchart.yaml.j2
dest: /var/lib/rancher/k3s/server/manifests/cert-manager.yaml
owner: root
group: root
mode: "0644"
become: true
# Before the ClusterIssuer, because that's what references it. Ordering
# between files isn't actually enforced (see the header) — this is for the
# reader, not the controller.
- name: Look up the DNS provider credentials from Vault
when: k3s_cert_manager_vault_path | length > 0
ansible.builtin.set_fact:
cert_manager_vault_secrets: >-
{{ lookup('community.hashi_vault.vault_kv2_get',
k3s_cert_manager_vault_path,
engine_mount_point=vault_kv_mount,
url=vault_addr,
auth_method=vault_auth_method,
role_id=vault_role_id | default(omit),
secret_id=vault_secret_id | default(omit)).secret }}
no_log: true
# Separate file and mode from everything else here, same split
# roles/k3s_postgres makes: this is the only manifest carrying a credential,
# so it's the only one that isn't safe at 0644.
- name: Deploy the DNS provider credentials Secret
when: k3s_cert_manager_vault_path | length > 0
ansible.builtin.template:
src: dns-credentials.yaml.j2
dest: /var/lib/rancher/k3s/server/manifests/cert-manager-dns-credentials.yaml
owner: root
group: root
mode: "0600"
become: true
no_log: true
- name: Deploy the ClusterIssuer manifest
ansible.builtin.template:
src: cluster-issuer.yaml.j2
dest: /var/lib/rancher/k3s/server/manifests/cert-manager-cluster-issuer.yaml
owner: root
group: root
mode: "0644"
become: true

59
build/config/ansible/roles/k3s_cert_manager/templates/cert-manager.helmchart.yaml.j2 Normal file
View file

@ -0,0 +1,59 @@
{#
Managed by Ansible (roles/k3s_cert_manager) — do not edit on the node.
A HelmChart CR, same pattern as roles/k3s_metallb and roles/k3s_monitoring:
k3s's bundled helm-controller reconciles it, the same mechanism it uses to
install its own Traefik.
-#}
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
name: cert-manager
namespace: kube-system
spec:
chart: cert-manager
repo: {{ k3s_cert_manager_chart_repo }}
version: "{{ k3s_cert_manager_chart_version }}"
targetNamespace: {{ k3s_cert_manager_namespace }}
createNamespace: true
valuesContent: |-
# The chart installs its own CRDs. Without this they'd have to be applied
# separately before the chart, which the helm-controller gives no ordering
# hook for — and the ClusterIssuer this role also renders would then never
# become appliable.
crds:
enabled: true
# Sized for a Raspberry Pi 4, same reasoning as k3s_monitoring_values and
# k3s_postgres_resources. cert-manager is idle almost all of the time —
# it wakes to renew a certificate every 60 days — so these are set for
# "doesn't get OOM-killed during a renewal", not for throughput.
resources:
requests:
cpu: 10m
memory: 64Mi
limits:
memory: 128Mi
webhook:
resources:
requests:
cpu: 10m
memory: 64Mi
limits:
memory: 128Mi
cainjector:
resources:
requests:
cpu: 10m
memory: 96Mi
limits:
memory: 192Mi
# DNS-01 self-check queries the authoritative nameservers directly rather
# than going through the cluster's resolver. Without this, CoreDNS
# forwards to whatever the LAN's DNS is, and a split-horizon setup —
# where the internal view of the zone doesn't carry the _acme-challenge
# TXT record the public view does — makes cert-manager wait for a record
# it will never see, until the order times out.
dns01RecursiveNameservers: "1.1.1.1:53,9.9.9.9:53"
dns01RecursiveNameserversOnly: true

30
build/config/ansible/roles/k3s_cert_manager/templates/cluster-issuer.yaml.j2 Normal file
View file

@ -0,0 +1,30 @@
{#
Managed by Ansible (roles/k3s_cert_manager) — do not edit on the node.
Plain manifest, not a HelmChart — same pattern as
roles/k3s_metallb/templates/metallb-config.yaml.j2 and
roles/k3s_postgres/templates/postgres-cluster.yaml.j2: this references a
CRD (cert-manager.io) that only exists once the chart alongside it has
installed, and k3s's deploy controller retries rather than giving up.
ClusterIssuer rather than Issuer so it's usable from every namespace —
an app's Ingress just annotates itself with
`cert-manager.io/cluster-issuer: {{ k3s_cert_manager_issuer_name }}` and
needs no ACME account or DNS credentials of its own.
-#}
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
name: {{ k3s_cert_manager_issuer_name }}
spec:
acme:
server: {{ k3s_cert_manager_acme_server }}
email: {{ k3s_cert_manager_acme_email }}
# Where cert-manager keeps the ACME *account* key it generates on first
# registration — not any certificate's key. In the cert-manager namespace
# because a ClusterIssuer's secrets always resolve there, regardless of
# which namespace the Certificate using it lives in.
privateKeySecretRef:
name: {{ k3s_cert_manager_acme_key_secret }}
solvers:
{{ [k3s_cert_manager_solver] | to_nice_yaml(indent=2) | indent(6, first=true) }}

23
build/config/ansible/roles/k3s_cert_manager/templates/dns-credentials.yaml.j2 Normal file
View file

@ -0,0 +1,23 @@
{#
Managed by Ansible (roles/k3s_cert_manager) — do not edit on the node.
The DNS provider API credentials the ACME DNS-01 solver authenticates with,
passed through from Vault verbatim — same rule roles/k3s_app's Secret
follows, so the key names in k3s_cert_manager_solver are whatever you chose
when populating the Vault path, and this role stays ignorant of which
provider is in use.
In the cert-manager namespace because a ClusterIssuer resolves every Secret
it references there, never in the namespace of the Certificate being
issued.
-#}
apiVersion: v1
kind: Secret
metadata:
name: {{ k3s_cert_manager_credentials_secret }}
namespace: {{ k3s_cert_manager_namespace }}
type: Opaque
stringData:
{% for key, value in (cert_manager_vault_secrets | default({})) | dictsort %}
{{ key }}: {{ value | string | to_json }}
{% endfor %}

100
build/config/ansible/roles/k3s_maintenance/tasks/main.yml Normal file
View file

@ -0,0 +1,100 @@
---
# Rolling "reboot this node if unattended-upgrades left one pending" for one
# k3s node — one iteration of the serial: 1 loop in
# playbooks/k3s_maintenance.yml. roles/unattended_upgrades installs updates on
# its own schedule with Automatic-Reboot disabled, so a kernel/library update
# can sit applied-but-inactive on a node indefinitely; this is what actually
# reboots it — cordoned and drained first, so workloads move off before the
# node disappears rather than during.
#
# kubectl commands are delegated to the control-plane node and run as
# `k3s kubectl`, k3s's own bundled client — no separate kubectl install or
# local kubeconfig needed. Works the same whether the node having its turn
# right now *is* the control plane: delegating to itself, over SSH, before it
# reboots itself.
- name: Sanity-check there is exactly one control-plane node
ansible.builtin.assert:
that: groups['k3s_control_plane'] | length == 1
fail_msg: >-
k3s_maintenance delegates kubectl to groups['k3s_control_plane'][0] —
it assumes a single control-plane node. Update this role before adding
a second one for HA.
quiet: true
- name: Check whether a reboot is required
ansible.builtin.stat:
path: /var/run/reboot-required
register: k3s_reboot_required
- name: Node is up to date — nothing to do
ansible.builtin.debug:
msg: "{{ inventory_hostname }}: no reboot required, skipping."
when: not k3s_reboot_required.stat.exists
- name: Reboot this node if unattended-upgrades left one pending
when: k3s_reboot_required.stat.exists
block:
# --force: this is a homelab, not a cluster with a policy against bare
# pods — better to evict them than have a stray one block every rebuild.
# --delete-emptydir-data: emptyDir contents are expected to be
# disposable; anything that isn't shouldn't be using emptyDir.
- name: Cordon and drain the node
ansible.builtin.command:
argv:
- k3s
- kubectl
- drain
- "{{ inventory_hostname }}"
- --ignore-daemonsets
- --delete-emptydir-data
- --force
- --timeout=120s
delegate_to: "{{ groups['k3s_control_plane'][0] }}"
become: false
changed_when: true
- name: Reboot the node
ansible.builtin.reboot:
reboot_timeout: 300
# Polls rather than trusting the reboot handshake alone — the node can be
# reachable over SSH before k3s (and, if this is the control-plane node
# itself, the API server it just took down with it) has finished coming
# back up.
- name: Wait for the node to report Ready again
ansible.builtin.command:
argv:
- k3s
- kubectl
- wait
- --for=condition=Ready
- "node/{{ inventory_hostname }}"
- --timeout=20s
delegate_to: "{{ groups['k3s_control_plane'][0] }}"
become: false
register: k3s_node_ready
changed_when: false
failed_when: false
until: k3s_node_ready.rc == 0
retries: 12
delay: 15
- name: Fail if the node never came back Ready
ansible.builtin.fail:
msg: >-
{{ inventory_hostname }} rebooted but never reported Ready again —
it's left cordoned; check it by hand before re-running this against
the rest of the cluster.
when: k3s_node_ready.rc != 0
- name: Uncordon the node
ansible.builtin.command:
argv: [k3s, kubectl, uncordon, "{{ inventory_hostname }}"]
delegate_to: "{{ groups['k3s_control_plane'][0] }}"
become: false
changed_when: true
- name: Node rebooted and rejoined the cluster
ansible.builtin.debug:
msg: "{{ inventory_hostname }}: rebooted, drained and uncordoned cleanly."

31
build/config/ansible/roles/k3s_metallb/defaults/main.yml Normal file
View file

@ -0,0 +1,31 @@
---
# Ansible-side knobs for roles/k3s_metallb — MetalLB, the LoadBalancer
# implementation for the homelab-utils cluster. Installed via k3s's own
# bundled helm-controller, same mechanism as roles/k3s_monitoring (see
# tasks/main.yml) — no helm binary or extra Ansible collection needed on the
# controller.
#
# MetalLB replaces k3s's bundled ServiceLB (Klipper), it doesn't sit
# alongside it — both would otherwise try to satisfy the same LoadBalancer
# Services and fight over IPs. That's why `k3s_extra_args` in
# inventory/group_vars/k3s_cluster.yml carries `--disable=servicelb`.
k3s_metallb_namespace: metallb-system
k3s_metallb_chart_repo: https://metallb.github.io/metallb
# Pinned, same reasoning as k3s_monitoring_chart_version — a rebuild months
# from now should reproduce today's stack, not whatever's newest at the
# time. Bump deliberately; check the current release first at
# https://github.com/metallb/metallb/releases.
k3s_metallb_chart_version: "0.14.9"
# The pool of IPs MetalLB hands out to LoadBalancer-type Services, as a
# "first-last" range or CIDR (e.g. "192.168.50.240-192.168.50.250"). Must be
# addresses on the cluster's LAN that nothing else — DHCP, static
# assignments, the hosts in inventory/hosts.yml — will ever claim: MetalLB
# doesn't coordinate with your router, it just hands out whatever's in this
# range. No default on purpose; set it in
# inventory/group_vars/k3s_cluster.yml before the first run. tasks/main.yml
# fails fast if it's still empty.
k3s_metallb_address_range: ""

39
build/config/ansible/roles/k3s_metallb/tasks/main.yml Normal file
View file

@ -0,0 +1,39 @@
---
# Drops a HelmChart CR for MetalLB, plus its IPAddressPool/L2Advertisement
# config, into k3s's auto-deploying manifests directory; k3s's bundled
# helm-controller and deploy controller reconcile them — same mechanism as
# roles/k3s_monitoring (see its tasks/main.yml for why this needs no helm
# binary or kubeconfig on the controller).
#
# The config manifest references CRDs that only exist once the HelmChart
# above has actually installed the chart, so it's templated in the same
# pass rather than gated behind a "wait for CRDs" step: k3s's deploy
# controller retries a manifest referencing not-yet-existing CRDs until they
# show up, instead of failing once and giving up.
- name: Fail fast if no MetalLB address range is configured
ansible.builtin.assert:
that:
- k3s_metallb_address_range | length > 0
fail_msg: >-
k3s_metallb_address_range is empty — set it in
inventory/group_vars/k3s_cluster.yml before running playbooks/k3s.yml.
run_once: true
- name: Deploy the MetalLB HelmChart manifest
ansible.builtin.template:
src: metallb.helmchart.yaml.j2
dest: /var/lib/rancher/k3s/server/manifests/metallb.yaml
owner: root
group: root
mode: "0644"
become: true
- name: Deploy the MetalLB address pool config
ansible.builtin.template:
src: metallb-config.yaml.j2
dest: /var/lib/rancher/k3s/server/manifests/metallb-config.yaml
owner: root
group: root
mode: "0644"
become: true

26
build/config/ansible/roles/k3s_metallb/templates/metallb-config.yaml.j2 Normal file
View file

@ -0,0 +1,26 @@
# Managed by Ansible (roles/k3s_metallb) — do not edit on the node.
#
# Plain manifests, not a HelmChart — these are MetalLB's own CRs, not chart
# values. k3s's deploy controller applies anything dropped in this
# directory, not only HelmChart CRs, and retries until the CRDs the
# HelmChart alongside this file installs actually exist.
#
# L2 (ARP/NDP) mode, not BGP — the simplest fit for a flat home LAN with no
# BGP-speaking router. One pool, one advertisement, both named "default".
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
name: default
namespace: {{ k3s_metallb_namespace }}
spec:
addresses:
- {{ k3s_metallb_address_range }}
---
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
name: default
namespace: {{ k3s_metallb_namespace }}
spec:
ipAddressPools:
- default

18
build/config/ansible/roles/k3s_metallb/templates/metallb.helmchart.yaml.j2 Normal file
View file

@ -0,0 +1,18 @@
# Managed by Ansible (roles/k3s_metallb) — do not edit on the node.
#
# A HelmChart CR, same pattern as
# roles/k3s_monitoring/templates/kube-prometheus-stack.helmchart.yaml.j2 —
# k3s's bundled helm-controller reconciles it, the same mechanism it uses to
# install its own bundled Traefik and ServiceLB. No helm binary, kubeconfig,
# or extra Ansible collection needed on the controller.
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
name: metallb
namespace: kube-system
spec:
chart: metallb
repo: {{ k3s_metallb_chart_repo }}
version: "{{ k3s_metallb_chart_version }}"
targetNamespace: {{ k3s_metallb_namespace }}
createNamespace: true

86
build/config/ansible/roles/k3s_monitoring/defaults/main.yml Normal file
View file

@ -0,0 +1,86 @@
---
# Ansible-side knobs for roles/k3s_monitoring — the kube-prometheus-stack
# Helm chart, deployed via k3s's own bundled helm-controller rather than a
# helm binary or the kubernetes.core collection (see tasks/main.yml). Lean by
# design: OpenLens's Metrics feature reads straight off the Prometheus API
# through the k8s API server proxy, so there's no need for Grafana or
# Alertmanager in-cluster — both are disabled below, mainly to save RAM on
# the Pis.
k3s_monitoring_namespace: monitoring
k3s_monitoring_chart_repo: https://prometheus-community.github.io/helm-charts
# Pinned, same reasoning as k3s_version in inventory/group_vars/k3s_cluster.yml
# — a rebuild months from now should reproduce today's stack, not whatever's
# newest at the time. Bump deliberately; check the current release first at
# https://github.com/prometheus-community/helm-charts/releases (tags look
# like kube-prometheus-stack-X.Y.Z).
k3s_monitoring_chart_version: "88.3.0"
# Handed to the chart verbatim as valuesContent (see templates/). Notes on
# the less obvious choices:
# - grafana/alertmanager: off. OpenLens supplies its own dashboards and
# this cluster doesn't page anyone, so both would just be extra Pi RAM.
# - kubeControllerManager/kubeScheduler/kubeProxy/kubeEtcd: off. k3s bundles
# the control plane inside one static binary instead of exposing these as
# separate systemd units on their usual ports, so kube-prometheus-stack's
# default scrape targets for them sit permanently "down" — a known false
# alarm on k3s, not a sign anything's actually broken. kubelet (and the
# cAdvisor/node metrics it serves) is the one k3s does expose normally,
# and it's also the one OpenLens's node/pod metrics actually need.
# - prometheus retention/storage: short retention, no PVC template (so it
# runs on emptyDir). This stack exists to answer "what are the nodes
# doing right now", not to keep months of history, so losing the TSDB on
# a pod restart is an acceptable trade for not standing up persistent
# storage across 4 SD cards.
# - every component's resources: sized for a Raspberry Pi 4, not a
# datacenter node.
k3s_monitoring_values:
grafana:
enabled: false
alertmanager:
enabled: false
kubeApiServer:
enabled: true
kubeControllerManager:
enabled: false
kubeScheduler:
enabled: false
kubeProxy:
enabled: false
kubeEtcd:
enabled: false
kubelet:
enabled: true
prometheusOperator:
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
memory: 128Mi
prometheus:
prometheusSpec:
retention: 3d
scrapeInterval: 30s
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
memory: 512Mi
kube-state-metrics:
resources:
requests:
cpu: 20m
memory: 32Mi
limits:
memory: 64Mi
prometheus-node-exporter:
resources:
requests:
cpu: 10m
memory: 16Mi
limits:
memory: 32Mi

16
build/config/ansible/roles/k3s_monitoring/tasks/main.yml Normal file
View file

@ -0,0 +1,16 @@
---
# Drops a HelmChart CR into k3s's auto-deploying manifests directory on the
# control-plane node; k3s's bundled helm-controller reconciles it, the same
# mechanism it uses to install its own bundled Traefik and ServiceLB. See
# templates/kube-prometheus-stack.helmchart.yaml.j2 for why this needs no
# helm binary or extra Ansible collection, and README.md "Metrics
# (Prometheus)" for how to point OpenLens at the result.
- name: Deploy the kube-prometheus-stack HelmChart manifest
ansible.builtin.template:
src: kube-prometheus-stack.helmchart.yaml.j2
dest: /var/lib/rancher/k3s/server/manifests/kube-prometheus-stack.yaml
owner: root
group: root
mode: "0644"
become: true

24
build/config/ansible/roles/k3s_monitoring/templates/kube-prometheus-stack.helmchart.yaml.j2 Normal file
View file

@ -0,0 +1,24 @@
---
# Managed by Ansible (roles/k3s_monitoring) — do not edit on the node.
#
# A HelmChart CR, not a `helm install`: k3s ships its own helm-controller
# that watches /var/lib/rancher/k3s/server/manifests/ (this file's
# destination) the same way it watches for any other auto-deploying
# manifest, and reconciles HelmChart resources found there — the same
# mechanism k3s uses to install its own bundled Traefik and ServiceLB. That
# means no helm binary, kubeconfig, or extra Ansible collection is needed on
# the controller; re-running the playbook just re-templates this file and
# the controller reconciles the diff.
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
name: kube-prometheus-stack
namespace: kube-system
spec:
chart: kube-prometheus-stack
repo: {{ k3s_monitoring_chart_repo }}
version: "{{ k3s_monitoring_chart_version }}"
targetNamespace: {{ k3s_monitoring_namespace }}
createNamespace: true
valuesContent: |
{{ k3s_monitoring_values | to_nice_yaml(indent=2) | indent(4, first=true) }}

18
build/config/ansible/roles/k3s_node/defaults/main.yml Normal file
View file

@ -0,0 +1,18 @@
---
# Ubuntu Server's boot partition. Same path on Raspberry Pi OS, so this
# doesn't need to change if the Pis are ever reimaged to that instead.
k3s_boot_cmdline_path: /boot/firmware/cmdline.txt
# k3s (like any Kubernetes distro) needs the memory cgroup controller, which
# isn't always on by default on a Pi kernel. Ubuntu's Pi images usually ship
# these already — this task is a safety net for that assumption, not the
# primary mechanism, and a no-op on most runs. See tasks/prep.yml.
k3s_cgroup_params:
- cgroup_memory=1
- cgroup_enable=memory
# Safety-net defaults for inventory/group_vars/k3s_cluster.yml's extra-args
# vars (see tasks/server.yml, tasks/agent.yml) — empty so the role doesn't
# break if a var is left undefined there.
k3s_extra_args: []
k3s_server_extra_args: []

100
build/config/ansible/roles/k3s_node/tasks/agent.yml Normal file
View file

@ -0,0 +1,100 @@
---
# Joins this node as a k3s agent (worker). Only ever runs after server.yml has
# succeeded somewhere — playbooks/k3s.yml targets k3s_control_plane before
# k3s_workers — so K3S_URL below always points at an already-live API server.
#
# --node-name pins the k8s node object to the Ansible inventory_hostname —
# see the matching comment in server.yml, same reason.
- name: Bootstrap the k3s agent
no_log: true # K3S_TOKEN passes through this block
block:
- name: Look up the k3s cluster secrets
ansible.builtin.set_fact:
k3s_secrets: >-
{{ lookup('community.hashi_vault.vault_kv2_get',
k3s_vault_path,
engine_mount_point=vault_kv_mount,
url=vault_addr,
auth_method=vault_auth_method,
role_id=vault_role_id | default(omit),
secret_id=vault_secret_id | default(omit)).secret }}
- name: Compute the desired k3s agent exec line
ansible.builtin.set_fact:
k3s_agent_exec: "agent --node-name {{ inventory_hostname }} {{ k3s_extra_args | join(' ') }}"
# Compared against what k3s was last installed with (see the copy task
# below) so a change to k3s_extra_args gets applied on the next run
# instead of silently sitting unused: the version check below has no way
# to notice an exec-line-only change. Missing file (first install) counts
# as changed.
- name: Read the exec line k3s was last installed with
ansible.builtin.slurp:
src: /etc/rancher/k3s/.ansible_install_exec
register: k3s_installed_exec_raw
failed_when: false
check_mode: false
- name: Determine whether the exec line has changed
ansible.builtin.set_fact:
k3s_exec_changed: >-
{{ k3s_installed_exec_raw.content is not defined
or (k3s_installed_exec_raw.content | b64decode) != k3s_agent_exec }}
- name: Check the installed k3s version
ansible.builtin.command: k3s --version
register: k3s_installed_version
changed_when: false
failed_when: false
check_mode: false
# The installer is safe to re-run — it's a no-op if the requested version
# and exec line are already active — but skipping it when neither changed
# avoids restarting the agent (and briefly dropping the node's kubelet)
# on every playbook run. Restarting the k3s-agent process itself (as
# opposed to rebooting the node, which roles/k3s_maintenance handles
# separately) doesn't touch already-running pods — containerd keeps them
# up underneath it. playbooks/k3s.yml still runs this play with
# serial: 1, so at most one node's kubelet is ever bouncing at a time.
- name: Install/upgrade k3s agent
ansible.builtin.shell: curl -sfL https://get.k3s.io | sh -
environment:
INSTALL_K3S_VERSION: "{{ k3s_version }}"
INSTALL_K3S_EXEC: "{{ k3s_agent_exec }}"
# The one control-plane node, addressed by its inventory IP rather
# than delegating a lookup to it — simple and correct as long as this
# cluster stays single-server. Revisit if it ever gets HA control
# plane nodes.
K3S_URL: "https://{{ hostvars[groups['k3s_control_plane'][0]].ansible_host }}:6443"
K3S_TOKEN: "{{ k3s_secrets.K3S_TOKEN }}"
when: >-
k3s_installed_version.rc != 0
or k3s_version not in k3s_installed_version.stdout
or k3s_exec_changed
- name: Ensure the k3s-agent service is enabled and running
ansible.builtin.systemd_service:
name: k3s-agent
enabled: true
state: started
# k3s creates /etc/rancher/k3s itself once it has something to put there
# (a config.yaml, the generated kubeconfig on the server) — an agent with no
# extra config doesn't necessarily end up with anything else prompting that,
# so this marker can't assume the directory already exists.
- name: Ensure /etc/rancher/k3s exists
ansible.builtin.file:
path: /etc/rancher/k3s
state: directory
owner: root
group: root
mode: "0755"
- name: Record the exec line k3s was installed with
ansible.builtin.copy:
dest: /etc/rancher/k3s/.ansible_install_exec
content: "{{ k3s_agent_exec }}"
owner: root
group: root
mode: "0600"

24
build/config/ansible/roles/k3s_node/tasks/main.yml Normal file
View file

@ -0,0 +1,24 @@
---
# Converges one node of the cluster: the same prep either way, then branch on
# which k3s role it plays. k3s_node_role is set per group, not per host — see
# inventory/group_vars/k3s_control_plane.yml (server) and k3s_workers.yml
# (agent) — so adding a fifth Pi later is a hosts.yml edit, not a role change.
- name: Validate k3s_node_role
ansible.builtin.assert:
that: k3s_node_role in ['server', 'agent']
fail_msg: >-
k3s_node_role is '{{ k3s_node_role }}'; expected 'server' or 'agent'.
Check inventory/group_vars/k3s_control_plane.yml and k3s_workers.yml.
quiet: true
- name: Prepare the node for k3s
ansible.builtin.include_tasks: prep.yml
- name: Install the k3s server (control plane)
ansible.builtin.include_tasks: server.yml
when: k3s_node_role == 'server'
- name: Install the k3s agent (worker)
ansible.builtin.include_tasks: agent.yml
when: k3s_node_role == 'agent'

80
build/config/ansible/roles/k3s_node/tasks/prep.yml Normal file
View file

@ -0,0 +1,80 @@
---
# OS-level prerequisites, identical for a server and an agent node. Runs
# before either install so a from-scratch Pi (fresh Ubuntu Server image, SSH
# + the `ansible` user already set up) can go straight to a working cluster
# in one playbook run — the "easy to rebuild" part of the design.
- name: Read the current kernel boot parameters
ansible.builtin.command: cat {{ k3s_boot_cmdline_path }}
register: k3s_cmdline_current
changed_when: false
check_mode: false
- name: Work out which cgroup parameters are missing
ansible.builtin.set_fact:
k3s_cmdline_missing: >-
{{ k3s_cgroup_params | reject('in', k3s_cmdline_current.stdout) | list }}
# cmdline.txt is one line, space-separated — rewritten whole rather than
# appended in place, since there's no line-based anchor to insert after.
# Existing file mode is left alone (no `mode:` here) rather than guessed at.
- name: Add missing cgroup parameters to the boot command line
ansible.builtin.copy:
dest: "{{ k3s_boot_cmdline_path }}"
content: >-
{{ (k3s_cmdline_current.stdout.split() + k3s_cmdline_missing) | join(' ') }}
owner: root
group: root
register: k3s_cmdline_updated
when: k3s_cmdline_missing | length > 0
- name: Reboot to apply updated boot parameters
ansible.builtin.reboot:
reboot_timeout: 300
when: k3s_cmdline_updated is changed
- name: Check active swap devices
ansible.builtin.command: swapon --summary
register: k3s_swap_active
changed_when: false
check_mode: false
- name: Turn off active swap
ansible.builtin.command: swapoff -a
when: k3s_swap_active.stdout | trim | length > 0
- name: Comment out swap entries in fstab
ansible.builtin.replace:
path: /etc/fstab
regexp: '^([^#\n]*\sswap\s.*)$'
replace: '# \1'
# Best-effort: Ubuntu's zram-backed swap ships under different unit names
# across releases, and most won't be present at all. A missing unit is not a
# failure here — only an already-active one that we failed to disable would
# leave swap coming back on the next boot, and swapoff -a above already
# handles the running instance for this boot.
#
# `failed_when: false` rather than `ignore_errors: true`: systemd_service
# raises "Could not find the requested service" as a hard module failure
# when the unit is absent, and that's reported here as a genuine task
# failure regardless of ignore_errors — failed_when overrides the result
# directly instead of trying to catch it after the fact.
- name: Disable Ubuntu's zram-backed swap, if present
ansible.builtin.systemd_service:
name: "{{ item }}"
enabled: false
state: stopped
loop:
- zram-config.service
- systemd-zram-setup@zram0.service
register: k3s_zram_disable
failed_when: false
changed_when: k3s_zram_disable is changed
- name: Ensure curl is installed
ansible.builtin.apt:
name: curl
state: present
update_cache: true
cache_valid_time: 3600

103
build/config/ansible/roles/k3s_node/tasks/server.yml Normal file
View file

@ -0,0 +1,103 @@
---
# Bootstraps this node as the (single) k3s control plane. playbooks/k3s.yml
# runs the k3s_control_plane play before k3s_workers, so by the time
# agent.yml runs anywhere, K3S_URL below already points at something live.
#
# The token comes from Vault as a fixed, pre-shared value rather than letting
# k3s generate one on first install — see docs/vault-secrets.md
# (homelab/k3s-homelab-utils). That's what makes a full rebuild (wipe both SD
# cards, reinstall) reproduce the same cluster identity instead of needing the
# new token hunted down and re-distributed by hand.
#
# --node-name pins the k8s node object to the Ansible inventory_hostname
# rather than whatever the OS hostname happens to be — roles/k3s_maintenance
# addresses nodes by inventory_hostname when draining/uncordoning, and that
# only works if the two names are guaranteed to match.
- name: Bootstrap the k3s server
no_log: true # K3S_TOKEN passes through this block
block:
- name: Look up the k3s cluster secrets
ansible.builtin.set_fact:
k3s_secrets: >-
{{ lookup('community.hashi_vault.vault_kv2_get',
k3s_vault_path,
engine_mount_point=vault_kv_mount,
url=vault_addr,
auth_method=vault_auth_method,
role_id=vault_role_id | default(omit),
secret_id=vault_secret_id | default(omit)).secret }}
- name: Compute the desired k3s server exec line
ansible.builtin.set_fact:
k3s_server_exec: >-
server --node-name {{ inventory_hostname }} --tls-san {{ k3s_api_tls_san }}
--write-kubeconfig-mode 644 {{ (k3s_extra_args + k3s_server_extra_args) | join(' ') }}
# Compared against what k3s was last installed with (see the copy task
# below) so a change to k3s_extra_args/k3s_server_extra_args — or
# k3s_api_tls_san — gets applied on the next run instead of silently
# sitting unused: the version check below has no way to notice an
# exec-line-only change. Missing file (first install) counts as changed.
- name: Read the exec line k3s was last installed with
ansible.builtin.slurp:
src: /etc/rancher/k3s/.ansible_install_exec
register: k3s_installed_exec_raw
failed_when: false
check_mode: false
- name: Determine whether the exec line has changed
ansible.builtin.set_fact:
k3s_exec_changed: >-
{{ k3s_installed_exec_raw.content is not defined
or (k3s_installed_exec_raw.content | b64decode) != k3s_server_exec }}
- name: Check the installed k3s version
ansible.builtin.command: k3s --version
register: k3s_installed_version
changed_when: false
failed_when: false
check_mode: false
# The installer is safe to re-run — it's a no-op if the requested version
# and exec line are already active — but skipping it when neither changed
# avoids restarting the service (and briefly dropping the API) on every
# playbook run. Restarting the k3s process itself (as opposed to
# rebooting the node, which roles/k3s_maintenance handles separately) is
# a brief control-plane/kubelet blip, not a pod outage — containerd keeps
# every already-running pod up underneath it.
- name: Install/upgrade k3s server
ansible.builtin.shell: curl -sfL https://get.k3s.io | sh -
environment:
INSTALL_K3S_VERSION: "{{ k3s_version }}"
INSTALL_K3S_EXEC: "{{ k3s_server_exec }}"
K3S_TOKEN: "{{ k3s_secrets.K3S_TOKEN }}"
when: >-
k3s_installed_version.rc != 0
or k3s_version not in k3s_installed_version.stdout
or k3s_exec_changed
- name: Ensure the k3s service is enabled and running
ansible.builtin.systemd_service:
name: k3s
enabled: true
state: started
# k3s creates /etc/rancher/k3s itself once it has something to put there
# (the generated kubeconfig, on the server) — defensive rather than relied
# on, so this marker doesn't assume the directory already exists.
- name: Ensure /etc/rancher/k3s exists
ansible.builtin.file:
path: /etc/rancher/k3s
state: directory
owner: root
group: root
mode: "0755"
- name: Record the exec line k3s was installed with
ansible.builtin.copy:
dest: /etc/rancher/k3s/.ansible_install_exec
content: "{{ k3s_server_exec }}"
owner: root
group: root
mode: "0600"

71
build/config/ansible/roles/k3s_postgres/defaults/main.yml Normal file
View file

@ -0,0 +1,71 @@
---
# Ansible-side knobs for roles/k3s_postgres — CloudNativePG (CNPG), the
# Postgres operator for the homelab-utils cluster's shared database.
# Installed via k3s's own bundled helm-controller, same mechanism as
# roles/k3s_metallb and roles/k3s_monitoring (see tasks/main.yml) — no helm
# binary or extra Ansible collection needed on the controller.
#
# Chosen over Bitnami's postgresql-ha (repmgr+pgpool, more moving parts per
# pod, and Bitnami's free chart/image catalog was restructured into a
# "legacy" repo in 2025 — a risky long-term dependency) and the Zalando
# operator (mature but Patroni-based, heavier, and its own config
# conventions fit less naturally into this repo's lean HelmChart-CR-only
# pattern). CNPG installs as one operator chart plus one plain Cluster CR —
# the same "HelmChart + plain config manifest" shape already used for
# MetalLB (chart + IPAddressPool).
#
# The app-portable half of this config (instance count, storage size/class)
# lives with the app it belongs to instead: see
# src/shared/postgres/ansible/kubernetes/vars.yml, loaded into `pg_config`
# by tasks/main.yml — same common/vars.yml + platform-vars.yml split
# compose_stack and lxc_app use for every other app.
k3s_postgres_operator_namespace: cnpg-system
k3s_postgres_chart_repo: https://cloudnative-pg.github.io/charts
# Pinned, same reasoning as k3s_monitoring_chart_version/k3s_metallb_chart_version
# — a rebuild months from now should reproduce today's operator, not
# whatever's newest at the time. Bump deliberately; check the current
# release first at https://github.com/cloudnative-pg/charts/releases.
k3s_postgres_chart_version: "0.29.0"
# Where the actual Postgres Cluster (not the operator) lives.
k3s_postgres_namespace: shared-postgres
# Postgres major/minor pinned via the operand image, independent of the
# chart/operator version above. This is a separate physical instance from
# the Docker-based shared/postgres (pinned to 13 there — see
# src/shared/postgres/common/vars.yml), so there's no need to match; pin to
# a current stable major instead. Multi-arch (amd64/arm64) upstream, same
# as every other image this cluster runs.
k3s_postgres_image: ghcr.io/cloudnative-pg/postgresql:18.4
# Same Vault path the Unraid/Proxmox shared-postgres instances already use
# (see docs/vault-secrets.md) — a separate physical instance, but one
# superuser identity for the "shared postgres" concept everywhere it runs.
k3s_postgres_vault_path: homelab/shared/postgres
# The LAN address the primary is published on, via the LoadBalancer Service
# declared in templates/postgres-cluster.yaml.j2. Must be inside
# k3s_metallb_address_range — MetalLB only assigns from its own pools, and a
# request for anything outside them leaves the Service pending forever rather
# than failing loudly.
#
# Empty here on purpose, same reasoning as k3s_metallb_address_range's own
# default: the address is a property of the LAN, not of this role, so it's set
# in inventory/group_vars/k3s_cluster.yml next to the pool it has to fall
# inside. Unlike the pool, leaving it empty is survivable — the Service is
# still created and MetalLB assigns the next free address; you just don't know
# which one until you look, which is fine for in-cluster clients and no good
# for anything that has to write the address down.
k3s_postgres_loadbalancer_ip: ""
# Resource requests/limits per instance — sized for a Raspberry Pi 4, not a
# datacenter node, same reasoning as k3s_monitoring_values.
k3s_postgres_resources:
requests:
cpu: 100m
memory: 256Mi
limits:
memory: 512Mi

91
build/config/ansible/roles/k3s_postgres/tasks/main.yml Normal file
View file

@ -0,0 +1,91 @@
---
# Deploys the shared Postgres cluster onto k3s: install CloudNativePG (the
# operator) via a HelmChart CR, then define the actual cluster as a plain
# Cluster CR — same two-manifest pattern as roles/k3s_metallb (chart CR +
# plain config manifest), for the same reason: k3s's deploy controller
# retries a manifest referencing CRDs the HelmChart above hasn't installed
# yet instead of failing outright, so the Cluster CR doesn't need to wait on
# the operator chart finishing first.
#
# Config is layered the same way compose_stack/lxc_app layer an app's
# common/vars.yml + platform vars.yml — src/shared/postgres/ is the same
# shared service already used on Unraid/Proxmox, just deployed a third way
# here. Secrets come from the same Vault path those platforms already use
# (homelab/shared/postgres) — this is a separate physical instance, but
# reuses the path rather than inventing a k3s-specific one, same "one path
# per app-concept" convention as every other stack.
- name: Set shared-postgres source facts
ansible.builtin.set_fact:
pg_local_dir: "{{ repo_root }}/src/shared/postgres"
- name: Load portable shared-postgres variables
ansible.builtin.include_vars:
file: "{{ pg_local_dir }}/common/vars.yml"
name: pg_common_vars
- name: Load Kubernetes-specific shared-postgres variables
ansible.builtin.include_vars:
file: "{{ pg_local_dir }}/ansible/kubernetes/vars.yml"
name: pg_platform_vars
- name: Merge shared-postgres configuration
ansible.builtin.set_fact:
pg_config: >-
{{ (pg_common_vars.env_defaults | default({}))
| combine(pg_platform_vars.env_defaults | default({})) }}
- name: Look up shared-postgres superuser credentials from Vault
ansible.builtin.set_fact:
pg_vault_secrets: >-
{{ lookup('community.hashi_vault.vault_kv2_get',
k3s_postgres_vault_path,
engine_mount_point=vault_kv_mount,
url=vault_addr,
auth_method=vault_auth_method,
role_id=vault_role_id | default(omit),
secret_id=vault_secret_id | default(omit)).secret }}
no_log: true
- name: Deploy the CloudNativePG operator HelmChart manifest
ansible.builtin.template:
src: cnpg-operator.helmchart.yaml.j2
dest: /var/lib/rancher/k3s/server/manifests/cnpg-operator.yaml
owner: root
group: root
mode: "0644"
become: true
# Namespace + Cluster CR, no secrets — safe to render even though the
# Cluster CR references the Secret below, for the same "later file, retried
# reconcile" reasoning as the Secret referencing this file's Namespace.
- name: Deploy the shared-postgres Cluster manifest
ansible.builtin.template:
src: postgres-cluster.yaml.j2
dest: /var/lib/rancher/k3s/server/manifests/shared-postgres-cluster.yaml
owner: root
group: root
mode: "0644"
become: true
# Separate file (and mode) from the Cluster manifest above purely because
# this one carries the Vault-sourced password — everything else here is safe
# at 0644, this one isn't.
#
# The dest filename deliberately doesn't track the Secret's own name, which
# gained a `-vault` suffix (see the template's header for why). k3s's deploy
# controller records which resources each manifest *file* created and prunes
# the ones that disappear from it, so renaming the Secret inside this same
# file makes the old `shared-postgres-superuser` object get garbage-collected
# on the next run. Renaming the file too would orphan it instead: Ansible
# doesn't remove files it no longer writes, so the old manifest would sit
# there keeping the stale Secret alive.
- name: Deploy the shared-postgres superuser Secret
ansible.builtin.template:
src: postgres-superuser-secret.yaml.j2
dest: /var/lib/rancher/k3s/server/manifests/shared-postgres-superuser-secret.yaml
owner: root
group: root
mode: "0600"
become: true
no_log: true

21
build/config/ansible/roles/k3s_postgres/templates/cnpg-operator.helmchart.yaml.j2 Normal file
View file

@ -0,0 +1,21 @@
# Managed by Ansible (roles/k3s_postgres) — do not edit on the node.
#
# A HelmChart CR, same pattern as
# roles/k3s_metallb/templates/metallb.helmchart.yaml.j2 and
# roles/k3s_monitoring — k3s's bundled helm-controller reconciles it, the
# same mechanism it uses to install its own bundled Traefik and ServiceLB.
# No helm binary, kubeconfig, or extra Ansible collection needed on the
# controller. This installs the CloudNativePG operator and its CRDs only —
# the actual Postgres cluster is templates/postgres-cluster.yaml.j2, a plain
# manifest applied once the CRD it references exists (see that file).
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
name: cloudnative-pg
namespace: kube-system
spec:
chart: cloudnative-pg
repo: {{ k3s_postgres_chart_repo }}
version: "{{ k3s_postgres_chart_version }}"
targetNamespace: {{ k3s_postgres_operator_namespace }}
createNamespace: true

104
build/config/ansible/roles/k3s_postgres/templates/postgres-cluster.yaml.j2 Normal file
View file

@ -0,0 +1,104 @@
# Managed by Ansible (roles/k3s_postgres) — do not edit on the node.
#
# Plain manifests, not a HelmChart — same pattern as
# roles/k3s_metallb/templates/metallb-config.yaml.j2: the Cluster CR below
# references a CRD (postgresql.cnpg.io) that only exists once the
# cloudnative-pg HelmChart (cnpg-operator.yaml, same manifests directory) has
# actually installed the operator, so it's applied here rather than gated
# behind a "wait for CRDs" step — k3s's deploy controller retries a manifest
# referencing not-yet-existing CRDs until they show up, instead of failing
# once and giving up. templates/postgres-superuser-secret.yaml.j2 relies on
# the Namespace created here the same way, for the same reason.
apiVersion: v1
kind: Namespace
metadata:
name: {{ k3s_postgres_namespace }}
---
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
name: shared-postgres
namespace: {{ k3s_postgres_namespace }}
spec:
# 1 primary + 1 replica (src/shared/postgres/ansible/kubernetes/vars.yml).
# CNPG's own default pod anti-affinity (preferred, topology key hostname)
# spreads them across distinct nodes; the nodeAffinity below narrows which
# nodes those can be.
instances: {{ pg_config.POSTGRES_INSTANCES }}
imageName: {{ k3s_postgres_image }}
# Password-based superuser login, from the Secret Ansible renders
# alongside this file. Off by default in CNPG — without this, the operator
# ignores the secret's content and sets the postgres user's password to
# NULL, disabling remote login as it.
#
# The name must stay clear of `<cluster>-superuser` (i.e.
# `shared-postgres-superuser`), which is what CNPG calls the secret it
# generates for itself when this stanza is absent. Claiming that name for
# our own object stops password changes reaching the database — see
# postgres-superuser-secret.yaml.j2 for the full symptom.
enableSuperuserAccess: true
superuserSecret:
name: shared-postgres-superuser-vault
# A LoadBalancer Service for the primary, on top of the ClusterIP -rw/-ro/-r
# Services CNPG creates for every Cluster. Declared here as a CNPG *managed
# service* rather than as a Service manifest of our own: selectorType: rw
# means the operator maintains the selector, so the LB follows a failover to
# the other instance the same way the built-in -rw Service does. A
# hand-written Service would need its selector re-pointed by hand after
# every promotion.
#
# This is what makes the cluster reachable from the Ansible controller at a
# fixed host:port, which is the thing per-app database provisioning needs —
# the community.postgresql tasks compose_stack/lxc_app use can't talk to a
# ClusterIP, and `kubectl port-forward` isn't a stable address. It does put
# the superuser on the LAN: acceptable here for the same reason the Unraid
# instance's published port is, and the LAN is the boundary either way.
managed:
services:
additional:
- selectorType: rw
serviceTemplate:
metadata:
name: shared-postgres-lb
{% if k3s_postgres_loadbalancer_ip %}
# Pinned rather than left to MetalLB's next-free pick, because
# this address ends up in config elsewhere (inventory, an app's
# vars.yml) and shouldn't move when the Service is recreated.
annotations:
metallb.universe.tf/loadBalancerIPs: "{{ k3s_postgres_loadbalancer_ip }}"
{% endif %}
spec:
type: LoadBalancer
# No ports: — the operator fills in Postgres's own (5432) from
# the same template it builds the -rw Service from; naming them
# here would only risk disagreeing with it.
# POSTGRES_PORT (common/vars.yml) is 5432, the same port CNPG always
# listens on inside the pod/Service — nothing to override here.
storage:
size: {{ pg_config.POSTGRES_STORAGE_SIZE }}
storageClass: {{ pg_config.POSTGRES_STORAGE_CLASS }}
resources:
{{ k3s_postgres_resources | to_nice_yaml(indent=2) | indent(4, first=true) }}
# Workers only — k3s-ctrl-01 stays free of app pods, same boundary the
# cluster already keeps for monitoring/MetalLB.
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/control-plane
operator: DoesNotExist
# No bootstrap: stanza — CNPG's default initdb bootstrap creates a
# `postgres` superuser (above) plus a default `app` database owned by an
# auto-generated `app` role/secret. That default app database is unused
# today: no app is deployed onto k3s yet, and real per-app database
# provisioning here is a deferred design problem (see CLAUDE.md → "Key
# decisions") — the community.postgresql approach compose_stack/lxc_app
# use needs a controller-reachable host:port, which this cluster's
# in-cluster -rw Service isn't without further work.

49
build/config/ansible/roles/k3s_postgres/templates/postgres-superuser-secret.yaml.j2 Normal file
View file

@ -0,0 +1,49 @@
# Managed by Ansible (roles/k3s_postgres) — do not edit on the node.
#
# Superuser credentials for the shared-postgres Cluster
# (postgres-cluster.yaml.j2), sourced from the same Vault path
# (homelab/shared/postgres) the Unraid/Proxmox shared-postgres instances
# already use — see docs/vault-secrets.md. CNPG requires
# `type: kubernetes.io/basic-auth` with username/password keys, and
# `enableSuperuserAccess: true` on the Cluster (set there) for this secret to
# actually be usable for password login rather than just ignored.
#
# The `-vault` suffix is load-bearing, and removing it reintroduces a bug
# that is very hard to read off the symptoms. CNPG generates its own
# superuser secret named `<cluster>-superuser` when the Cluster doesn't name
# one — for a Cluster called `shared-postgres` that is exactly
# `shared-postgres-superuser`, the name this file used to claim. Writing our
# own object at the operator's reserved name meant the operator treated it as
# one it had already authored and reconciled, so a password change in Vault
# propagated as far as this Secret and stopped there: the role in the
# database kept whatever was set at bootstrap. Every check short of querying
# pg_authid agrees the change landed, while `psql` from the LAN keeps
# returning "password authentication failed for user postgres". Any name
# outside the `<cluster>-*` pattern the operator owns avoids it.
#
# Namespace comes from postgres-cluster.yaml.j2, a separate file in the same
# manifests directory — see that file's header for why applying this before
# the Namespace exists isn't a problem.
#
# The username is layered the same way every other value in this repo is —
# Vault on top, common/vars.yml underneath — rather than required from Vault.
# It isn't a secret (only the password is), and the Vault path is allowed to
# carry just POSTGRES_SUPERUSER_PASSWORD; setting POSTGRES_SUPERUSER there
# still wins, which is what docs/vault-secrets.md means by "Vault is
# authoritative since it's paired with the password".
apiVersion: v1
kind: Secret
metadata:
name: shared-postgres-superuser-vault
namespace: {{ k3s_postgres_namespace }}
type: kubernetes.io/basic-auth
# to_json, not bare interpolation: it emits a double-quoted YAML scalar with
# escaping handled, so a password containing `#`, `:`, a leading `*`/`&`, or
# leading/trailing whitespace survives intact. Unquoted, YAML silently
# reinterprets those (`#` starts a comment, an all-digit password becomes an
# int) and the Secret ends up holding something other than what Vault has —
# which fails authentication while every value still looks right in playbook
# output. Alphanumeric-only passwords hide the problem rather than fix it.
stringData:
username: {{ (pg_vault_secrets.POSTGRES_SUPERUSER | default(pg_config.POSTGRES_SUPERUSER)) | to_json }}
password: {{ pg_vault_secrets.POSTGRES_SUPERUSER_PASSWORD | to_json }}

115
build/config/ansible/roles/k3s_traefik/defaults/main.yml Normal file
View file

@ -0,0 +1,115 @@
---
# Ansible-side knobs for roles/k3s_traefik — the ingress controller k3s
# bundles. The odd one out among the k3s_* cluster services: every other one
# installs a chart this repo chose (roles/k3s_metallb, k3s_monitoring,
# k3s_postgres, k3s_cert_manager), whereas Traefik is already installed by
# k3s itself before Ansible ever connects. So this role installs nothing —
# it only adjusts what's there, via a HelmChartConfig (see
# templates/traefik.helmchartconfig.yaml.j2 for why that, and not an edit to
# k3s's own traefik.yaml).
#
# Scope today is the dashboard, which a stock k3s serves 404s for. That's
# not a broken install: Traefik builds the dashboard regardless
# (`api.dashboard` defaults on), but the Traefik chart stopped creating the
# router that reaches it in v28, so there is nothing routing /dashboard/ on
# any entrypoint until something puts that router back.
# Where k3s installs Traefik, and so where the HelmChartConfig has to live —
# a HelmChartConfig only applies to the HelmChart of the same name in the
# same namespace.
k3s_traefik_namespace: kube-system
# --- The dashboard, reachable by port-forward --------------------------------
#
# Re-enables the chart's own dashboard IngressRoute, which binds to Traefik's
# internal `traefik` entrypoint (port 9000). That entrypoint is not published
# on Traefik's Service, so this exposes nothing to the LAN; it makes exactly
# one thing work:
#
# kubectl -n kube-system port-forward deploy/traefik 9000:9000
# # then http://127.0.0.1:9000/dashboard/ — the trailing slash is required
#
# Left on even when the dashboard is also exposed on a hostname below, and
# that's deliberate rather than redundant. Once the exposed route is behind
# Authentik, the dashboard depends on Authentik being up, which depends on
# CNPG being up, which depends on the cluster being healthy — precisely the
# things you would open the dashboard to diagnose. Port-forward talks to the
# pod and traverses none of it, so it stays as the break-glass path. Don't
# "tidy this up" once the hostname works.
k3s_traefik_dashboard_enabled: true
# --- The dashboard, exposed on a hostname ------------------------------------
#
# Empty (the default) means the port-forward above is the only way in, and
# tasks/main.yml renders no IngressRoute at all — it removes one it previously
# rendered, so clearing this is a real teardown rather than a no-op.
#
# Set it to a hostname (e.g. traefik.turtlesystems.uk) to publish the
# dashboard on the `websecure` entrypoint with a cert-manager certificate and
# the forward-auth middleware below in front of it. Doing so requires
# k3s_traefik_dashboard_auth_address to be set as well — the role refuses to
# run otherwise, because the failure mode of getting this wrong is a
# read-only view of every route, service and middleware in the cluster
# published unauthenticated to the LAN.
#
# The hostname needs a DNS record pointing at Traefik's MetalLB address
# (`kubectl -n kube-system get svc traefik`), same as any app's Ingress.
k3s_traefik_dashboard_host: ""
# The ClusterIssuer roles/k3s_cert_manager creates. Must match
# k3s_cert_manager_issuer_name — stated rather than referenced because that
# lives in the other role's defaults, which aren't in scope here. Same
# convention as CERT_ISSUER in src/authentik/ansible/kubernetes/vars.yml.
#
# An explicit Certificate rather than an Ingress annotation, unlike every app
# on this cluster: cert-manager watches Ingress resources, and the dashboard
# can't be one. It's served by Traefik's internal `api@internal` service,
# which has no Kubernetes Service for an Ingress to point at — hence an
# IngressRoute, which cert-manager doesn't watch.
k3s_traefik_dashboard_cert_issuer: letsencrypt
k3s_traefik_dashboard_tls_secret: traefik-dashboard-tls
# --- Authentication in front of the exposed dashboard ------------------------
#
# A Traefik forwardAuth middleware pointing at Authentik, which now runs on
# this cluster (roles/k3s_app, src/authentik). No default, and required as
# soon as k3s_traefik_dashboard_host is set: a default here would be a guess
# at another app's service name, and a wrong guess that happens to be
# unreachable is a 500, not an open dashboard — but one that happens to
# resolve to the wrong thing is worse. Set it explicitly, next to the host,
# in inventory/group_vars/k3s_cluster.yml.
#
# For Authentik's embedded outpost, in-cluster:
#
# http://authentik-server.authentik.svc.cluster.local/outpost.goauthentik.io/auth/traefik
#
# The provider in Authentik must be a Proxy Provider in **forward auth
# (domain level)** mode, not single-application. Single-application mode
# additionally requires /outpost.goauthentik.io/ to be routed to Authentik on
# *this* host, which from an IngressRoute in kube-system means referencing a
# Service in the authentik namespace — a cross-namespace reference Traefik
# rejects unless started with providers.kubernetescrd.allowCrossNamespace,
# which it isn't. Domain-level mode handles the redirect on Authentik's own
# hostname instead and needs no such route, so it's the mode that fits.
k3s_traefik_dashboard_auth_address: ""
k3s_traefik_dashboard_auth_middleware: dashboard-auth
# What Authentik sets on the way back through, forwarded to the dashboard.
# The dashboard itself reads none of them — it has no notion of a user — so
# this is really about the headers existing for anything else that reuses
# this middleware later, and about matching Authentik's documented list
# rather than inventing a shorter one.
k3s_traefik_dashboard_auth_response_headers:
- X-authentik-username
- X-authentik-groups
- X-authentik-entitlements
- X-authentik-email
- X-authentik-name
- X-authentik-uid
- X-authentik-jwt
- X-authentik-meta-jwks
- X-authentik-meta-outpost
- X-authentik-meta-provider
- X-authentik-meta-app
- X-authentik-meta-version

59
build/config/ansible/roles/k3s_traefik/tasks/main.yml Normal file
View file

@ -0,0 +1,59 @@
---
# Adjusts the Traefik k3s installs for itself, rather than installing
# anything: a HelmChartConfig merged over k3s's own Traefik HelmChart, plus —
# only when the dashboard is being published on a hostname — the
# Certificate/Middleware/IngressRoute trio that puts it behind Authentik.
# See defaults/main.yml and templates/traefik.helmchartconfig.yaml.j2.
#
# Same delivery mechanism as every other k3s_* role: template a file into
# /var/lib/rancher/k3s/server/manifests/ and let k3s's bundled helm-controller
# and deploy controller reconcile it. No helm binary, kubeconfig or extra
# collection on the controller.
- name: Fail fast if the dashboard would be exposed without authentication
when: k3s_traefik_dashboard_host | length > 0
ansible.builtin.assert:
that:
- k3s_traefik_dashboard_auth_address | length > 0
fail_msg: >-
k3s_traefik_dashboard_host is set but k3s_traefik_dashboard_auth_address
is empty — set both in inventory/group_vars/k3s_cluster.yml, or neither.
Publishing the dashboard without the forward-auth middleware would put a
read-only view of every router, service and middleware on this cluster
on the LAN unauthenticated, so this refuses rather than defaulting to an
address that might not be Authentik.
quiet: true
run_once: true
- name: Deploy the Traefik HelmChartConfig
ansible.builtin.template:
src: traefik.helmchartconfig.yaml.j2
dest: /var/lib/rancher/k3s/server/manifests/traefik-config.yaml
owner: root
group: root
mode: "0644"
become: true
- name: Deploy the exposed dashboard route
when: k3s_traefik_dashboard_host | length > 0
ansible.builtin.template:
src: dashboard-ingressroute.yaml.j2
dest: /var/lib/rancher/k3s/server/manifests/traefik-dashboard.yaml
owner: root
group: root
mode: "0644"
become: true
# Clearing k3s_traefik_dashboard_host is a teardown, not just a stop-managing:
# k3s's deploy controller garbage-collects the objects a manifest created when
# the manifest is removed, so deleting this file withdraws the public route,
# its middleware and its certificate. Same reasoning as roles/k3s_app's
# removal path — with the difference that this one needs no `state: absent`
# tombstone, because the hostname is a single value rather than a list entry
# that could be silently dropped.
- name: Withdraw the exposed dashboard route when no host is configured
when: k3s_traefik_dashboard_host | length == 0
ansible.builtin.file:
path: /var/lib/rancher/k3s/server/manifests/traefik-dashboard.yaml
state: absent
become: true

83
build/config/ansible/roles/k3s_traefik/templates/dashboard-ingressroute.yaml.j2 Normal file
View file

@ -0,0 +1,83 @@
{#
Managed by Ansible (roles/k3s_traefik) — do not edit on the node.
Plain manifests, not a HelmChart — same pattern as
roles/k3s_metallb/templates/metallb-config.yaml.j2 and
roles/k3s_cert_manager/templates/cluster-issuer.yaml.j2: CRs belonging to
charts installed elsewhere, dropped in the auto-deploying directory and
retried by k3s's deploy controller until the CRDs they need exist.
Rendered only when k3s_traefik_dashboard_host is set; tasks/main.yml
deletes this file when it isn't, and k3s's deploy controller
garbage-collects what the file created.
traefik.io/v1alpha1, not traefik.containo.us/v1alpha1 — the group changed
with Traefik v3, which is what current k3s bundles.
-#}
{# Explicit Certificate rather than the cert-manager.io/cluster-issuer
annotation every app's Ingress uses: cert-manager watches Ingress
resources, and this route can't be one (see below). DNS-01, so this
issues without the hostname resolving anywhere yet. -#}
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: traefik-dashboard
namespace: {{ k3s_traefik_namespace }}
spec:
secretName: {{ k3s_traefik_dashboard_tls_secret }}
issuerRef:
name: {{ k3s_traefik_dashboard_cert_issuer }}
kind: ClusterIssuer
dnsNames:
- {{ k3s_traefik_dashboard_host }}
---
{# Authentik, in forward-auth (domain level) mode — see defaults/main.yml for
why domain level and not single-application. Fails closed: if Authentik is
down this returns an error rather than passing the request through, which
is the correct direction and the reason the port-forward path stays. -#}
apiVersion: traefik.io/v1alpha1
kind: Middleware
metadata:
name: {{ k3s_traefik_dashboard_auth_middleware }}
namespace: {{ k3s_traefik_namespace }}
spec:
forwardAuth:
address: {{ k3s_traefik_dashboard_auth_address }}
# Traefik strips X-Forwarded-* from client requests by default; Authentik
# needs them to know which host and scheme the user actually asked for,
# and builds its redirect back out of them. Safe here because the only
# thing that can set them is Traefik itself — nothing reaches this
# middleware without passing through the entrypoint first.
trustForwardHeader: true
authResponseHeaders:
{% for header in k3s_traefik_dashboard_auth_response_headers %}
- {{ header }}
{% endfor %}
---
{# An IngressRoute rather than an Ingress, and not by preference: the
dashboard is served by api@internal, a Traefik-internal service with no
Kubernetes Service behind it, so there is nothing for an Ingress backend
to name. That single fact is why the Certificate above is explicit and why
the middleware is attached here rather than by annotation. -#}
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
name: traefik-dashboard
namespace: {{ k3s_traefik_namespace }}
spec:
entryPoints:
- websecure
routes:
# Both prefixes, because the dashboard is a static bundle under
# /dashboard/ that reads its data from /api — serving the first without
# the second gets you a page that loads and then stays empty.
- kind: Rule
match: Host(`{{ k3s_traefik_dashboard_host }}`) && (PathPrefix(`/dashboard`) || PathPrefix(`/api`))
middlewares:
- name: {{ k3s_traefik_dashboard_auth_middleware }}
namespace: {{ k3s_traefik_namespace }}
services:
- kind: TraefikService
name: api@internal
tls:
secretName: {{ k3s_traefik_dashboard_tls_secret }}

34
build/config/ansible/roles/k3s_traefik/templates/traefik.helmchartconfig.yaml.j2 Normal file
View file

@ -0,0 +1,34 @@
{#
Managed by Ansible (roles/k3s_traefik) — do not edit on the node.
A HelmChartConfig, not a HelmChart, because this is the one chart on the
cluster this repo doesn't install: k3s installs Traefik itself, rewriting
/var/lib/rancher/k3s/server/manifests/traefik.yaml on every server start
and overwriting whatever is in it. A HelmChartConfig carrying the same
name and namespace as that HelmChart is the supported way in — k3s's
helm-controller merges its valuesContent over the chart's values on the
next reconcile. Editing traefik.yaml directly works until the next
restart, and then silently doesn't.
Hence also the filename this renders to (traefik-config.yaml): anything
called traefik.yaml here belongs to k3s and would be reclaimed.
-#}
apiVersion: helm.cattle.io/v1
kind: HelmChartConfig
metadata:
name: traefik
namespace: {{ k3s_traefik_namespace }}
spec:
valuesContent: |-
ingressRoute:
dashboard:
# Traefik builds the dashboard either way — `api.dashboard` is on by
# default. What the chart stopped shipping in v28 is the router that
# reaches it, which is why a stock k3s answers 404 to
# :9000/dashboard/ while :9000/ping happily returns 200. This puts
# that router back, on Traefik's internal `traefik` entrypoint
# (9000), which isn't published on the Service — so it is reachable
# by `kubectl port-forward` and nothing else. See defaults/main.yml
# for why that stays true even once the dashboard is also exposed on
# a hostname.
enabled: {{ k3s_traefik_dashboard_enabled | bool | lower }}

2
build/config/ansible/roles/lxc_app/defaults/main.yml Normal file
View file

@ -0,0 +1,2 @@
---
vault_kv_mount: kv

124
build/config/ansible/roles/lxc_app/tasks/main.yml Normal file
View file

@ -0,0 +1,124 @@
---
# Installs one app (one loop iteration of `app` from playbooks/proxmox.yml)
# natively into a Proxmox LXC: merge portable + Proxmox config, fetch secrets
# from Vault, provision its database if it declares one, run the app's own
# install steps, then manage its systemd unit.
#
# The Proxmox counterpart to `compose_stack`. Everything up to "Run
# app-specific install steps" is deliberately the same shape as that role —
# same vars merge, same Vault lookup, same DB provisioning — so config lives
# in one place regardless of which platform an app lands on. What differs is
# the deployment primitive: a systemd service built from packages/binaries
# rather than `docker compose up -d`.
#
# UNPROVEN: `src/shared/postgres/` and `src/forgejo/` both ship an
# `ansible/proxmox/install.yml` now, but neither has been run end to end
# against a real LXC. The structure mirrors proven code; the details are not
# yet exercised.
- name: Set app facts
ansible.builtin.set_fact:
app_local_dir: "{{ repo_root }}/src/{{ app.src }}"
- name: Load portable app variables
ansible.builtin.include_vars:
file: "{{ app_local_dir }}/common/vars.yml"
name: app_common_vars
- name: Load Proxmox-specific app variables
ansible.builtin.include_vars:
file: "{{ app_local_dir }}/ansible/proxmox/vars.yml"
name: app_platform_vars
- name: Merge app configuration
ansible.builtin.set_fact:
# Portable values first, Proxmox-specific overrides on top. Available to
# the app's install.yml, alongside `vault_secrets` below.
app_config: >-
{{ (app_common_vars.env_defaults | default({}))
| combine(app_platform_vars.env_defaults | default({})) }}
- name: Look up app secrets from Vault
ansible.builtin.set_fact:
vault_secrets: >-
{{ lookup('community.hashi_vault.vault_kv2_get',
app.vault_path,
engine_mount_point=vault_kv_mount,
url=vault_addr,
auth_method=vault_auth_method,
role_id=vault_role_id | default(omit),
secret_id=vault_secret_id | default(omit)).secret }}
no_log: true
- name: Provision application database
when: app.db is defined
no_log: true
block:
- name: Look up Postgres superuser credentials from Vault
ansible.builtin.set_fact:
pg_admin_secrets: >-
{{ lookup('community.hashi_vault.vault_kv2_get',
app.db.admin_vault_path,
engine_mount_point=vault_kv_mount,
url=vault_addr,
auth_method=vault_auth_method,
role_id=vault_role_id | default(omit),
secret_id=vault_secret_id | default(omit)).secret }}
- name: Ensure application database role exists
community.postgresql.postgresql_user:
name: "{{ app.db.user }}"
password: "{{ vault_secrets[app.db.password_vault_key] }}"
login_host: "{{ app.db.provision_host }}"
login_port: "{{ app.db.provision_port }}"
login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}"
login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}"
state: present
delegate_to: localhost
become: false
# Role first, then the database with `owner:` — the shape roles/k3s_app
# already uses, and not the `postgresql_user` + `priv: ALL` this role was
# written with. Two independent reasons it had to change:
#
# 1. `priv` was deprecated in community.postgresql 3.x and REMOVED in
# 4.0.0, so the old call is a hard "Unsupported parameters" failure
# on any current collection. requirements.yml asks for >=3.0.0,
# which installs 4.x.
# 2. On this instance it would have been wrong even if it still worked.
# `priv: ALL` grants database-level privileges (CONNECT, CREATE,
# TEMPORARY); since Postgres 15 the `public` schema no longer grants
# CREATE to PUBLIC, so a role holding all of those still cannot
# create a table. The Proxmox cluster is 17 (the Unraid one is 13,
# which is why the same code never failed that way there). Making
# the app role own the database covers it on both: on 15+ `public`
# is owned by `pg_database_owner`, which resolves to whoever owns
# the database.
#
# Ownership also makes a pg_dump restore land correctly, for the reason
# spelled out at the same tasks in roles/k3s_app.
- name: Ensure application database exists
community.postgresql.postgresql_db:
name: "{{ app.db.name }}"
owner: "{{ app.db.user }}"
login_host: "{{ app.db.provision_host }}"
login_port: "{{ app.db.provision_port }}"
login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}"
login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}"
state: present
delegate_to: localhost
become: false
# Everything genuinely app-specific — fetching the binary or package, creating
# the service user, laying out data directories, rendering the app's own
# config file from `app_config` + `vault_secrets`, and installing a systemd
# unit — lives with the app, not here. See src/<app>/ansible/proxmox/.
- name: Run app-specific install steps
ansible.builtin.include_tasks: "{{ app_local_dir }}/ansible/proxmox/install.yml"
- name: Enable and start the service
ansible.builtin.systemd_service:
name: "{{ app.service_name | default(app.name) }}"
enabled: true
state: started
daemon_reload: true

61
build/config/ansible/roles/pve_backup/defaults/main.yml Normal file
View file

@ -0,0 +1,61 @@
---
# Backup storage on the NAS, and the vzdump job that writes to it. Override
# per-node in inventory/host_vars/<node>.yml.
# Name the storage appears under in Proxmox. Also what `pve_backup_job`
# targets.
pve_backup_storage: nas-backup
# `nfs` or `pbs`. NFS is the plain option and needs nothing on the NAS beyond
# an export; PBS gets deduplication and incremental backups but needs a Proxmox
# Backup Server to exist first. See tasks/main.yml for what each consumes.
pve_backup_storage_type: nfs
# NFS (pve_backup_storage_type: nfs)
pve_backup_nfs_server: ""
pve_backup_nfs_export: ""
# 4.2 rather than 3: better locking and no separate portmapper.
pve_backup_nfs_options: vers=4.2
# PBS (pve_backup_storage_type: pbs)
pve_backup_pbs_server: ""
pve_backup_pbs_datastore: ""
pve_backup_pbs_username: ""
pve_backup_pbs_password: ""
pve_backup_pbs_fingerprint: ""
# How many archives the storage keeps. Proxmox prunes archives, not their
# contents — anything inside a guest (such as Forgejo's nightly pg_dump output)
# is pruned by that guest, on its own schedule.
pve_backup_prune:
keep-daily: 7
keep-weekly: 4
keep-monthly: 6
# systemd calendar event. Guests that dump a database into their own filesystem
# first must finish before this — see src/forgejo/ansible/proxmox/README.md.
pve_backup_schedule: "02:00"
# Stable marker used to find this job again on later runs. The Proxmox API
# assigns job IDs itself, so matching on the comment is what makes this
# idempotent rather than creating a duplicate job every deploy.
pve_backup_comment: managed-by-homelab-iac
# Which guests to back up. `true` covers everything on the node, including
# guests added later without an edit here; set it false and list VMIDs in
# pve_backup_vmids to be selective.
pve_backup_all: true
pve_backup_vmids: []
# snapshot: no downtime, and the guest filesystem is captured as if it had
# crashed at that instant. Fine for Forgejo — git repositories tolerate it, and
# the database is a separate consistent dump rather than live files.
pve_backup_mode: snapshot
pve_backup_compress: zstd
pve_backup_enabled: true
# Optional: address Proxmox mails backup reports to. Left empty means the node
# default applies. PVE 8 moved most of this to its notification-target system;
# `mailto` still works, but if you want anything richer, configure a
# notification target in the UI rather than extending this role.
pve_backup_mailto: ""

161
build/config/ansible/roles/pve_backup/tasks/main.yml Normal file
View file

@ -0,0 +1,161 @@
---
# Configures backups on a Proxmox node: a storage on the NAS to write archives
# to, and a scheduled vzdump job that fills it.
#
# This is the off-box copy for every Proxmox guest. Guests deliberately keep
# all their state on their own rootfs rather than bind-mounting anything in,
# because vzdump excludes bind mounts — see src/forgejo/terraform/README.md.
# Where an app's data lives outside the container (the shared Postgres), the
# app dumps it into its own filesystem on a timer that runs before the window
# below, so one archive is one restore point.
#
# Driven through `pvesm`/`pvesh` rather than modules: as of writing,
# `community.proxmox` has no storage or backup-job module. Swap them in if that
# changes — the shape here is create-if-absent, update-if-drifted, which is
# what a module would do anyway.
#
# UNVERIFIED — this has not been run against a real node. `pvesm add` and
# `pvesh create /cluster/backup` accept slightly different options across PVE
# versions; check yours with `pvesm help add`, `pvesh help create
# /cluster/backup`, and a `--check` run before trusting it.
- name: Check that the storage backend is configured
ansible.builtin.assert:
that:
- pve_backup_storage_type in ['nfs', 'pbs']
- pve_backup_storage_type != 'nfs' or (pve_backup_nfs_server | length > 0
and pve_backup_nfs_export | length > 0)
- pve_backup_storage_type != 'pbs' or (pve_backup_pbs_server | length > 0
and pve_backup_pbs_datastore | length > 0)
- pve_backup_all or (pve_backup_vmids | length > 0)
fail_msg: >-
pve_backup is missing required settings for
pve_backup_storage_type={{ pve_backup_storage_type }}. Set them in
inventory/host_vars/{{ inventory_hostname }}.yml.
quiet: true
- name: Build the prune-backups option string
ansible.builtin.set_fact:
pve_backup_prune_string: >-
{% for key, value in pve_backup_prune.items() %}{{ key }}={{ value }}{% if not loop.last %},{% endif %}{% endfor %}
# --- Storage -------------------------------------------------------------
- name: Read the configured storages
ansible.builtin.command:
argv: [pvesh, get, /storage, --output-format, json]
register: pve_storages
changed_when: false
check_mode: false
- name: Add the backup storage (NFS)
ansible.builtin.command:
argv:
- pvesm
- add
- nfs
- "{{ pve_backup_storage }}"
- --server
- "{{ pve_backup_nfs_server }}"
- --export
- "{{ pve_backup_nfs_export }}"
- --options
- "{{ pve_backup_nfs_options }}"
- --content
- backup
- --prune-backups
- "{{ pve_backup_prune_string }}"
when:
- pve_backup_storage_type == 'nfs'
- pve_backup_storage not in (pve_storages.stdout | from_json | map(attribute='storage') | list)
- name: Add the backup storage (PBS)
# The password goes on a command line, which is visible in the node's process
# list for the moment the command runs. `pvesm` has no stdin form, so the
# alternative is configuring PBS by hand — acceptable for a one-time create
# that only fires when the storage is absent.
ansible.builtin.command:
argv:
- pvesm
- add
- pbs
- "{{ pve_backup_storage }}"
- --server
- "{{ pve_backup_pbs_server }}"
- --datastore
- "{{ pve_backup_pbs_datastore }}"
- --username
- "{{ pve_backup_pbs_username }}"
- --password
- "{{ pve_backup_pbs_password }}"
- --fingerprint
- "{{ pve_backup_pbs_fingerprint }}"
- --content
- backup
- --prune-backups
- "{{ pve_backup_prune_string }}"
when:
- pve_backup_storage_type == 'pbs'
- pve_backup_storage not in (pve_storages.stdout | from_json | map(attribute='storage') | list)
no_log: true
# --- Backup job ----------------------------------------------------------
- name: Read the configured backup jobs
ansible.builtin.command:
argv: [pvesh, get, /cluster/backup, --output-format, json]
register: pve_backup_jobs
changed_when: false
check_mode: false
# Proxmox assigns job IDs itself, so there is nothing stable to key on except
# a comment we set. Without this the role would add a duplicate job on every
# run.
- name: Find a job this role already created
ansible.builtin.set_fact:
pve_backup_existing: >-
{{ (pve_backup_jobs.stdout | from_json)
| selectattr('comment', 'defined')
| selectattr('comment', 'equalto', pve_backup_comment)
| list | first | default({}) }}
- name: Build the vzdump job arguments
ansible.builtin.set_fact:
pve_backup_job_args: >-
{{ ['--schedule', pve_backup_schedule,
'--storage', pve_backup_storage,
'--mode', pve_backup_mode,
'--compress', pve_backup_compress,
'--comment', pve_backup_comment,
'--enabled', ('1' if pve_backup_enabled else '0')]
+ (['--all', '1'] if pve_backup_all
else ['--all', '0', '--vmid', pve_backup_vmids | join(',')])
+ (['--mailto', pve_backup_mailto] if pve_backup_mailto | length > 0 else []) }}
- name: Create the vzdump job
ansible.builtin.command:
argv: "{{ ['pvesh', 'create', '/cluster/backup'] + pve_backup_job_args }}"
when: pve_backup_existing | length == 0
# Compared field by field rather than blindly re-applying, so the role reports
# a change only when there is one. The VMID list is compared as the
# comma-joined string Proxmox stores, which makes it order-sensitive — reorder
# pve_backup_vmids and you get one no-op update.
- name: Work out whether the existing job still matches
ansible.builtin.set_fact:
pve_backup_drifted: >-
{{ pve_backup_existing.schedule | default('') != pve_backup_schedule
or pve_backup_existing.storage | default('') != pve_backup_storage
or pve_backup_existing.mode | default('') != pve_backup_mode
or (pve_backup_existing.enabled | default(1) | int == 1) != pve_backup_enabled
or (pve_backup_existing.all | default(0) | int == 1) != pve_backup_all
or (not pve_backup_all
and pve_backup_existing.vmid | default('') != pve_backup_vmids | join(',')) }}
when: pve_backup_existing | length > 0
- name: Update the vzdump job if its settings drifted
ansible.builtin.command:
argv: "{{ ['pvesh', 'set', '/cluster/backup/' ~ pve_backup_existing.id] + pve_backup_job_args }}"
when:
- pve_backup_existing | length > 0
- pve_backup_drifted | bool

19
build/config/ansible/roles/unattended_upgrades/defaults/main.yml Normal file
View file

@ -0,0 +1,19 @@
---
# Origins unattended-upgrades installs from — Ubuntu's own stock list
# (security, plus the ESM ones, which are inert without Ubuntu Pro attached
# rather than an error). Override per-host/group for less than this.
unattended_upgrades_origins:
- "${distro_id}:${distro_codename}"
- "${distro_id}:${distro_codename}-security"
- "${distro_id}ESMApps:${distro_codename}-apps-security"
- "${distro_id}ESM:${distro_codename}-infra-security"
# Never reboot automatically. playbooks/k3s_maintenance.yml (role
# k3s_maintenance) does that instead — draining the node first, one at a
# time — which is the entire reason this role exists rather than just
# `apt install unattended-upgrades` with Ubuntu's own defaults (which reboot
# at 02:00 unprompted, k3s node or not).
unattended_upgrades_automatic_reboot: false
unattended_upgrades_remove_unused_deps: true
unattended_upgrades_remove_unused_kernel_packages: true

5
build/config/ansible/roles/unattended_upgrades/handlers/main.yml Normal file
View file

@ -0,0 +1,5 @@
---
- name: restart unattended-upgrades
ansible.builtin.systemd_service:
name: unattended-upgrades
state: restarted

43
build/config/ansible/roles/unattended_upgrades/tasks/main.yml Normal file
View file

@ -0,0 +1,43 @@
---
# Debian/Ubuntu-generic — not k3s-specific, kept independent of roles/k3s_node
# so it can be pointed at any apt-based host later (a Proxmox guest, say)
# without dragging k3s along. Currently only applied to k3s_cluster, from
# playbooks/k3s.yml.
#
# Installs updates hands-off, but leaves rebooting to
# playbooks/k3s_maintenance.yml — see unattended_upgrades_automatic_reboot in
# defaults/main.yml for why.
- name: Install unattended-upgrades
ansible.builtin.apt:
name:
- unattended-upgrades
- update-notifier-common
state: present
update_cache: true
cache_valid_time: 3600
- name: Configure unattended-upgrades
ansible.builtin.template:
src: 50unattended-upgrades.j2
dest: /etc/apt/apt.conf.d/50unattended-upgrades
owner: root
group: root
mode: "0644"
notify: restart unattended-upgrades
- name: Enable periodic apt updates and unattended-upgrades
ansible.builtin.copy:
dest: /etc/apt/apt.conf.d/20auto-upgrades
content: |
APT::Periodic::Update-Package-Lists "1";
APT::Periodic::Unattended-Upgrade "1";
owner: root
group: root
mode: "0644"
- name: Ensure unattended-upgrades is enabled and running
ansible.builtin.systemd_service:
name: unattended-upgrades
enabled: true
state: started

17
build/config/ansible/roles/unattended_upgrades/templates/50unattended-upgrades.j2 Normal file
View file

@ -0,0 +1,17 @@
// Managed by Ansible (roles/unattended_upgrades) — edits here are overwritten
// on the next run.
Unattended-Upgrade::Allowed-Origins {
{% for origin in unattended_upgrades_origins %}
"{{ origin }}";
{% endfor %}
};
Unattended-Upgrade::Remove-Unused-Dependencies "{{ unattended_upgrades_remove_unused_deps | lower }}";
Unattended-Upgrade::Remove-Unused-Kernel-Packages "{{ unattended_upgrades_remove_unused_kernel_packages | lower }}";
// The whole point of this role: never reboot on its own. A reboot-required
// package (kernel, libc, ...) is left applied-but-inactive until
// playbooks/k3s_maintenance.yml (role k3s_maintenance) drains this node and
// reboots it deliberately, one node at a time.
Unattended-Upgrade::Automatic-Reboot "{{ unattended_upgrades_automatic_reboot | lower }}";

23
build/config/terraform/.terraform.lock.hcl generated Normal file
View file

@ -0,0 +1,23 @@
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/bpg/proxmox" {
version = "0.111.1"
hashes = [
"h1:ML2D3UUZTM99yrll/EBXj7wBYMb8xmQgomqFNybEoxY=",
"zh:18fb7c31a08dde6bffa1a4d4a211e604d6d17eec7092fd59331b3db3c6f3742c",
"zh:1cd60761538289d4dd2a1086b3ae62a7b0bdd4b1a2f824e9a44e243413168dba",
"zh:2eb76f6fc8299b6820ff678c8252332cc3366e226b5ae2e61748fd2449c1ed92",
"zh:45e6f7ebd0bf48911d37060359a4f359b5743b3092e985295733990e406d0416",
"zh:4aa8ba912eae37975d2e983394d173e595ca34fc76b5bf220b37d0e99d76e98c",
"zh:58e0789923103a77d502a0a9fc3eb920625e8eb935ec2d4ac0d006aebd1d186c",
"zh:6df8aa85fb8865915537e946c19b02538ad188018a629759c213c6f03730f642",
"zh:6ed47bc00d0913a1d0880618fa1376115e9edab6b4a658c081061a7f0e4ca360",
"zh:c5b10ff4f33df7e4c29e8f1127d49845b561b37b57517e844fb0954d7923d65e",
"zh:d016510e14b738499f0db9d9b3aafe82fc6877fb4ab4e9f831fb68a8d70a1385",
"zh:d941f394069bbf24351b363da1c64383f487067aaee0a84f9b96476d4912e212",
"zh:ddf271dbc2632ae8ffa8de3972f243ee47d260cb2ac90aa784f2746d98e21a0f",
"zh:ed0caa3501c42f611b7e9622c9b1df69fd85dc25a3cd88d3076381829688cd62",
"zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597",
]
}

66
build/config/terraform/README.md Normal file
View file

@ -0,0 +1,66 @@
# build/config/terraform
Centralized Terraform config for Proxmox — provider, backend, and the module
calls that say what infrastructure exists. The counterpart to
`build/config/ansible`, which holds inventory and the shared roles.
## Division of labour
Terraform's job stops at the guest. It creates the LXC (template, cores,
memory, disk, IP) and nothing more; Ansible then installs the app into it via
the `lxc_app` role. That split is why Ansible is the common tool across both
platforms and Terraform is Proxmox-only:
| | Unraid | Proxmox |
|---|---|---|
| Provision the host | n/a — it already exists | Terraform (this directory) |
| Install the app | Ansible + `compose_stack` (Docker Compose) | Ansible + `lxc_app` (native + systemd) |
| App config | `src/<app>/common/vars.yml` + `src/<app>/ansible/unraid/vars.yml` | `src/<app>/common/vars.yml` + `src/<app>/ansible/proxmox/vars.yml` |
Per-app LXC specs live in `src/<app>/terraform/` as modules, called from
`main.tf` here.
## State
`backend "pg"` — specifically against the **CloudNativePG cluster on the k3s
Pis**, reached on its MetalLB address (`k3s_postgres_loadbalancer_ip`,
192.168.50.81). *Not* the shared Postgres this configuration itself
provisions on Proxmox, which would be circular: Terraform would need the
database to exist in order to create the container the database runs in.
State goes somewhere Terraform has no hand in building, which breaks the
cycle outright. Local state would make Terraform workstation-only; CI needs
to see the same state.
The connection string holds a password, so it is passed at init rather than
committed:
```sh
terraform init \
-backend-config="conn_str=postgres://terraform:$PG_PASSWORD@192.168.50.81:5432/terraform_state"
```
The bootstrap dependency this creates is on **k3s, not Unraid**: the cluster
and its CNPG instance must be up, with a `terraform_state` database and a
`terraform` role created on it, before `terraform init` works. That is the
one cross-platform dependency the rest of the repo avoids, and it is
deliberate — the alternative is a local-state-then-`init -migrate-state`
dance that has to be got right exactly once. Ordering is in
`docs/postgres-proxmox.md`.
## Credentials
Proxmox API token and the Postgres connection string both come from Vault
(`homelab/ci/proxmox`, `homelab/ci/terraform`), exported as environment
variables before running — the same pattern `group_vars/all.yml` uses for
`VAULT_ADDR` and friends. Nothing authenticating to anything is committed.
```sh
export TF_VAR_proxmox_endpoint='https://turtle-proxmox-01.home.turtlesystems.co.uk:8006/'
export TF_VAR_proxmox_api_token='root@pam!terraform=<secret>'
```
## Status
Not yet applied against anything. The provider version is intentionally
unpinned until the first `terraform init` — pin what it resolves and commit
`.terraform.lock.hcl` (which is not gitignored).

102
build/config/terraform/main.tf Normal file
View file

@ -0,0 +1,102 @@
# Per-app LXC modules are called from here, so there is one place that knows
# what infrastructure exists. Each module lives with its app in
# src/<app>/terraform/ and outputs the guest's address, which the Ansible
# side then picks up from the Proxmox API (inventory/proxmox.yml) rather than
# by wiring outputs into inventory by hand.
# The shared Postgres, and the first module here that is infrastructure other
# modules depend on rather than an app in its own right. Declared first for
# readability only — Terraform orders by dependency, not by position, and
# nothing else in this file references it. The ordering that does matter is
# on the Ansible side, where a guest's `db:` provisioning needs this cluster
# already answering; see the runbook in docs/postgres-proxmox.md.
module "postgres" {
source = "../../../src/shared/postgres/terraform"
node_name = var.proxmox_node
template_file_id = var.lxc_template_file_id
ssh_public_keys = var.ssh_public_keys
hostname = "postgres"
vm_id = 161
ip_address = "192.168.50.54/24"
gateway = "192.168.50.254"
cores = 2
memory = 2048
# Sized for every database this host will ever hold, not for the OS. The
# one-volume rule below means growth happens here rather than by adding a
# second disk: `terraform apply` turns an increase into a `pct resize` of
# the rootfs, which on ZFS is a refquota change — online, no data move, no
# filesystem grow step. It is one-way, though; shrinking is a replacement
# and `prevent_destroy` blocks it, so overshoot rather than creep upward.
disk_size = 256
# The ZFS pool, so the guest's volume can be replicated at all, and the
# second node to replicate it to. Both are the whole point of this module
# differing from the one below — see src/shared/postgres/terraform/main.tf.
datastore_id = "AppData"
replication_target_node = var.proxmox_replication_node
}
module "forgejo" {
source = "../../../src/forgejo/terraform"
# The second node, not `var.proxmox_node` — a deliberate split of the two
# guests across the cluster rather than an oversight. Written as a literal
# here for the same reason `hostname`/`vm_id`/`ip_address` are: it states
# where this one guest goes. `var.proxmox_replication_node` happens to hold
# the same string today, but it means "where Postgres replicates to", and
# borrowing it would tie Forgejo's placement to a decision about the
# database.
#
# Safe to change only while this container does not exist. Once it does, the
# provider treats `node_name` as a replacement — and `prevent_destroy` in
# the module turns that into a failed plan, which is the intended outcome:
# moving a live Forgejo between nodes is a Proxmox migration, not a
# `terraform apply`.
node_name = "turtle-proxmox-02"
template_file_id = var.lxc_template_file_id
ssh_public_keys = var.ssh_public_keys
hostname = "forgejo"
vm_id = 160
ip_address = "192.168.50.52/24"
gateway = "192.168.50.254"
cores = 2
memory = 2048
# Overrides the module default only in the sense of restating it; both are
# 256 GiB. Growing this is one-way, same as the Postgres guest above:
# `terraform apply` turns an increase into an online `pct resize` of the
# rootfs, but shrinking is a replacement and `prevent_destroy` blocks it.
# Sized generously because *everything* Forgejo owns is on this volume —
# repos, LFS, attachments, indexers, and the local database dumps.
disk_size = 256
# The ZFS pool, which exists on both nodes, rather than the module's
# `local-lvm` default — this cluster has no storage by that name. No
# replication job to go with it, unlike the Postgres guest above: Forgejo's
# off-box copy is the vzdump archive `pve_backup` writes to the NAS, which
# is a complete filesystem restore point precisely because nothing here is
# bind-mounted.
datastore_id = "AppData"
}
# Consumed by the Proxmox nodes' host_vars, which can pin the vzdump job to
# explicit VMIDs. Kept as an output rather than restated there so the two
# can't drift.
output "backup_vmids" {
description = "VMIDs of the guests this configuration creates, for the vzdump job."
value = [module.postgres.vm_id, module.forgejo.vm_id]
}
# The address every app's `db.provision_host` and `DB_HOST` has to agree with.
# An output rather than something to look up in the Proxmox UI, because it is
# copied into several files by hand and this is the one authoritative copy.
output "postgres_address" {
description = "LAN address of the shared Postgres LXC."
value = module.postgres.ip_address
}

42
build/config/terraform/providers.tf Normal file
View file

@ -0,0 +1,42 @@
terraform {
required_version = ">= 1.6"
required_providers {
proxmox = {
source = "bpg/proxmox"
# Deliberately unpinned until the first real `terraform init` — pin to
# whatever it resolves, and commit .terraform.lock.hcl (which is not
# gitignored) so everyone and CI get the same provider.
# version = "~> 0.0"
}
}
# State in Postgres rather than a local file: CI and your workstation need
# to see the same state, and a local file makes Terraform workstation-only.
#
# Specifically the CloudNativePG cluster on k3s, reached at
# k3s_postgres_loadbalancer_ip — *not* the shared Postgres this
# configuration itself provisions on Proxmox. That would be circular:
# Terraform would need the database to exist in order to create the
# container the database runs in. Putting state on a cluster this
# configuration has no hand in building breaks the cycle outright, which is
# why it is worth the cross-platform dependency the rest of the repo
# otherwise avoids — `terraform apply` now needs the Pis up.
#
# Left empty on purpose — the connection string contains a password, so it
# is supplied at init time instead of being committed:
#
# terraform init \
# -backend-config="conn_str=postgres://terraform:$PG_PASSWORD@192.168.50.81:5432/terraform_state"
#
# Bootstrap ordering: the k3s cluster and its CNPG instance must exist (and
# a `terraform_state` database be created on it) before `terraform init`
# succeeds. See docs/postgres-proxmox.md.
backend "pg" {}
}
provider "proxmox" {
endpoint = var.proxmox_endpoint
api_token = var.proxmox_api_token
insecure = var.proxmox_insecure
}

78
build/config/terraform/variables.tf Normal file
View file

@ -0,0 +1,78 @@
variable "proxmox_endpoint" {
description = "Proxmox VE API endpoint, e.g. https://turtle-proxmox-01.home.turtlesystems.co.uk:8006/"
type = string
}
variable "proxmox_api_token" {
description = <<-EOT
Proxmox API token in `USER@REALM!TOKENID=SECRET` form. Sourced from Vault
(homelab/ci/proxmox), never committed — pass via the TF_VAR_proxmox_api_token
environment variable.
EOT
type = string
sensitive = true
}
variable "proxmox_insecure" {
description = "Skip TLS verification — true while Proxmox has a self-signed cert."
type = bool
default = true
}
variable "proxmox_node" {
description = <<-EOT
Proxmox node guests are created on. Must be the node's **short** name as
the cluster knows it (`pvecm nodes`), never an FQDN, even though the API
endpoint is one.
pveproxy compares this against its own short hostname to decide whether a
request is local. An FQDN never matches, so it proxies the request onward
— to itself — over a connection it verifies against the cluster CA, which
a self-signed cert fails. The result is an HTTP 596 carrying an OpenSSL
`certificate verify failed`, which reads like a TLS misconfiguration on
this side and is not: `proxmox_insecure` governs the provider's own
connection and has no bearing on Proxmox's internal one.
EOT
type = string
default = "turtle-proxmox-02"
}
variable "proxmox_replication_node" {
description = <<-EOT
The other node in the cluster, which guests on a ZFS pool replicate to.
Only the shared Postgres module uses this today; it lives here rather than
in that module's own defaults because it names a node of this cluster, the
same as `proxmox_node` does.
Null disables replication. Set it explicitly even on a single-node
cluster — a null default is indistinguishable from an oversight, and the
failure mode is a database with no second copy that looks entirely
healthy.
EOT
type = string
default = "turtle-proxmox-01"
}
variable "lxc_template_file_id" {
description = <<-EOT
Container template every LXC module is built from, e.g.
`local:vztmpl/debian-12-standard_12.7-1_amd64.tar.zst`. Must already be
downloaded on the node — `pveam list local` shows what is there, `pveam
available`/`pveam download local <name>` fetches one. Not defaulted: the
exact filename moves with each point release, so a stale default would fail
at apply time rather than here.
EOT
type = string
}
variable "ssh_public_keys" {
description = <<-EOT
Public keys installed for root in every container, so Ansible can reach
them. The matching private key is in Vault at `homelab/ci/ssh` — see "SSH
access" in the top-level README.md. Public keys aren't secret, but they are
site-specific, so pass rather than commit:
export TF_VAR_ssh_public_keys='["'"$(cat ~/.ssh/unraid_ansible.pub)"'"]'
EOT
type = list(string)
}

420
docs/authentik-migration.md Normal file
View file

@ -0,0 +1,420 @@
# Migrating Authentik from Unraid to k3s
A one-time, hand-run cutover: move the Authentik database off Unraid onto the
cluster's CloudNativePG instance, bring Authentik up on k3s from
`src/authentik/`, verify it, then change DNS.
This is deliberately not automated. It runs once, most of its steps are
irreversible in the wrong order, and the verification between them is a
judgement call rather than a task result — the sort of thing a playbook makes
harder rather than easier. The repo half *is* automated: everything from
"deploy Authentik" onwards is `playbooks/k3s.yml`, and re-running it later is
an ordinary converge.
Read the whole thing before starting. The dump is taken with Authentik
stopped, so there's a service outage from step 5 to step 11 — budget an hour
and do it when nobody needs to log in.
---
## What moves, and what doesn't
| | Where it lives now | How it gets across |
|---|---|---|
| Users, groups, applications, providers, flows, tokens, certificates | Postgres | `pg_dump` \| `psql` (step 7) |
| Uploaded icons and flow backgrounds | `/media` on disk | `scp` + `kubectl cp` (step 9) |
| Session cookies | signed by `AUTHENTIK_SECRET_KEY` | carry the key over (step 2) |
| Cache / task queue | in-process | nothing — recent Authentik has no Redis, and the chart has no Redis dependency |
Authentik stores its certificates (including the signing keys OIDC/SAML
providers use) **in the database**, not in a `certs/` directory, so a
successful database restore carries them. That is what lets integrations like
Shelfarr's OIDC (`src/arr/`) keep working without being re-registered.
The one piece with no automatic path is anything you mounted into the Unraid
container by hand — custom templates, a `certs/` directory you populated
yourself. Step 1 is where you find out whether you have any.
---
## 1. Gather the facts
Everything below needs these. Run on the Unraid box (web terminal, or SSH).
```sh
# Which containers make up the current deployment, and what image/tag.
docker ps -a --format '{{.Names}}\t{{.Image}}\t{{.Status}}' | grep -i authentik
```
Write down the **image tag** — that is the Authentik version, and step 3
depends on it.
```sh
# Every mount, so you can see where /media lives and spot anything unexpected.
docker inspect authentik-server \
--format '{{range .Mounts}}{{.Source}} -> {{.Destination}}{{println}}{{end}}'
# The database connection it currently uses, and the secret key.
docker inspect authentik-server \
--format '{{range .Config.Env}}{{println .}}{{end}}' | grep -E 'POSTGRES|SECRET_KEY'
```
That last command tells you the thing this document can't assume: **which
Postgres the Unraid Authentik talks to**. Two possibilities, and they change
step 7:
- `AUTHENTIK_POSTGRESQL__HOST=shared-postgres` (or `192.168.50.1`) — it uses
the shared instance this repo already manages, `src/shared/postgres/`. The
superuser password is in Vault at `homelab/shared/postgres`.
- anything else (`authentik-postgresql`, `postgres`, a container name from
Authentik's own compose bundle) — the Unraid UI deployment brought its own
Postgres container. You'll need *its* credentials, which are in that
container's own environment:
```sh
docker inspect <that-container> \
--format '{{range .Config.Env}}{{println .}}{{end}}' | grep POSTGRES
```
and it is probably not published on a host port, which step 7 handles.
Also note `AUTHENTIK_SECRET_KEY` from the output above — step 2 needs it
verbatim.
---
## 2. Populate Vault
```sh
vault kv put kv/homelab/authentik \
AUTHENTIK_SECRET_KEY='<the value from step 1>' \
AUTHENTIK_POSTGRESQL__PASSWORD='<a new strong password>'
```
Two different rules for those two values, and it's worth being clear about
why:
- **`AUTHENTIK_SECRET_KEY` must be the existing one.** It signs session
cookies. A fresh value doesn't break anything permanently, but it
invalidates every active session at the moment DNS flips — so instead of a
silent cutover, everyone gets logged out and the migration announces
itself. Copy it across.
- **`AUTHENTIK_POSTGRESQL__PASSWORD` can be new.** You are creating a new
role on a different Postgres; nothing in the dump references it. Generate a
fresh one.
If you're also using cert-manager's DNS-01 solver (step 4), populate its
credentials now too:
```sh
vault kv put kv/homelab/k3s-cert-manager \
CLOUDFLARE_API_TOKEN='<token with Zone:DNS:Edit on the zone>'
```
---
## 3. Confirm the repo's values
Three values are guesses until you check them. All are flagged in-file.
**`src/authentik/common/vars.yml`**
- `AUTHENTIK_VERSION` — **set this to the image tag from step 1**, not to
whatever is newest. Authentik runs Django migrations on startup against
whatever schema it finds, and they are one-way: a newer Authentik pointed
at an older dump upgrades the schema silently, and if the cutover then has
to be rolled back, the Unraid instance can no longer read its own database.
Migrate first, upgrade second, in a separate commit.
- `AUTHENTIK_HOST` / `AUTHENTIK_URL` — the hostname you'll be moving. Must
match what's there today, or every OIDC/SAML redirect URI registered in
Authentik breaks.
**`build/config/ansible/inventory/group_vars/k3s_cluster.yml`**
- `k3s_cert_manager_solver` — empty by default, and
`roles/k3s_cert_manager` refuses to run until it's set. The commented
Cloudflare example in that file is the common case; for anything else take
the stanza from
[cert-manager's DNS-01 docs](https://cert-manager.io/docs/configuration/acme/dns01/).
While you're testing, point `k3s_cert_manager_acme_server` at Let's Encrypt
staging. Production allows five failed validations per hostname per hour, and
exhausting it means waiting rather than retrying.
---
## 4. Deploy cert-manager and confirm the issuer works
```sh
cd build/config/ansible
ansible-playbook playbooks/k3s.yml --tags cert-manager
```
Then check it actually registered — a `ClusterIssuer` is created whether or
not its solver is valid, so "the resource exists" proves nothing:
```powershell
$env:KUBECONFIG = "$PWD\local\k3s\homelab-utils.kubeconfig"
kubectl get clusterissuer letsencrypt -o jsonpath='{.status.conditions[*].message}'
```
You want `The ACME account was registered with the ACME server`. Anything
else, fix it here — not after Authentik is deployed and waiting on a
certificate.
Worth knowing: with DNS-01, the certificate can be issued **before** DNS
points at the cluster. The challenge is a TXT record, not a request to your
web server, so cert-manager never needs to be reachable. That's why TLS is
sorted out in this step and not after the flip.
---
## 5. Freeze the Unraid instance
Stop writes before dumping. Skipping this gives you a dump that's missing
whatever happened during it — usually invisible, occasionally a user account
that no longer exists.
```sh
docker stop authentik-server authentik-worker
```
Leave its Postgres running — step 7 reads from it. **The outage starts
here.**
---
## 6. Create the role and database on CNPG
The restore needs the `authentik` role to exist first so the dump's objects
land with the right ownership.
```powershell
$env:KUBECONFIG = "$PWD\local\k3s\homelab-utils.kubeconfig"
kubectl -n shared-postgres exec -i shared-postgres-1 -- psql -v ON_ERROR_STOP=1 <<'SQL'
CREATE ROLE authentik LOGIN PASSWORD 'the-password-from-step-2';
CREATE DATABASE authentik OWNER authentik;
SQL
```
`kubectl exec` into a CNPG pod lands you as the `postgres` superuser over the
local socket, so bare `psql` needs no credentials.
You could instead let `roles/k3s_app` create these — its tasks are idempotent
and will run in step 8 regardless — but doing it by hand here keeps the
database creation and the restore adjacent, and avoids deploying Authentik
against an empty database first (which would have it run initial migrations
and build a schema the restore then has to fight).
---
## 7. Dump and restore
Run the whole thing **inside the CNPG pod**, piping `pg_dump` straight into
`psql`. Three reasons this is better than dumping to a file on Windows:
- `pg_dump` must be at least the version of the server being dumped, and the
CNPG pod ships Postgres 18 client tools — dumping the Unraid Postgres 13
from there is the supported direction. Your workstation probably has no
`pg_dump` at all.
- PowerShell re-encodes bytes passing through `>` and `|`, which silently
corrupts a dump file. Nothing on the Windows filesystem means nothing to
corrupt.
- No intermediate file holding the entire user database in cleartext.
### If Authentik used the shared Postgres (published on `192.168.50.1:5432`)
```powershell
$env:KUBECONFIG = "$PWD\local\k3s\homelab-utils.kubeconfig"
$srcPw = Read-Host 'Unraid postgres superuser password'
kubectl -n shared-postgres exec -i shared-postgres-1 -- `
env "PGPASSWORD=$srcPw" bash -c @'
set -euo pipefail
{ echo "SET ROLE authentik;"
pg_dump -h 192.168.50.1 -p 5432 -U postgres -d authentik \
--no-owner --no-privileges
} | psql -v ON_ERROR_STOP=1 -d authentik
'@
```
`SET ROLE authentik` before the dump stream, combined with `--no-owner`, is
what gets ownership right: the session creates every object as `authentik`,
so the application role owns its own schema. Without it, everything lands
owned by `postgres` and Authentik can read but not migrate.
`ON_ERROR_STOP=1` matters — without it `psql` reports success having skipped
every statement that failed.
### If it used its own Postgres container (not published)
Publish it temporarily, then use the command above with the container's own
credentials and port:
```sh
# on Unraid
docker run -d --rm --name pgbridge -p 55432:5432 \
--network container:<authentik-postgres-container> alpine/socat \
TCP-LISTEN:5432,fork TCP:127.0.0.1:5432
```
then `-h 192.168.50.1 -p 55432 -U <its user>` in the `pg_dump` above, and
`docker stop pgbridge` afterwards. Simpler alternative if you'd rather not:
`docker exec <container> pg_dump ... > /mnt/user/appdata/authentik.sql` on
Unraid, `scp` it to your workstation, and `kubectl exec -i ... -- psql -d
authentik < authentik.sql` — but a plain-SQL file through PowerShell
redirection is exactly the encoding hazard noted above, so run that last
command from WSL.
### Check the restore
```powershell
kubectl -n shared-postgres exec -i shared-postgres-1 -- `
psql -d authentik -c '\dt' -c 'SELECT count(*) FROM authentik_core_user;'
```
You should see a long table list and a user count matching what the old
instance had. An empty table list means the restore did nothing.
---
## 8. Deploy Authentik on k3s
```sh
cd build/config/ansible
ansible-playbook playbooks/k3s.yml --tags apps
```
A green run means the manifests landed and the database was provisioned — not
that the workload came up. Watch it actually start:
```powershell
kubectl -n authentik get pods -w
```
The first start takes several minutes on a Pi: pulling the image, then the
worker checking migrations. Because the deployed version matches the dumped
version (step 3), it should find nothing to migrate.
If pods stay `Pending`, check the PVC — `local-path` uses
`WaitForFirstConsumer`, so a `Pending` PVC alongside a `Pending` pod is
normal only until the scheduler picks a node:
```powershell
kubectl -n authentik describe pvc authentik-media
kubectl -n authentik logs -l app.kubernetes.io/component=worker --tail=50
```
---
## 9. Copy `/media` across
Two hops rather than a pipe, again to keep binary data out of a PowerShell
pipeline:
```powershell
# from the media path you noted in step 1
scp -r root@192.168.50.1:/mnt/user/appdata/authentik/media ./local/authentik-media
$pod = kubectl -n authentik get pod -l app.kubernetes.io/component=server `
-o jsonpath='{.items[0].metadata.name}'
kubectl -n authentik cp ./local/authentik-media/. "${pod}:/media"
```
`local/` is already gitignored, so the staging copy won't be committed.
Skip this entirely if you never uploaded custom icons or backgrounds —
Authentik falls back to its built-in assets, and the directory is empty on a
stock install.
---
## 10. Verify before touching DNS
The hostname still resolves to Unraid, so test by overriding resolution
rather than changing it. `--resolve` sends the request to Traefik with the
right SNI and `Host` header:
```powershell
curl.exe -sv --resolve auth.turtlesystems.uk:443:192.168.50.80 `
https://auth.turtlesystems.uk/-/health/ready/
```
Check, in order:
1. The TLS certificate is the Let's Encrypt one, not Traefik's self-signed
default (`kubectl -n authentik get certificate` → `READY True`). If you
used the staging server in step 3, it will be untrusted — expected; switch
to production and re-run `--tags cert-manager` once satisfied.
2. `/-/health/ready/` returns 204.
3. Add a hosts-file entry pointing the name at `192.168.50.80` and log in
through a browser with a **real user account**. This is the check that
matters — it exercises the restored password hashes, the flows, and the
session cookie signed with the carried-over secret key.
4. Open an existing application from the Authentik dashboard and confirm the
OIDC/SAML round-trip still works (Shelfarr is the one this repo knows
about — see `src/arr/`).
Remove the hosts-file entry afterwards.
---
## 11. Flip DNS
Point `auth.turtlesystems.uk` at `192.168.50.80` (Traefik's MetalLB address).
Lower the record's TTL a few hours beforehand if you can — it shortens the
window in which a rollback is still invisible to clients.
---
## Rollback
Viable until users start making changes in the new instance; after that,
rolling back loses whatever they did.
1. Point DNS back at Unraid.
2. `docker start authentik-server authentik-worker`.
The old database was only ever read, never written, so it's exactly as it was
at step 5. Nothing to restore.
Do **not** run both instances at once against their separate databases — they
diverge immediately, and there is no way to merge them afterwards. That is
the same "never declare an app on two platforms" rule from CLAUDE.md, and it
is why `src/authentik/ansible/unraid/` is a README rather than a compose file.
---
## After it has settled
Once you're confident, in this order:
1. **Delete the Unraid containers.** A UI action, not `state: absent` — this
repo never deployed them, so it has nothing to tear down. Keep the appdata
directory a while longer.
2. **Drop the old database**, if it was on the shared Postgres:
```sh
docker exec -it shared-postgres psql -U postgres -c 'DROP DATABASE authentik;'
```
Take a copy first if you want one; nothing else does.
3. **Upgrade Authentik**, now as an ordinary change: bump
`AUTHENTIK_VERSION` in `src/authentik/common/vars.yml`, read the release
notes for anything between your version and the target, and
`ansible-playbook playbooks/k3s.yml --tags apps -e only_apps=authentik`.
Do this as its own commit, so a bad upgrade is one `git revert` away from
the version you know worked.
## Backups, afterwards
Worth stating plainly: the cluster's CNPG has **no backups configured**
(CLAUDE.md → "Key decisions" — HA replication only). Before this migration,
Authentik's database sat on Unraid, inside whatever covers `/mnt/user/appdata`.
After it, the only redundancy is a streaming replica on another Pi, which
protects against a dead SD card and not against a bad migration, a dropped
table, or a mistake in this document.
The gap this leaves is real and this cutover widens it. CNPG's answer is a
`ScheduledBackup` to object storage or an NFS volume on the NAS; that's not
built here yet, and it's the obvious next piece of work after Authentik is
settled.

287
docs/forgejo-proxmox.md Normal file
View file

@ -0,0 +1,287 @@
# Bringing up Forgejo on Proxmox
A run-once bootstrap, same shape as `postgres-proxmox.md` and deliberately
downstream of it — Forgejo's database lives on the shared Postgres that
runbook creates, so none of this works until that one is finished.
The end state: an LXC on the `AppData` pool of **turtle-proxmox-02** (VMID
160, 192.168.50.52), running Forgejo natively under systemd, with its database
on the shared Postgres LXC at 192.168.50.54 and a nightly `pg_dump` landing on
its own disk in time for the node's 02:00 vzdump.
## Why turtle-proxmox-02
Nothing forces it. Postgres is on `-01`, so putting Forgejo on `-02` splits
the two guests across the cluster: a node going down takes one of them with
it rather than both. Note what that does *not* buy — Forgejo with its database
unreachable is not a working forge, so this is about not losing both
filesystems at once, not about staying up.
It is set as a literal `node_name` on the `forgejo` module in
`build/config/terraform/main.tf`, not from `var.proxmox_node`. **Change it
only before the container exists**: the provider treats `node_name` as a
replacement, and `prevent_destroy` in `src/forgejo/terraform/main.tf` turns
that into a failed plan. Moving a live Forgejo is a Proxmox migration
(`pct migrate`) followed by editing the literal to match, not a
`terraform apply`.
## 1. Secrets
`homelab/forgejo` needs five keys. Four of them may already exist from the
Unraid stack; **`LFS_JWT_SECRET` almost certainly does not** — the Compose
deployment let Forgejo generate it, and the native install renders `app.ini`
in full with `INSTALL_LOCK = true`, so there is no first boot for Forgejo to
invent one on.
```sh
vault kv get kv/homelab/forgejo
```
If `LFS_JWT_SECRET` is missing, generate one and patch it in without
disturbing the others:
```sh
vault kv patch kv/homelab/forgejo \
LFS_JWT_SECRET="$(docker run --rm codeberg.org/forgejo/forgejo:10 forgejo generate secret)"
```
Full key table in `vault-secrets.md`. `install.yml` asserts on all five before
it touches the container, so a missing one fails on the first task with a
message naming it rather than three tasks later on an undefined variable.
## 2. The container template
The template must be present on **turtle-proxmox-02**, not just on `-01` where
Postgres was built. If `StorageOne` is not shared across the cluster this is a
separate download:
```sh
ssh root@turtle-proxmox-02 'pveam list StorageOne'
# if the ubuntu-26.04 template is absent:
ssh root@turtle-proxmox-02 'pveam update && pveam download StorageOne ubuntu-26.04-standard_26.04-1_amd64.tar.zst'
```
Whatever is there has to match `lxc_template_file_id` exactly — the filename
moves with each point release, which is why that variable has no default.
## 3. Terraform
State is on the CNPG cluster, so the Pis have to be up (see
`postgres-proxmox.md` § "Why this order").
```sh
cd build/config/terraform
terraform init -backend-config=... # as per postgres-proxmox.md
terraform plan -out=deploy.plan -target=module.forgejo
terraform apply deploy.plan
```
Read the plan before applying. It should be **one resource to add** and
nothing to change or destroy; anything touching `module.postgres` — in
particular a *replacement* — means stop and work out why, because that guest
is live.
Note `deploy.plan` embeds the state and every input variable, `sensitive` ones
included, so it holds the Proxmox API token in the clear. `.gitignore` covers
`*.plan`; delete it once applied rather than leaving it in the tree.
Confirm the guest exists and Ansible can see it:
```sh
ssh root@turtle-proxmox-02 'pct list | grep 160'
cd ../ansible && ansible-inventory --list --yaml proxmox_guests | grep -A2 forgejo
```
If it is absent from the inventory but present in `pct list`, the `terraform`
tag is missing — that filter is what tells this repo's guests from hand-made
ones, and a guest without it is silently never deployed to.
## 4. Install Forgejo into it
```sh
cd build/config/ansible
ansible-playbook playbooks/proxmox.yml -e only_stacks=forgejo
```
This provisions the `forgejo` database and role on 192.168.50.54, then
installs the binary, `app.ini`, the systemd unit and the dump timer. The
database step runs from **your controller**, not from inside the container
(`delegate_to: localhost`), so 192.168.50.54:5432 has to be reachable from
wherever you run this.
Check:
```sh
ssh root@192.168.50.52 'systemctl is-active forgejo && systemctl list-timers forgejo-dbdump.timer'
curl -sI http://192.168.50.52:3000/ | head -1
```
## 4b. The first admin user
A fresh install has **no way in**: `INSTALL_LOCK = true` skips the setup
wizard, which is where the first admin would normally be created, and
`DISABLE_REGISTRATION = true` means you cannot self-register either (both in
`src/forgejo/ansible/proxmox/templates/app.ini.j2`). Nothing in `install.yml`
creates a user, so the account has to be made from the CLI inside the
container:
```sh
ssh root@192.168.50.52
sudo -u git forgejo admin user create --admin --username <you> --email <you>@turtlesystems.co.uk --password '<strong-password>' --config /etc/forgejo/app.ini --work-path /var/lib/forgejo
```
Three things that are easy to get wrong:
- **`sudo -u git`, not root.** `RUN_USER` is `git`; running the CLI as root
leaves root-owned files under `/var/lib/forgejo` that Forgejo then cannot
write. `app.ini` is `0640 root:git`, so the `git` user can read it.
- **`--config` is not optional.** Forgejo looks for `custom/conf/app.ini`
under the work path by default; this install puts it at
`/etc/forgejo/app.ini` (`FORGEJO_CONFIG_DIR` in `vars.yml`). Without the
flag the CLI reads a config that isn't there and never reaches the
database.
- Recent versions default `--must-change-password` to true, so expect a
forced change on first web login. Pass `--must-change-password=false` to
skip it. Confirm the flags for the pinned `FORGEJO_RELEASE` with
`forgejo admin user create --help` rather than assuming.
Then log in at `http://192.168.50.52:3000/user/login` — by IP, because
`ROOT_URL` is `https://git.turtlesystems.uk` and nothing resolves there until
the cutover in step 7. The login form works over the IP; some links and
redirects Forgejo renders will point at the not-yet-live hostname.
**Skip this step if you are restoring a database in step 6.** Users live in
the database, so a dump brings its own admin back and an account created here
is overwritten by the restore. Create one only if this is genuinely an empty
forge — or after the restore, if the dump turns out to have no usable admin.
## 5. Node backups
VMID 160 is on `-02`, so it is `-02`'s vzdump job that covers it.
`pve_backup_all: true` in `group_vars/proxmox_nodes.yml` means no edit is
needed — but the job has to actually exist on that node:
```sh
ansible-playbook playbooks/pve_host.yml --limit turtle-proxmox-02
ssh root@turtle-proxmox-02 'cat /etc/pve/jobs.cfg'
```
The 01:30 dump and the 02:00 vzdump are a pair. Move one, move the other —
see `src/forgejo/ansible/proxmox/README.md`.
## 6. Repository data
**Ansible does not move any of this.** A fresh install is a working, empty
forge; the repositories are a separate restore into
`/var/lib/forgejo/data/forgejo-repositories`.
The original instructions here were an `rsync` off nas2, which no longer
exists as a machine to read from — whatever repository data survives has to
come from wherever it went when that host was retired. After restoring:
```sh
ssh root@192.168.50.52 'chown -R git:git /var/lib/forgejo/data && systemctl restart forgejo'
```
Restore the database the same way, from whatever dump you have, before
starting Forgejo against it — an empty database with populated repositories
gives back every file and no issues, pull requests, users or permissions.
## 7. Cutover
Last, once the above is verified:
- Point `git.turtlesystems.uk` at this container.
- **SSH clone URLs move from port 2222 to 22.** The Unraid stack published
2222 to dodge the host's own sshd; this container has its own IP and uses
its own sshd, with Forgejo managing the `git` user's `authorized_keys`.
Every existing remote needs editing — there is no redirect for this.
- The Forgejo Actions runner registration is tied to the instance. Re-register
it against the new host, or CI stops running (`.forgejo/workflows/`).
## 8. Sign-in through Authentik (optional)
Forgejo supports OIDC, and `app.ini` is already set up for it — but only the
*policy* half. The provider itself cannot be configured from `app.ini`:
Forgejo keeps authentication sources in its **database**. That makes this the
one part of Forgejo's configuration Ansible does not own, and the reason it is
a runbook step rather than a task in `install.yml`. It is run once and then
carried forward by the nightly `pg_dump`, the same as every other row in that
database.
**Do this after step 7, not before.** The redirect URI has to match `ROOT_URL`
(`https://git.turtlesystems.uk`), so DNS and TLS must already be live.
Configuring it against `http://192.168.50.52:3000` means doing it twice.
### On the Authentik side
Create an OAuth2/OIDC **Provider** plus an **Application** for it, as for any
other app on `auth.turtlesystems.uk`. The redirect URI is:
```
https://git.turtlesystems.uk/user/oauth2/authentik/callback
```
The last path segment is the *name of the auth source in Forgejo*, not a fixed
string — it has to match the `--name` below. Store the generated client secret
in Vault (`vault-secrets.md` → `OIDC_CLIENT_SECRET`).
### On the Forgejo side
```sh
ssh root@192.168.50.52
sudo -u git forgejo admin auth add-oauth --name authentik --provider openidConnect --key <client-id> --secret <client-secret> --auto-discover-url https://auth.turtlesystems.uk/application/o/forgejo/.well-known/openid-configuration --config /etc/forgejo/app.ini --work-path /var/lib/forgejo
```
Same `sudo -u git` and `--config` requirements as step 4b, for the same
reasons. Verify the flag names against the pinned `FORGEJO_RELEASE` with
`forgejo admin auth add-oauth --help` before running — and list what exists
afterwards with `forgejo admin auth list`, which is also how you find the `id`
for `update-oauth` if the secret is ever rotated.
No restart is needed: the source is a database row, not a file Forgejo reads
at boot.
### What the app.ini side already does
Set in `src/forgejo/ansible/proxmox/templates/app.ini.j2`, so a redeploy keeps
them — the comments there carry the detail:
| Setting | Effect |
|---|---|
| `DISABLE_REGISTRATION = false` | It blocks OIDC auto-registration too, not just the local signup form. Left `true`, Authentik logins authenticate and are then refused an account. |
| `ALLOW_ONLY_EXTERNAL_REGISTRATION = true` | Restores "no self-service signup" without blocking Authentik. |
| `ENABLE_AUTO_REGISTRATION = true` | Creates the Forgejo account on first successful SSO login. |
| `ACCOUNT_LINKING = auto` | Attaches the OIDC identity to an existing local account with the same email. |
`ACCOUNT_LINKING` is the one that matters on the first login: if the
break-glass admin from step 4b uses the same email address as your Authentik
account, signing in through Authentik lands you *in that admin account*. With
linking off you would arrive as a second, unprivileged user and have to
promote it from the CLI.
### Keep the local admin
Do not delete the step 4b account or convert it to SSO-only. The login path is
now Forgejo → Authentik → CloudNativePG → a healthy k3s cluster, and any of
those failing takes SSO down with it — including the cases you would most want
to log into the forge to investigate. This is the same reasoning
`roles/k3s_traefik` uses for keeping the port-forward-only dashboard route
alive alongside the Authentik-published one.
### Git over HTTPS
Users who arrive through Authentik have no Forgejo password, so HTTPS clones
need a personal access token. SSH keys are unaffected — and since step 7 put
git-over-SSH on port 22 with Forgejo managing the `git` user's
`authorized_keys`, SSH is the smoother default to point people at.
## Rolling back
There is no `state: absent` for `lxc_app` — removal would need a per-app
`uninstall.yml` and Forgejo has none. To back out: stop the service
(`systemctl stop forgejo`), point DNS back at whatever was serving before, and
leave the container in place. Destroying it needs the `prevent_destroy` block
removed by hand first, which is deliberate.

237
docs/postgres-proxmox.md Normal file
View file

@ -0,0 +1,237 @@
# Bringing up the shared Postgres on Proxmox
A run-once bootstrap, in the order the dependencies actually demand. Each step
exists because the one after it cannot start without it; if you already have a
piece, skip it and check the assertion at the end of the section.
The end state: a Debian LXC on the `AppData` ZFS pool of `turtle-proxmox-01`,
running Postgres 17, replicated every five minutes to `turtle-proxmox-02`, and
answering on 192.168.50.54:5432 for every app that declares a `db:`.
## Why this order
Two chains have to be satisfied before `terraform apply` will run at all:
- **State.** Terraform stores state in Postgres. Storing it in the database
this configuration provisions would be circular, so it goes on the
CloudNativePG cluster on the k3s Pis instead — a cluster Terraform has no
hand in building. That cluster therefore has to be up first.
- **Inventory.** `playbooks/proxmox.yml` targets `proxmox_guests`, a group
that only exists because of the API-backed dynamic inventory. Ansible cannot
install into a container Terraform has just made until it can see it.
## 1. The k3s cluster and its CNPG instance
Needed only for Terraform state. If the cluster is already up, confirm the
LoadBalancer answers and move on.
```sh
cd build/config/ansible
ansible-playbook playbooks/k3s.yml --tags services
```
Then, against `k3s_postgres_loadbalancer_ip` (192.168.50.81), create the state
database and its role. This is the only hand-run SQL in the whole flow —
everything else provisions itself:
```sql
CREATE ROLE terraform LOGIN PASSWORD '<generate-a-strong-password>';
CREATE DATABASE terraform_state OWNER terraform;
```
Store the resulting connection string at `homelab/ci/terraform` in Vault (see
`vault-secrets.md`), then check it:
```sh
psql "postgres://terraform:$PG_PASSWORD@192.168.50.81:5432/terraform_state" -c '\conninfo'
```
## 2. Secrets
`homelab/shared/postgres` is the same path the Unraid and k3s instances
already use — one superuser identity for the "shared postgres" concept
wherever it runs. If it is populated, nothing to do:
```sh
vault kv get kv/homelab/shared/postgres
```
`homelab/ci/proxmox` needs a Proxmox API token, and `homelab/ci/ssh` the key
pair whose public half Terraform installs into the container.
## 3. The container template
On the node, once:
```sh
pveam available | grep debian-12
pveam download local debian-12-standard_12.7-1_amd64.tar.zst
```
## 4. Terraform
```sh
cd build/config/terraform
terraform init -backend-config="conn_str=$PG_CONN_STR"
```
Pin the provider to whatever this resolves and commit the resulting
`.terraform.lock.hcl`. Then:
```sh
export TF_VAR_proxmox_endpoint='https://turtle-proxmox-01.home.turtlesystems.co.uk:8006/'
export TF_VAR_proxmox_api_token='...' # homelab/ci/proxmox
export TF_VAR_lxc_template_file_id='local:vztmpl/debian-12-standard_12.7-1_amd64.tar.zst'
export TF_VAR_ssh_public_keys='["'"$(cat ~/.ssh/unraid_ansible.pub)"'"]'
terraform apply -target=module.postgres
```
`-target` on purpose: the root module also defines the Forgejo LXC, and
bringing that up is a separate decision with its own migration story
(`inventory/host_vars/forgejo.yml`). Drop the flag once you want both.
If the apply 403s on `changing feature flags (except nesting) is only allowed
for root@pam`, the container already exists without nesting enabled and the
`ansible@pam` token cannot add it. Set it as root on the node, then re-plan —
it will read clean, and Postgres's systemd unit needs it:
```sh
ssh root@turtle-proxmox-01.home.turtlesystems.co.uk 'pct set 161 --features nesting=1'
ssh root@turtle-proxmox-01.home.turtlesystems.co.uk 'pct reboot 161'
```
Check the replication job exists — this is the step most easily missed,
because a container with no job looks identical in the storage view:
```sh
ssh root@turtle-proxmox-01 'pvesr status'
```
## 5. Install Postgres into it
```sh
cd build/config/ansible
ansible-inventory -i inventory/proxmox.yml --graph # the guest should appear
ansible-playbook playbooks/proxmox.yml -e only_stacks=postgres
```
Then from the controller, confirming both that it listens on the LAN and that
the Vault password took:
```sh
psql "postgres://postgres:$PGPASSWORD@192.168.50.54:5432/postgres" -c 'SELECT version();'
```
## 6. Node backups
Replication covers a dead node, not a dropped table. `roles/pve_backup` writes
vzdump archives to the NAS:
```sh
ansible-playbook playbooks/pve_host.yml
```
## Growing the disk
Everything lives on the rootfs — that is what makes a replication snapshot
atomic (`src/shared/postgres/terraform/README.md`), so there is no second
volume to add when space runs short. Raise `disk_size` on the `postgres`
module in `build/config/terraform/main.tf` and apply.
Check the pool has the room first. Terraform will not: ZFS lets an apply
overcommit the pool happily, and the failure surfaces later as a write inside
the guest hitting ENOSPC.
```sh
ssh root@turtle-proxmox-01 'zpool list AppData; zfs list -o name,used,avail,refquota -r AppData'
ssh root@turtle-proxmox-02 'zpool list AppData' # the target needs the room too
```
Then:
```sh
cd build/config/terraform
terraform plan # expect an in-place update to disk.size, NOT a replacement
terraform apply
```
A **replacement** in that plan means something other than the size changed —
`prevent_destroy` will refuse it, which is the point. Never work around it
here; replacing this container destroys every database on it.
The resize itself is a `pct resize` of the rootfs, which on ZFS-backed storage
is a refquota change rather than a partition operation. It applies to a
running guest, ships no data, and needs nothing done inside the container
afterwards — no `resize2fs`, no Postgres restart. Confirm from inside:
```sh
ssh root@192.168.50.54 'df -h /'
```
Two things the larger number does *not* change: replication traffic, which is
a function of what actually gets written rather than of the quota, and vzdump
archive size, which covers used data only. Both track the databases, not the
headroom.
Shrinking is not available — it is a replacement, and `prevent_destroy` blocks
it — so overshoot rather than raising this every few months.
## Failing over
Replication makes the far copy a volume, not a running guest, so failover is
deliberate. Planned, with both nodes up, is an ordinary migration — fast,
because replication means only the delta has to ship:
```sh
ssh root@turtle-proxmox-01 'pct migrate 161 turtle-proxmox-02 --restart'
```
Unplanned, with the source node gone, means telling Proxmox to run the guest
from the replicated volume, and it loses every transaction committed since the
last successful send — up to `replication_schedule`, five minutes. Check what
you are about to accept first:
```sh
ssh root@turtle-proxmox-02 'pvesr status' # look at "Last Sync"
```
Automating this is what a HA resource would add, and it is deliberately not
configured: automatic failover on a two-node cluster with no third vote is a
good way to get both nodes deciding they are the survivor.
## Afterwards: repointing clients
The address is written down in several files that cannot discover it. The
authoritative copy is `ip_address` on the `postgres` module in
`build/config/terraform/main.tf`, published as the `postgres_address` output.
| Where | What |
|---|---|
| `inventory/host_vars/<guest>.yml` | `db.provision_host` on each app that declares one |
| `src/<app>/ansible/proxmox/vars.yml` | that app's own `DB_HOST` |
| `src/<app>/ansible/unraid/vars.yml` | ditto, for anything still on Unraid — no `unraid_shared` network alias reaches this host |
Apps on the k3s cluster keep using the in-cluster CloudNativePG instance and
are unaffected.
## Migrating an existing database onto it
Per database, not `pg_dumpall` — the roles are recreated by the `db:` block on
each app's own entry, so only the data needs moving:
```sh
pg_dump -h <old-host> -U postgres -Fc forgejo > forgejo.dump
# create the role and database by deploying the app once with its `db:` block,
# then:
pg_restore -h 192.168.50.54 -U postgres -d forgejo --no-owner --role=forgejo forgejo.dump
```
`--no-owner --role=` rather than a straight restore: the dump carries
ownership from the old cluster, and the role provisioning in `lxc_app` /
`compose_stack` has already created the owner here with a Vault-managed
password. Restoring ownership from the dump would fight it.
Stop the application before dumping. A dump of a live database is consistent
as of its start, so anything written during it is silently absent from the
restore.

551
docs/vault-secrets.md Normal file
View file

@ -0,0 +1,551 @@
# Vault secret layout
KV v2 mount: `kv` (matches `vault_kv_mount` in
`build/config/ansible/inventory/group_vars/all.yml`).
One path per stack, fetched whole via `community.hashi_vault.vault_kv2_get`.
Paths are prefixed `homelab/`, not `unraid/`: an app may run as a Compose
stack on Unraid or natively in a Proxmox LXC, and its secrets are the same
either way. If you populated the old `unraid/*` paths before this rename,
move them and delete the originals:
```sh
for p in shared/postgres forgejo ci/ssh; do
vault kv get -format=json "kv/unraid/$p" \
| jq -r '.data.data | to_entries | map("\(.key)=\(.value)") | .[]' \
| xargs vault kv put "kv/homelab/$p"
vault kv metadata delete "kv/unraid/$p"
done
```
## Prerequisites
- [Vault CLI](https://developer.hashicorp.com/vault/docs/install) installed
on whatever machine you're running these commands from (your workstation
during bootstrap — not the Unraid box, and not the same thing as the
Ansible controller needing `hvac`, though in practice it's the same WSL
shell for both).
- `VAULT_ADDR` set to your Vault instance:
```sh
export VAULT_ADDR='https://vault.example.internal:8200'
```
If Vault is using a self-signed cert, either trust it properly or, for a
homelab-only instance, set `export VAULT_SKIP_VERIFY=true` — don't do this
against anything reachable outside your LAN.
## Login
Log in interactively once per shell session (or whenever the token expires);
the resulting token is cached to `~/.vault-token` and picked up automatically
by later `vault` commands and by `VAULT_TOKEN` if you export it.
Token auth (simplest, fine for a single-operator homelab):
```sh
vault login
# prompts for the token
```
If you're using a different auth backend, log in with that method instead —
adjust to whatever your Vault instance actually has enabled:
```sh
vault login -method=userpass username=<your-username>
vault login -method=oidc # opens a browser
vault login -method=ldap username=<your-username>
```
Confirm the login worked and check the token's TTL:
```sh
vault token lookup
```
## One-time setup: enable the KV v2 mount
Only needed once, the first time this Vault instance is used for this repo
(skip if `kv/` already exists — check with `vault secrets list`):
```sh
vault secrets enable -path=kv -version=2 kv
```
## `homelab/shared/postgres`
| Key | Purpose |
|---|---|
| `POSTGRES_SUPERUSER` | Superuser name (matches `vars.yml` default `postgres`, but Vault is authoritative since it's paired with the password below) |
| `POSTGRES_SUPERUSER_PASSWORD` | Superuser password |
```sh
vault kv put kv/homelab/shared/postgres \
POSTGRES_SUPERUSER=postgres \
POSTGRES_SUPERUSER_PASSWORD='<generate-a-strong-password>'
```
Also seeds the k3s cluster's shared Postgres (`roles/k3s_postgres`,
CloudNativePG) — a separate physical instance from the Unraid/Proxmox ones,
but the same superuser identity, same path. See README.md → "Postgres
(CloudNativePG)".
## `homelab/forgejo`
| Key | Purpose |
|---|---|
| `DB_PASSWORD` | Password for the `forgejo` Postgres role (Ansible creates the role with this password) |
| `SECRET_KEY` | Forgejo `security.SECRET_KEY` |
| `INTERNAL_TOKEN` | Forgejo `security.INTERNAL_TOKEN` |
| `JWT_SECRET` | Forgejo `oauth2.JWT_SECRET` |
| `LFS_JWT_SECRET` | Forgejo `server.LFS_JWT_SECRET` — **Proxmox only**, see below |
| `OIDC_CLIENT_SECRET` | Authentik OAuth2 provider secret — **optional**, see below |
Generate the Forgejo secrets once and store them, rather than letting Forgejo
auto-generate on first boot — that keeps a from-scratch redeploy (fresh data
volume) reproducible instead of silently rotating tokens:
```sh
docker run --rm codeberg.org/forgejo/forgejo:10 forgejo generate secret
```
Run that four times (once per secret) and write them all together with the
DB password:
```sh
vault kv put kv/homelab/forgejo \
DB_PASSWORD='<generate-a-strong-password>' \
SECRET_KEY='<output-of-generate-secret>' \
INTERNAL_TOKEN='<output-of-generate-secret>' \
JWT_SECRET='<output-of-generate-secret>' \
LFS_JWT_SECRET='<output-of-generate-secret>'
```
`LFS_JWT_SECRET` is only consumed by the Proxmox native install, which renders
`app.ini` in full and sets `INSTALL_LOCK` — nothing is left for Forgejo to
generate on first boot, and a value it invented for itself would be overwritten
on the next deploy anyway. The Unraid Compose stack doesn't reference it; the
key being present just means one extra unused line in the rendered `.env`. If
you only run the Unraid stack today, add it before the Proxmox cutover —
`src/forgejo/ansible/proxmox/install.yml` asserts on it rather than failing
three tasks later with an undefined-variable error:
```sh
vault kv patch kv/homelab/forgejo LFS_JWT_SECRET='<output-of-generate-secret>'
```
### `OIDC_CLIENT_SECRET`
Only needed if Forgejo signs users in through Authentik (step 8 of
`docs/forgejo-proxmox.md`). Unlike every other key on this path it is **not
read by Ansible** — nothing templates it, and `install.yml` does not assert on
it, so its absence is not an error. Forgejo keeps authentication sources in
its database, not in `app.ini`, so the value is consumed once by a
`forgejo admin auth add-oauth` run and lives in the database from then on.
It is kept here anyway, rather than only in Authentik, for the same reason as
the rest: a rebuild that restores from a `pg_dump` gets the auth source back
with the dump, but a rebuild from *nothing* has to re-run that command, and
this is where it looks for the value. Generated by Authentik when you create
the provider, not by you:
```sh
vault kv patch kv/homelab/forgejo OIDC_CLIENT_SECRET='<from-the-authentik-provider>'
```
The matching client ID is not a secret and is not stored here — it is an
argument to that one command, recorded in the runbook step alongside it.
## `homelab/arr`
| Key | Purpose |
|---|---|
| `SECRET_KEY_BASE` | Signs sessions and cookies |
| `ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY` | Encrypts every credential Shelfarr stores — indexer API keys, download client passwords, OIDC secrets |
| `ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY` | As above, for values Shelfarr needs to query on |
| `ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT` | As above |
| `SHELFARR_SETTING_OIDC_CLIENT_SECRET` | Client secret of the Authentik provider for Shelfarr |
| `PROWLARR_API_KEY` | Prowlarr's API key, for anything integrating with it |
One path for the whole stack, not one per container — `compose_stack` fetches
`vault_path` once and renders every key into a single `.env` that both
services read, so Prowlarr's key belongs here rather than under a path of its
own.
Shelfarr's entrypoint generates the first four on first run if they're unset, into
`/rails/storage/.secret_key_base` and `/rails/storage/.encryption_keys` — i.e.
into the appdata volume. Set them explicitly for the same reason Forgejo's
three secrets are fixed above: a from-scratch redeploy on a fresh volume would
otherwise generate new ones, and everything encrypted with the old values
becomes unreadable.
```sh
vault kv put kv/homelab/arr \
SECRET_KEY_BASE="$(openssl rand -hex 64)" \
ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY="$(openssl rand -base64 32)" \
ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY="$(openssl rand -base64 32)" \
ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT="$(openssl rand -base64 32)"
```
Set them **before the first start**, and set the three encryption keys as a
group — the entrypoint only falls back to its generated file when the primary
key is absent, so a partial set silently mixes provided and generated values.
Changing them after Shelfarr has stored anything means re-entering every
credential through Admin → Settings.
> **Do not add `RAILS_MASTER_KEY` here.** Despite the name it is not a
> generatable secret: it decrypts the `config/credentials.yml.enc` compiled
> into the upstream image, so only upstream's own key works. Supplying a value
> of your own aborts startup with
> `ActiveSupport::MessageEncryptor::InvalidMessage` during `db:prepare`.
The OIDC client secret is the one key here that isn't generated — copy it off
the Authentik provider:
```sh
vault kv patch kv/homelab/arr \
SHELFARR_SETTING_OIDC_CLIENT_SECRET='<authentik-client-secret>'
```
`SHELFARR_SETTING_` is how upstream maps environment variables onto the
Admin → Settings store; the non-secret half of the OIDC config lives in
`src/arr/common/vars.yml`. Both halves have to be named in
`docker-compose.yml`'s `environment:` — `compose_stack` renders every Vault
key and every `env_defaults` entry into `.env`, but Compose reads `.env` only
to interpolate `${...}` in the compose file. A var that nothing references
never reaches the container, and Shelfarr just carries on with OIDC off.
Prowlarr's API key is generated the same way as Shelfarr's secrets, and for
the same reason — left unset, Prowlarr writes one of its own into
`/config/config.xml` on first start, where the only way to find it is the UI:
```sh
vault kv patch kv/homelab/arr PROWLARR_API_KEY="$(openssl rand -hex 16)"
```
Prowlarr binds config-file settings to environment as `PROWLARR__` plus the
section and key, so this reaches it as `PROWLARR__AUTH__APIKEY`. Releases
before the auth section was split read the same value as `PROWLARR__APIKEY`;
an unrecognised variable is ignored rather than fatal, so the symptom of the
wrong form is a key in the UI that doesn't match Vault, not a container that
won't start.
## `homelab/k3s-homelab-utils`
| Key | Purpose |
|---|---|
| `K3S_TOKEN` | Cluster join token — shared by the server and every agent |
Not app config — this is what makes `playbooks/k3s.yml` reproducible.
`k3s server` normally invents a token on first install and writes it to
`/var/lib/rancher/k3s/server/node-token`; every agent then has to be told that
value. Pinning it in Vault instead means a full rebuild (wipe both SD cards,
reinstall) reproduces the same cluster identity, because both the control
plane and every worker fetch the *same* value independently rather than one
generating it and handing it to the other.
Generate it once, before the first run of `playbooks/k3s.yml`:
```sh
vault kv put kv/homelab/k3s-homelab-utils \
K3S_TOKEN="$(openssl rand -hex 32)"
```
Changing this after the cluster is up means re-running the playbook against
every node — the server re-issues its cert bundle around the new token and
every agent needs to reconnect with it, which is disruptive enough that
there's no automatic rotation path here, only a manual one.
## `homelab/authentik`
| Key | Purpose |
|---|---|
| `AUTHENTIK_SECRET_KEY` | Signs session cookies |
| `AUTHENTIK_POSTGRESQL__PASSWORD` | Password for the `authentik` role on the cluster's shared Postgres |
The first app on the k3s cluster, deployed by `roles/k3s_app` from the
`k3s_apps:` list in `inventory/group_vars/k3s_cluster.yml`. Both keys are
named for the environment variables Authentik itself reads, because
`roles/k3s_app` passes Vault keys through to the app's Kubernetes Secret
verbatim — so one value at `AUTHENTIK_POSTGRESQL__PASSWORD` serves both the
database provisioning step and the running app, with nothing restated.
```sh
vault kv put kv/homelab/authentik \
AUTHENTIK_SECRET_KEY='<carried over from the Unraid deployment>' \
AUTHENTIK_POSTGRESQL__PASSWORD="$(openssl rand -base64 32)"
```
`AUTHENTIK_SECRET_KEY` is the exception to "generate a fresh secret" — it
must be copied from the existing Unraid deployment, not invented. It signs
session cookies, so a new value logs every user out at the moment DNS moves.
See `docs/authentik-migration.md`, which is the only reason this path has a
"copy the old value" step at all; a from-scratch install would generate both.
## `homelab/k3s-cert-manager`
| Key | Purpose |
|---|---|
| `CLOUDFLARE_API_TOKEN` | DNS provider API token for cert-manager's ACME DNS-01 solver |
Consumed by `roles/k3s_cert_manager`, which renders every key at this path
into a Secret in the `cert-manager` namespace — so the key name above is a
convention, not a requirement: store whatever your DNS provider needs and
reference that name from `k3s_cert_manager_solver` in
`inventory/group_vars/k3s_cluster.yml`.
```sh
vault kv put kv/homelab/k3s-cert-manager \
CLOUDFLARE_API_TOKEN='<token with Zone:DNS:Edit on the zone>'
```
DNS-01 rather than HTTP-01 because HTTP-01 needs Let's Encrypt to reach the
cluster from the internet on port 80, and this LAN deliberately isn't
reachable. Scope the token to the single zone it manages — cert-manager only
ever needs to write `_acme-challenge` TXT records.
## `homelab/ci/ssh`
Not consumed by `compose_stack` — this is the SSH private key Ansible itself
connects to the Unraid hosts with. It lives in Vault rather than as a
Forgejo Actions secret so there's one place secrets come from, not two; see
README.md "SSH access" for generating the key pair and installing the public
half on each host.
| Key | Purpose |
|---|---|
| `PRIVATE_KEY` | OpenSSH private key `ansible_user` (root) authenticates with |
```sh
# `@path` reads the value from a file, which handles the embedded newlines
# in a private key correctly — don't try to inline it as a normal KEY=value.
# Use $HOME, not `~`: this is a plain command argument, not a shell
# assignment, so bash won't tilde-expand a `~` sitting after the `@` — it'd
# get passed through literally and vault would look for a file named `~`.
vault kv put kv/homelab/ci/ssh PRIVATE_KEY=@"$HOME/.ssh/unraid_ansible"
```
## `homelab/ci/ssh-k3s`
The SSH private key Ansible connects to the 4 k3s Pis with. A separate key
pair from `homelab/ci/ssh` above, deliberately — the two host groups don't
share a trust boundary, so a leaked key for one shouldn't also be a working
key for the other. Same shape as the Unraid key otherwise:
| Key | Purpose |
|---|---|
| `PRIVATE_KEY` | OpenSSH private key the `ansible` user (per `inventory/hosts.yml` → `k3s_cluster`) authenticates with |
```sh
ssh-keygen -t ed25519 -f ~/.ssh/k3s_ansible -C "ansible@homelab-utils" -N ""
```
Create the `ansible` user on each of the 4 Pis with NOPASSWD sudo and install
the public half as its `authorized_keys` — see README.md "SSH access". Then:
```sh
vault kv put kv/homelab/ci/ssh-k3s PRIVATE_KEY=@"$HOME/.ssh/k3s_ansible"
```
Unlike the Unraid key above, nothing needs loading into `ssh-agent` to use
this: `playbooks/k3s_ssh_key.yml` reads this path and writes the key to
`local/k3s/homelab-utils.key` on the controller, and both `playbooks/k3s.yml`
and `playbooks/k3s_maintenance.yml` import it as their first play. A manual
run needs only `VAULT_ADDR`/`VAULT_TOKEN` in the environment (the "Login"
section above — no AppRole involvement yet, both k3s playbooks being
manual-only for now). See README.md "SSH access" for the fallback flags if
Vault is unreachable.
## `homelab/ci/proxmox`
Proxmox VE API token, used by two things: the `community.proxmox` dynamic
inventory plugin (`build/config/ansible/inventory/proxmox.yml`), and the
Terraform provider that creates LXCs. A token rather than a password — same
reasoning as the SSH key, a password means an interactive prompt or a
plaintext secret on disk.
Create the user and its token on either node (ACLs are cluster-wide), then
grant it a role. This is more involved than it looks, and the failure modes
give errors that don't obviously point at permissions:
```sh
pveum user add ansible@pam
pveum user token add ansible@pam terraform --privsep 0
pveum role add TerraformProv -privs "Datastore.Allocate \
Datastore.AllocateSpace Datastore.AllocateTemplate Datastore.Audit \
Pool.Allocate SDN.Use Sys.Audit Sys.Console Sys.Modify \
VM.Allocate VM.Audit VM.Clone VM.Console VM.Migrate VM.PowerMgmt \
VM.Replicate \
VM.Config.CPU VM.Config.Disk VM.Config.HWType VM.Config.Memory \
VM.Config.Network VM.Config.Options"
pveum acl modify / --users 'ansible@pam' --roles TerraformProv
```
Three things that otherwise cost an afternoon:
- **`--privsep 0` is load-bearing.** A token created with privilege separation
on (the UI default) carries its *own* ACL and ignores what the user was
granted, so `pveum acl modify --users` looks like it did nothing. Either
turn it off as above, or grant the token as well with
`--tokens 'ansible@pam!terraform'`. With a purpose-made user rather than
`root@pam` there is nothing gained by maintaining both layers — least
privilege is already enforced on the user.
- **Privilege names are version-sensitive.** `VM.Monitor` appears in most
guides for this and is rejected outright by current PVE (it is a QEMU
monitor privilege, meaningless for LXC). `SDN.Use` is required from PVE 8.2
on, where attaching a container to `vmbr0` is gated on
`/sdn/zones/localnetwork/vmbr0`. Check against your own node rather than
against any list, this one included: `pvesh get /access/roles/Administrator`
prints the complete valid set.
- **`VM.Replicate` is there for the shared Postgres**, whose module creates a
`pvesr` job (`replication_target_node` — see
`src/shared/postgres/terraform/`). A module without replication won't need
it.
Verify the grant landed before reaching for Terraform — this answers the
token's effective permissions, not the user's:
```sh
pveum user permissions ansible@pam --token terraform
```
Every `terraform apply` failure here is an HTTP 403 naming the exact missing
privilege, so the recovery loop is `pveum role modify TerraformProv --privs
"<name>" --append` and re-run. Note `--append`: without it, `--privs` replaces
the whole list rather than adding to it.
Then:
| Key | Purpose |
|---|---|
| `PROXMOX_URL` | API endpoint, e.g. `https://turtle-proxmox-01.home.turtlesystems.co.uk:8006/` |
| `PROXMOX_USER` | Token owner, e.g. `ansible@pam` |
| `PROXMOX_TOKEN_ID` | Token ID (the part after `!`) |
| `PROXMOX_TOKEN_SECRET` | Token secret — shown once at creation |
```sh
vault kv put kv/homelab/ci/proxmox \
PROXMOX_URL='https://turtle-proxmox-01.home.turtlesystems.co.uk:8006/' \
PROXMOX_USER='ansible@pam' \
PROXMOX_TOKEN_ID='terraform' \
PROXMOX_TOKEN_SECRET='<shown-once-at-creation>'
```
Ansible's inventory plugin reads these from the environment, so export them
before running anything that touches `proxmox_guests`. Terraform wants the
token in one combined string instead:
```sh
export TF_VAR_proxmox_api_token="${PROXMOX_USER}!${PROXMOX_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}"
```
## `homelab/ci/terraform`
Connection string for the Postgres database Terraform keeps its state in
(`backend "pg"` — see `build/config/terraform/README.md`). Separate from the
superuser credentials above: Terraform gets its own role, scoped to its own
database.
| Key | Purpose |
|---|---|
| `PG_CONN_STR` | `postgres://terraform:<password>@192.168.50.81:5432/terraform_state` |
The host is the **CloudNativePG cluster on k3s**
(`k3s_postgres_loadbalancer_ip`), not either shared Postgres this repo
deploys — Terraform cannot keep its state in a database it provisions itself.
Create the role and database once, against that cluster, then store the
string:
```sh
vault kv put kv/homelab/ci/terraform \
PG_CONN_STR='postgres://terraform:<generate-a-strong-password>@192.168.50.81:5432/terraform_state'
```
## Verifying what was written
```sh
vault kv get kv/homelab/shared/postgres
vault kv get kv/homelab/forgejo
```
`vault kv put` (used above) replaces the whole secret at that path — to add
or change a single key without touching the others, use `vault kv patch`
instead:
```sh
vault kv patch kv/homelab/forgejo DB_PASSWORD='<new-password>'
```
## Adding a new stack's secrets
```sh
vault kv put kv/homelab/<stack-name> KEY1=value1 KEY2=value2
```
Then reference the same path as `vault_path` in the stack's entry under
`build/config/ansible/inventory/host_vars/<host>.yml`.
## AppRole for CI (Forgejo Actions)
The bootstrap commands above use your own human token. The Forgejo Actions
runner (`.forgejo/workflows/deploy.yml`) authenticates as an AppRole instead,
scoped to read-only on the `kv/homelab/*` paths. One-time setup:
```sh
# 1. Policy: read-only on every stack's secrets
vault policy write unraid-deploy - <<'EOF'
path "kv/data/homelab/*" {
capabilities = ["read"]
}
EOF
# 2. Enable AppRole auth (skip if already enabled — check `vault auth list`)
vault auth enable approle
# 3. Create the role, bound to that policy
vault write auth/approle/role/unraid-deploy \
token_policies="unraid-deploy" \
token_ttl=15m \
token_max_ttl=1h
# 4. Fetch the role ID (stable, not secret on its own)
vault read auth/approle/role/unraid-deploy/role-id
# 5. Generate a secret ID (treat as a secret — shown once)
vault write -f auth/approle/role/unraid-deploy/secret-id
```
Store the `role_id` and `secret_id` from steps 4–5 as `VAULT_ROLE_ID` and
`VAULT_SECRET_ID` in the Forgejo repo's Actions secrets (alongside
`VAULT_ADDR`) — that's what `deploy.yml` and
`build/config/ansible/inventory/group_vars/all.yml`
(`vault_auth_method: approle`) expect.
These never live in a file in this repo, in either mode:
- **CI:** Forgejo repo → Settings → Actions → Secrets. `deploy.yml` reads
them from there via `${{ secrets.VAULT_ROLE_ID }}` /
`${{ secrets.VAULT_SECRET_ID }}` and exports them as job env vars.
- **Manual runs** (bootstrap, or any ad hoc `ansible-playbook` invocation):
export them as shell env vars first — `group_vars/all.yml` picks them up
via `lookup('env', ...)`:
```sh
export VAULT_AUTH_METHOD=approle
export VAULT_ROLE_ID='<from step 4>'
export VAULT_SECRET_ID='<from step 5>'
```
(Bootstrap can also just use your own human token instead — leave
`VAULT_AUTH_METHOD` unset, it defaults to `token`, and `vault login` from
the "Login" section above is enough.)
Secret IDs can be regenerated (step 5) and old ones revoked without touching
the role itself if one ever leaks:
```sh
vault write auth/approle/role/unraid-deploy/secret-id-accessor/destroy \
secret_id_accessor='<accessor-from-step-5-output>'
```

22
src/arr/ansible/proxmox/README.md Normal file
View file

@ -0,0 +1,22 @@
# ansible/proxmox/ — arr
Not built. Shelfarr runs as a Compose stack on nas1 (`../unraid/`), and there
is no reason to move it — this folder exists because every app keeps the same
shape whether or not both platforms are in use.
A native install would be a poor fit here in a way most apps aren't. Shelfarr
is a Rails app shipping two containers' worth of runtime (Ruby, a bundled
Solid Queue worker, and a .NET companion for the Libation sidecar), with no
released binary or package — upstream distributes container images only. Where
`forgejo` drops a single Go binary and a systemd unit into an LXC,
reproducing this natively means building the Rails app from source on the
guest.
If it ever needs to move, the realistic option is Docker inside a privileged
LXC rather than the `lxc_app` role, which would be a new deployment primitive
for this repo — decide that deliberately rather than by extending the role.
What wouldn't change: `../../common/vars.yml` already holds the portable
config (version, published port), and the Vault path stays `homelab/arr`.
Only paths and PUID/PGID are Unraid-specific, and those live in
`../unraid/vars.yml`.

101
src/arr/ansible/unraid/.env.example Normal file
View file

@ -0,0 +1,101 @@
# Reference only — real values are rendered by Ansible from vars.yml (non-
# secret) and Vault path homelab/arr (secret). Do not fill this in and
# deploy it directly.
# Non-secret (see ../../common/vars.yml)
SHELFARR_VERSION=2026.08.05.1
SHELFARR_HTTP_PORT=5056
# Non-secret (see vars.yml) — check the media paths against your own shares
SHELFARR_DATA_PATH=/mnt/user/appdata/arr/data
SHELFARR_AUDIOBOOKS_PATH=/mnt/user/Media/AudioBooks
SHELFARR_EBOOKS_PATH=/mnt/user/Media/Books
SHELFARR_DOWNLOADS_PATH=/mnt/user/downloads
PUID=99
PGID=100
CHOWN_ON_START=auto
# Docker network to join, and it must already exist on the target host —
# Compose declares it `external`, so it will not create it. Defined in
# Unraid's Docker settings, not by any stack in this repo.
SHELFARR_NETWORK=caddy-net
# Non-secret, supplied by the compose_stack role rather than either vars.yml —
# the path it copies ansible/unraid/icon.png to on the host, for the
# net.unraid.docker.icon label. Empty when the stack ships no icon.png, which
# just leaves Unraid's question-mark placeholder. Override it in vars.yml to
# point at a hosted PNG instead.
STACK_ICON=/mnt/user/appdata/icons/arr.png
# Secret — Vault key: SECRET_KEY_BASE
# Signs sessions and cookies. Generate with:
# openssl rand -hex 64
SECRET_KEY_BASE=
# Secret — Vault keys: ACTIVE_RECORD_ENCRYPTION_*
# Encrypt every credential Shelfarr stores (indexer API keys, download client
# passwords). Generate each with:
# openssl rand -base64 32
# Set these before the first start, and set all three together. Changing them
# later makes everything already encrypted with the old values unreadable.
ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY=
ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY=
ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT=
# Non-secret (see ../../common/vars.yml) — OIDC against Authentik.
# Upstream maps environment onto the settings store by SHELFARR_SETTING_ plus
# the uppercased setting key. There is no redirect-URI variable: the callback
# is fixed at <shelfarr-url>/auth/oidc/callback and is registered on the
# Authentik provider.
SHELFARR_SETTING_OIDC_ENABLED=true
SHELFARR_SETTING_OIDC_PROVIDER_NAME=Authentik
SHELFARR_SETTING_OIDC_ISSUER=https://auth.turtlesystems.uk/application/o/shelfarr/
SHELFARR_SETTING_OIDC_CLIENT_ID=
SHELFARR_SETTING_OIDC_SCOPES=openid profile email
SHELFARR_SETTING_OIDC_AUTO_CREATE_USERS=true
SHELFARR_SETTING_OIDC_DEFAULT_ROLE=user
SHELFARR_SETTING_OIDC_LINK_EXISTING_USERS=true
SHELFARR_SETTING_OIDC_AUTO_REDIRECT=false
# Secret — Vault key: SHELFARR_SETTING_OIDC_CLIENT_SECRET
# The client secret of the Authentik provider for Shelfarr.
SHELFARR_SETTING_OIDC_CLIENT_SECRET=
# Do NOT set RAILS_MASTER_KEY. It decrypts the credentials file built into the
# image, so only upstream's key works — a self-generated one aborts startup
# with ActiveSupport::MessageEncryptor::InvalidMessage.
# Only if the Libation companion is enabled — see docker-compose.yml
# LIBATION_CONFIG_PATH=/mnt/user/appdata/arr/libation/config
# LIBATION_BOOKS_PATH=/mnt/user/appdata/arr/libation/books
# LIBATION_CONTROL_PATH=/mnt/user/appdata/arr/libation/control
# Only if running behind a reverse proxy at a sub-path, e.g. /arr
# RAILS_RELATIVE_URL_ROOT=/
# --- Prowlarr ---------------------------------------------------------------
# Non-secret (see ../../common/vars.yml). LinuxServer tags are the upstream
# version plus their build suffix (1.37.0.5076-ls117); pin to a full one read
# off the registry rather than leaving this at latest.
PROWLARR_VERSION=latest
PROWLARR_HTTP_PORT=9696
# Non-secret, supplied by the compose_stack role — the path it copies
# icon-prowlarr.png to, for the prowlarr container's net.unraid.docker.icon
# label. Per-service, so it's separate from STACK_ICON above, which is the
# stack's own icon.png and labels shelfarr.
STACK_ICON_PROWLARR=/mnt/user/appdata/icons/arr-prowlarr.png
# Non-secret (see vars.yml)
PROWLARR_CONFIG_PATH=/mnt/user/appdata/arr/prowlarr
TZ=Europe/London
# Secret — Vault key: PROWLARR_API_KEY
# Bound to Prowlarr's config.xml via PROWLARR__AUTH__APIKEY, so the key is
# known before first boot and anything integrating with Prowlarr can be
# configured from the same Vault path. Generate with:
# openssl rand -hex 16
# Leave it out and Prowlarr generates its own on first start — then the value
# only exists in the UI and in /config/config.xml.
PROWLARR_API_KEY=

252
src/arr/ansible/unraid/docker-compose.yml Normal file
View file

@ -0,0 +1,252 @@
services:
shelfarr:
image: ghcr.io/pedro-revez-silva/shelfarr:${SHELFARR_VERSION}
container_name: arr
restart: unless-stopped
# Unraid's Docker page normally reads icon, WebUI link and console shell
# from the dockerMan template that created the container. A Compose stack
# has no template, so 6.10+ falls back to these labels; without them the
# container renders as a question mark with no WebUI or Console entry.
#
# STACK_ICON comes from the compose_stack role (a path under
# /mnt/user/appdata/icons, or a URL if vars.yml overrides it). `[IP]` is
# substituted by the webgui with the host's address; the port has to be
# the published one, hence SHELFARR_HTTP_PORT rather than the container's
# 80. If you drop the `ports:` mapping below and reach Shelfarr only
# through Caddy, replace the whole value with the proxied URL — the
# webgui has no way to know about the reverse proxy.
labels:
net.unraid.docker.icon: ${STACK_ICON}
net.unraid.docker.webui: "http://[IP]:${SHELFARR_HTTP_PORT}/"
net.unraid.docker.shell: bash
networks:
- proxy
environment:
# Runs the Solid Queue background worker inside the Puma process rather
# than as a second container. Upstream's own compose does this; Shelfarr
# is a single-user-scale app and a separate worker buys nothing here.
SOLID_QUEUE_IN_PUMA: "1"
PUID: ${PUID}
PGID: ${PGID}
CHOWN_ON_START: ${CHOWN_ON_START}
# Deliberately NOT RAILS_MASTER_KEY. That variable decrypts the
# config/credentials.yml.enc baked into the image at build time, so only
# upstream's own key works — setting it to a generated value aborts boot
# with ActiveSupport::MessageEncryptor::InvalidMessage.
#
# These four are the ones the entrypoint actually treats as overridable,
# and pinning them is what stops a fresh appdata volume from orphaning
# stored credentials (same reasoning as forgejo's SECRET_KEY). Left
# unset, the entrypoint generates them into /rails/storage/.secret_key_base
# and /rails/storage/.encryption_keys — fine until that directory is lost.
SECRET_KEY_BASE: ${SECRET_KEY_BASE}
# Encrypt every credential Shelfarr stores (indexer API keys, download
# client passwords). All three must be set together; the entrypoint only
# falls back to its generated file when the primary key is absent.
ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY: ${ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY}
ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY: ${ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY}
ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT: ${ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT}
# --- OIDC (Authentik) ------------------------------------------------
#
# Upstream maps environment onto the Admin → Settings store by
# `SHELFARR_SETTING_` + the uppercased setting key, so these ten are the
# whole of the SSO config; there is no redirect-URI variable, the
# callback path is fixed at /auth/oidc/callback and is registered on the
# Authentik side.
#
# Every one has to be named here. The .env `compose_stack` renders holds
# them already, but Compose reads .env only to interpolate `${...}` in
# this file — it does not inject it into the container. An unreferenced
# var reaches the host's Compose process and stops there, which is why
# OIDC stayed off with the values apparently set.
SHELFARR_SETTING_OIDC_ENABLED: ${SHELFARR_SETTING_OIDC_ENABLED}
SHELFARR_SETTING_OIDC_PROVIDER_NAME: ${SHELFARR_SETTING_OIDC_PROVIDER_NAME}
SHELFARR_SETTING_OIDC_ISSUER: ${SHELFARR_SETTING_OIDC_ISSUER}
SHELFARR_SETTING_OIDC_CLIENT_ID: ${SHELFARR_SETTING_OIDC_CLIENT_ID}
# Secret — Vault key of the same name, not either vars.yml.
SHELFARR_SETTING_OIDC_CLIENT_SECRET: ${SHELFARR_SETTING_OIDC_CLIENT_SECRET}
SHELFARR_SETTING_OIDC_SCOPES: ${SHELFARR_SETTING_OIDC_SCOPES}
SHELFARR_SETTING_OIDC_AUTO_CREATE_USERS: ${SHELFARR_SETTING_OIDC_AUTO_CREATE_USERS}
SHELFARR_SETTING_OIDC_DEFAULT_ROLE: ${SHELFARR_SETTING_OIDC_DEFAULT_ROLE}
SHELFARR_SETTING_OIDC_LINK_EXISTING_USERS: ${SHELFARR_SETTING_OIDC_LINK_EXISTING_USERS}
SHELFARR_SETTING_OIDC_AUTO_REDIRECT: ${SHELFARR_SETTING_OIDC_AUTO_REDIRECT}
volumes:
- ${SHELFARR_DATA_PATH}:/rails/storage
- ${SHELFARR_AUDIOBOOKS_PATH}:/audiobooks
- ${SHELFARR_EBOOKS_PATH}:/ebooks
- ${SHELFARR_DOWNLOADS_PATH}:/downloads
ports:
- "${SHELFARR_HTTP_PORT}:80"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:80/up"]
interval: 30s
timeout: 10s
retries: 3
start_period: 40s
# Indexer manager for the rest of the stack. Shelfarr doesn't talk to
# trackers itself — it asks Prowlarr, which owns the indexer definitions and
# their credentials. Both are on `proxy`, so Shelfarr reaches it as
# `http://prowlarr:9696` (Admin → Settings → Indexers) rather than via nas1's
# LAN IP and the published port.
prowlarr:
image: lscr.io/linuxserver/prowlarr:${PROWLARR_VERSION}
container_name: prowlarr
restart: unless-stopped
# Same reasoning as the labels on `shelfarr` above — a Compose stack has no
# dockerMan template, so without these the container is a question mark
# with no WebUI entry. STACK_ICON_PROWLARR rather than STACK_ICON: labels
# are per-service, and ${STACK_ICON} is the stack's own icon.png, which is
# Shelfarr's. This one comes from the committed icon-prowlarr.png in this
# directory — see README.md → "Icons on the Unraid Docker page".
labels:
net.unraid.docker.icon: ${STACK_ICON_PROWLARR}
net.unraid.docker.webui: "http://[IP]:${PROWLARR_HTTP_PORT}/"
# Alpine-based, so `sh` — same as forgejo, not `bash` like the
# Debian-based shelfarr above.
net.unraid.docker.shell: sh
networks:
- proxy
environment:
PUID: ${PUID}
PGID: ${PGID}
# LinuxServer's init sets the container clock from this. It matters more
# here than for Shelfarr: Prowlarr timestamps search history and applies
# per-indexer rate limits, and both read wrong at UTC.
TZ: ${TZ}
# Prowlarr binds its config file to environment via `PROWLARR__` plus the
# config section and key, double-underscore separated. Pinning the API
# key means Shelfarr's indexer config can be written from the same Vault
# path instead of being copied by hand out of the UI after first boot —
# the same argument as SECRET_KEY_BASE above, applied to a value that is
# otherwise generated into /config/config.xml on first start.
#
# Older releases read this as PROWLARR__APIKEY (no section). If the key
# in the UI doesn't match Vault after a deploy, that's which form this
# image wants; an unrecognised variable is ignored silently rather than
# failing the container, so the symptom is a generated key, not a crash.
PROWLARR__AUTH__APIKEY: ${PROWLARR_API_KEY}
volumes:
# config.xml, the indexer definitions, and Prowlarr's own SQLite
# database. This directory *is* the application state — same standing as
# SHELFARR_DATA_PATH, back it up the same way.
- ${PROWLARR_CONFIG_PATH}:/config
ports:
- "${PROWLARR_HTTP_PORT}:9696"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9696/ping"]
interval: 30s
timeout: 10s
retries: 3
start_period: 40s
networks:
# `external: true` means "attach to a network that already exists; don't
# create it, don't delete it on `down`". Without it Compose would create a
# *new* network named `arr_<key>` — the deploy would still succeed and
# Caddy would silently fail to resolve `arr`. `name:` is what stops the
# project-name prefix being applied.
#
# The local alias is a fixed `proxy` so the compose file stays static; which
# real network it maps to is a per-host decision in vars.yml
# (SHELFARR_NETWORK). Same shape as forgejo's `shared` → `unraid_shared`.
#
# The alias MUST be a literal, not `${SHELFARR_NETWORK}:`. Compose
# interpolates values, never mapping keys, so a variable used as the key
# stays the literal string `${SHELFARR_NETWORK}` while the service's
# `networks:` entry (a list value) expands to `caddy-net` — and the two no
# longer match:
# service "arr" refers to undefined network caddy-net
# `name:` below is a value, so it interpolates normally. That asymmetry is
# the whole reason for the alias indirection.
#
# caddy-net is defined in Unraid's Docker settings on nas1 rather than by a
# stack in this repo, so nothing here has to deploy before Shelfarr does.
# But it must exist on the target host — an absent external network fails
# the deploy outright, which is why `unraid_shared` (created by the shared
# Postgres stack, and that runs on nas2) is deliberately not used here.
# Shelfarr is SQLite-only and needs no database anyway.
proxy:
name: ${SHELFARR_NETWORK}
external: true
# Now that Caddy and Shelfarr share a network, Caddy should proxy to
# `arr:80` — the container name on a user-defined network — rather than
# to nas1's LAN IP and the published port. The `ports:` mapping above is kept
# only for direct access that bypasses the proxy; drop it if you don't want
# 5056 reachable on the LAN.
#
# Reaching the download client and indexers stays a runtime concern, not a
# compose one: they're configured in Admin → Settings. Anything not on
# caddy-net is addressed via nas1's LAN IP and its published port
# (e.g. http://192.168.50.1:8080), which works whichever network it's on.
# --- Import layout ---------------------------------------------------------
#
# The three media binds above follow upstream's documented layout, which works
# regardless of how your shares are arranged. The cost: /downloads and
# /audiobooks are separate mounts inside the container, and rename() across
# mounts returns EXDEV even when both sit on the same Unraid filesystem — so
# every import is a full copy rather than an instant move.
#
# If downloads and the library live under one share, replace the three binds
# with a single one and set the paths in Admin → Settings to match:
#
# - ${SHELFARR_MEDIA_PATH}:/media # e.g. /mnt/user/media
#
# Admin → Settings: /media/downloads, /media/audiobooks, /media/ebooks
#
# One mount means one filesystem, so imports become atomic moves and hardlinks
# work. Worth doing if your download client is already writing under the same
# share.
# --- Libation companion (Audible backup, beta) -----------------------------
#
# Left out for now; purely additive. To enable it: uncomment the service and
# the `volumes:` block below, add these three to the main service's
# environment, and add the two mounts to its `volumes:`, then redeploy. The
# appdata bind survives the container recreate, so nothing is lost.
#
# environment:
# SHELFARR_LIBATION_URL: http://shelfarr-libation:8080
# SHELFARR_LIBATION_TOKEN_FILE: /run/shelfarr-libation/token
# SHELFARR_LIBATION_IMPORT_ROOT: /imports/libation
# volumes:
# - ${LIBATION_BOOKS_PATH}:/imports/libation:ro
# - ${LIBATION_CONTROL_PATH}:/run/shelfarr-libation:ro
#
# Note the two services must then share a user-defined network so the main
# container can resolve `shelfarr-libation` by name — the default bridge has no
# DNS between containers.
#
# shelfarr-libation:
# image: ghcr.io/pedro-revez-silva/shelfarr-libation:${SHELFARR_VERSION}
# container_name: shelfarr-libation
# restart: unless-stopped
# expose:
# - "8080"
# environment:
# PUID: ${PUID}
# PGID: ${PGID}
# CHOWN_ON_START: ${CHOWN_ON_START}
# LIBATION_FILES_DIR: /config
# LIBATION_BOOKS_DIR: /data
# LIBATION_IN_PROGRESS_DIR: /config/in-progress
# COMPANION_STATE_DIR: /config/shelfarr-companion
# COMPANION_TOKEN_FILE: /control/token
# ASPNETCORE_URLS: http://0.0.0.0:8080
# volumes:
# - ${LIBATION_CONFIG_PATH}:/config
# - ${LIBATION_BOOKS_PATH}:/data
# - ${LIBATION_CONTROL_PATH}:/control
# healthcheck:
# test: ["CMD", "/companion/Shelfarr.Libation.Companion", "--healthcheck"]
# interval: 30s
# timeout: 10s
# retries: 3
# start_period: 40s
#
# Upstream uses named volumes for libation_config/books/control. Bind mounts
# under /mnt/user/appdata/arr/ instead, to match every other stack here
# and so the data is visible on the array rather than buried in Docker's
# storage. Add the paths to ansible/unraid/vars.yml when enabling.

BIN
src/arr/ansible/unraid/icon-prowlarr.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 25 KiB

BIN
src/arr/ansible/unraid/icon-shelfarr.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 126 KiB

59
src/arr/ansible/unraid/vars.yml Normal file
View file

@ -0,0 +1,59 @@
---
# Unraid-specific overrides, merged over ../../common/vars.yml (these win).
env_defaults:
# SQLite lives here — all four production databases (primary, cache, queue,
# cable) plus Active Storage. Shelfarr has no external database, so unlike
# forgejo this directory *is* the application state, and it is also where
# everything configured in Admin → Settings is kept. Back it up accordingly.
SHELFARR_DATA_PATH: /mnt/user/appdata/arr/data
# CHECK THESE AGAINST YOUR OWN SHARES before the first deploy — they are
# conventional Unraid layouts, not something read off nas1.
#
# `/mnt/user/...`, never `/mnt/cache/...` or `/mnt/disk1/...`: the latter
# bypass the user-share layer and strand files where the mover won't find
# them.
#
# See the note in docker-compose.yml about keeping downloads and the library
# on one mount — as three separate binds, imports are copies rather than
# instant moves.
SHELFARR_AUDIOBOOKS_PATH: /mnt/user/Media/Books/audio
SHELFARR_EBOOKS_PATH: /mnt/user/Media/Books/ebooks
SHELFARR_DOWNLOADS_PATH: /mnt/user/downloads
# 99:100 is nobody:users — what Unraid owns share content as. Upstream
# defaults to 1000:1000, which is a normal Linux desktop user and wrong
# here; files written as 1000 are invisible to everything else on the box.
PUID: "99"
PGID: "100"
# `auto` chowns only when it has to. `always` would fight Unraid's own
# permissions handling on every restart; `never` risks a container that
# can't write to its own appdata after a share rebuild.
CHOWN_ON_START: auto
# Prowlarr's config.xml, indexer definitions and SQLite database. Sibling of
# SHELFARR_DATA_PATH under the same stack directory — one appdata folder per
# stack, one subdirectory per container.
PROWLARR_CONFIG_PATH: /mnt/user/appdata/arr/prowlarr
# Container clock. Prowlarr timestamps its search history and enforces
# per-indexer rate limits against it; Shelfarr takes its time from the Rails
# default and doesn't read this.
TZ: Europe/London
# The stack's Docker network — both containers join it, which is how
# Shelfarr resolves `prowlarr` by name.
SHELFARR_NETWORK: caddy-net
# --- Unraid Docker page presentation (net.unraid.docker.* labels) --------
#
# STACK_ICON is normally left to the compose_stack role, which points it at
# /mnt/user/appdata/icons/<stack>.png when this directory contains an
# icon.png. Uncomment to use a hosted icon instead of committing one:
#
# STACK_ICON: https://example.org/arr.png
#
# PNG only — Unraid renders its placeholder for SVG and nothing at all for
# WebP, and a URL it can't reach when the Docker page renders fails the same
# way, which is the argument for committing the file.

64
src/arr/common/vars.yml Normal file
View file

@ -0,0 +1,64 @@
---
# Portable config — values that hold regardless of where Shelfarr runs.
# Platform-specific values (filesystem paths, the PUID/PGID the container
# writes as) live in ../ansible/<platform>/vars.yml and are merged over these
# at deploy time.
env_defaults:
# Pinned rather than `latest`. Upstream tags releases YYYY.MM.DD.N and the
# image tag carries no leading `v`, unlike the git tag — `v2026.08.05.1` is
# the release, `2026.08.05.1` is the image. One tag pins both the app image
# and the (currently unused) Libation companion.
SHELFARR_VERSION: "2026.08.09.1"
# Published port. 5056 is upstream's convention; the container listens on 80
# internally, which is left at its default rather than being made
# configurable — there is nothing else in the container to collide with.
SHELFARR_HTTP_PORT: "5056"
# SECRET_KEY_BASE and the three ACTIVE_RECORD_ENCRYPTION_* values are
# secrets — sourced from Vault (homelab/arr), never set here. The
# encryption trio is not just a session key: it encrypts every credential
# Shelfarr stores (indexer API keys, download client passwords), so losing
# it means re-entering all of them.
#
# RAILS_MASTER_KEY is deliberately absent. It belongs to the
# config/credentials.yml.enc compiled into the image, so any value you
# generate yourself fails to decrypt it and the container won't boot. See
# the note in ../ansible/unraid/docker-compose.yml.
SHELFARR_SETTING_OIDC_ENABLED: "true"
SHELFARR_SETTING_OIDC_PROVIDER_NAME: Authentik
SHELFARR_SETTING_OIDC_ISSUER: https://auth.turtlesystems.uk/application/o/shelfarr/
SHELFARR_SETTING_OIDC_CLIENT_ID: jf1IKMxxYjgZxZikvONdFfOTauFrFiC4AYmJEbmN
SHELFARR_SETTING_OIDC_SCOPES: openid profile email
SHELFARR_SETTING_OIDC_AUTO_CREATE_USERS: "true"
SHELFARR_SETTING_OIDC_DEFAULT_ROLE: user
SHELFARR_SETTING_OIDC_LINK_EXISTING_USERS: "true"
SHELFARR_SETTING_OIDC_AUTO_REDIRECT: "false"
# --- Prowlarr --------------------------------------------------------------
#
# LinuxServer's tags are the upstream version plus their own build suffix
# (`1.37.0.5076-ls117`), and the suffix can't be derived from the release
# number — so a pin has to be read off the registry rather than guessed.
# `latest` until then; check
# https://github.com/linuxserver/docker-prowlarr/pkgs/container/prowlarr and
# replace this with the full tag, which is what every other version in this
# repo does.
PROWLARR_VERSION: latest
# Published port. 9696 is upstream's default and what the container listens
# on internally; kept the same on the host so the WebUI label, the LAN URL
# and the docs all read alike.
PROWLARR_HTTP_PORT: "9696"
# PROWLARR_API_KEY is a secret — Vault (homelab/arr), never set here.
# Generate a 32-character hex string: openssl rand -hex 16
# Prowlarr generates its own into /config/config.xml if this is left empty,
# which works fine but means the value is only discoverable from the UI.
# STACK_ICON_PROWLARR is supplied by the compose_stack role, from the
# committed ../ansible/unraid/icon-prowlarr.png — not set here. Uncomment to
# use a hosted icon instead, exactly as with STACK_ICON:
#
# STACK_ICON_PROWLARR: https://example.org/prowlarr.png

11
src/arr/terraform/README.md Normal file
View file

@ -0,0 +1,11 @@
# terraform/ — arr
Not built, and not expected to be. Terraform's only job in this repo is
creating Proxmox LXCs; Shelfarr runs as a Compose stack on nas1, where the
host already exists.
This folder is a placeholder so the app keeps the same
`common/` + `ansible/` + `terraform/` shape as every other one. If Shelfarr
ever moves to Proxmox it would need an LXC defined here and wired into
`build/config/terraform/main.tf` — see `../ansible/proxmox/README.md` first
for why that move is less straightforward than it looks.

130
src/authentik/ansible/kubernetes/helmchart.yaml.j2 Normal file
View file

@ -0,0 +1,130 @@
{#
Managed by Ansible (roles/k3s_app) — do not edit on the node.
A HelmChart CR for k3s's bundled helm-controller, the same mechanism every
cluster service here already uses (roles/k3s_metallb, k3s_monitoring,
k3s_postgres, k3s_cert_manager). The difference is only in who owns the
file: those roles each carry their own template, whereas this one belongs
to the app and roles/k3s_app renders whatever *.yaml.j2 it finds here.
This is the k3s analogue of src/<app>/ansible/unraid/docker-compose.yml.
It is not static the way the compose files are — Helm values have no
${VAR} interpolation to defer to a rendered .env, so config is substituted
at render time from `app_config` (common/vars.yml + this platform's
vars.yml). What survives from the Unraid convention is the part that
matters: no secret is ever interpolated into this file. Secrets are
referenced by secretKeyRef against the Secret roles/k3s_app renders from
Vault, so this manifest stays 0644 on the node and safe to read.
No Redis anywhere below, deliberately: recent Authentik no longer requires
it, and the chart carries no Redis dependency (its only conditional
subchart is Bitnami Postgres, which is disabled here in favour of the
cluster's shared CNPG instance).
-#}
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
name: authentik
namespace: kube-system
spec:
chart: authentik
repo: https://charts.goauthentik.io
# Chart version and app version are the same string upstream, so
# AUTHENTIK_VERSION pins both. See common/vars.yml — this must match the
# version the database was dumped from at cutover time.
version: "{{ app_config.AUTHENTIK_VERSION }}"
targetNamespace: {{ app_config.K8S_NAMESPACE }}
createNamespace: true
valuesContent: |-
global:
# Explicit `env` entries rather than an `envFrom: [secretRef]`, for a
# precedence reason rather than a stylistic one: the chart builds its
# own env for the values under `authentik:` below, and container `env`
# deterministically wins over anything arriving via `envFrom`, whereas
# two envFrom sources resolve by list order the chart controls. This
# way the non-secret half stays readable as ordinary Helm values and
# the secret half is unambiguously authoritative.
env:
# Signs session cookies. Carried over from the Unraid deployment, not
# generated — a new value logs every user out the moment DNS flips.
- name: AUTHENTIK_SECRET_KEY
valueFrom:
secretKeyRef:
name: authentik-secrets
key: AUTHENTIK_SECRET_KEY
- name: AUTHENTIK_POSTGRESQL__PASSWORD
valueFrom:
secretKeyRef:
name: authentik-secrets
key: AUTHENTIK_POSTGRESQL__PASSWORD
authentik:
log_level: info
# Off: this is a homelab, and the default ships crash reports to
# Sentry.
error_reporting:
enabled: false
postgresql:
host: {{ app_config.DB_HOST }}
port: {{ app_config.DB_PORT }}
name: {{ app_config.DB_NAME }}
user: {{ app_config.DB_USER }}
# password comes from global.env above, never from here.
# The chart's bundled Bitnami Postgres. Off — this cluster has a shared
# CNPG instance (roles/k3s_postgres) and an app getting its own database
# server would defeat the point of having one.
postgresql:
enabled: false
server:
replicas: 1
resources:
requests:
cpu: 100m
memory: {{ app_config.SERVER_MEMORY_REQUEST }}
limits:
memory: {{ app_config.SERVER_MEMORY_LIMIT }}
ingress:
enabled: true
ingressClassName: {{ app_config.INGRESS_CLASS }}
annotations:
# cert-manager watches Ingresses for this annotation and creates
# the Certificate (and hence the Secret named under tls: below) on
# its own — no Certificate resource of ours to keep in sync.
cert-manager.io/cluster-issuer: {{ app_config.CERT_ISSUER }}
hosts:
- {{ app_config.AUTHENTIK_HOST }}
tls:
- secretName: authentik-tls
hosts:
- {{ app_config.AUTHENTIK_HOST }}
# /media — uploaded icons and flow backgrounds, the only Authentik
# state that isn't in Postgres. The same PVC is mounted by the worker
# below; see vars.yml for why one ReadWriteOnce volume across two pods
# is fine here and what it costs.
volumes:
- name: media
persistentVolumeClaim:
claimName: authentik-media
volumeMounts:
- name: media
mountPath: /media
worker:
replicas: 1
resources:
requests:
cpu: 100m
memory: {{ app_config.WORKER_MEMORY_REQUEST }}
limits:
memory: {{ app_config.WORKER_MEMORY_LIMIT }}
volumes:
- name: media
persistentVolumeClaim:
claimName: authentik-media
volumeMounts:
- name: media
mountPath: /media

33
src/authentik/ansible/kubernetes/media-pvc.yaml.j2 Normal file
View file

@ -0,0 +1,33 @@
{#
Managed by Ansible (roles/k3s_app) — do not edit on the node.
Authentik's /media volume, claimed here rather than left to the chart: the
chart has no key that creates a PVC, only `volumes:`/`volumeMounts:` that
reference one, so something has to declare it.
Owning it separately turns out to be the safer arrangement anyway. A PVC
created by the chart would carry Helm's ownership metadata and be a
candidate for deletion whenever the HelmChart CR is uninstalled or the
chart's templates change shape across an upgrade. This one is a plain
manifest whose lifecycle is its own, so `state: absent` on the app — which
deletes the manifests and lets k3s garbage-collect them — is the only thing
that touches it, and roles/k3s_app/tasks/remove.yml documents PVCs as
deliberately surviving a default teardown.
WaitForFirstConsumer on the local-path StorageClass means this stays
Pending until the first Authentik pod is scheduled; that is the provisioner
working as intended, not a failure, and it's what pins the volume — and
therefore both pods — to a single worker Pi.
-#}
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: authentik-media
namespace: {{ app_config.K8S_NAMESPACE }}
spec:
accessModes:
- ReadWriteOnce
storageClassName: {{ app_config.MEDIA_STORAGE_CLASS }}
resources:
requests:
storage: {{ app_config.MEDIA_STORAGE_SIZE }}

63
src/authentik/ansible/kubernetes/vars.yml Normal file
View file

@ -0,0 +1,63 @@
---
# Kubernetes-specific overrides, merged over ../../common/vars.yml (these
# win). Consumed by roles/k3s_app — the same "portable config + platform
# overrides" split compose_stack and lxc_app apply, on the third platform.
env_defaults:
K8S_NAMESPACE: authentik
# --- Database ---------------------------------------------------------
#
# CNPG's read-write Service, by in-cluster DNS. This is the *primary*, and
# it follows a failover — CNPG re-points the selector when it promotes the
# replica, so the name stays correct without Authentik being restarted.
#
# Deliberately not the LoadBalancer address (192.168.50.81): that exists so
# the Ansible controller can provision databases from off-cluster, and
# sending in-cluster traffic out to the LAN and back would put MetalLB's
# L2 speaker in the path of every query for no benefit. See
# roles/k3s_app/tasks/deploy.yml, which uses the LB address for exactly the
# one thing that needs it.
DB_HOST: shared-postgres-rw.shared-postgres.svc.cluster.local
DB_PORT: "5432"
# --- Ingress ----------------------------------------------------------
#
# k3s's bundled Traefik, already running on 192.168.50.80 (MetalLB's first
# pool address). Nothing here pins that IP — the Ingress attaches to the
# class and Traefik's own Service owns the address.
INGRESS_CLASS: traefik
# The ClusterIssuer roles/k3s_cert_manager creates. Must match
# k3s_cert_manager_issuer_name in inventory/group_vars/k3s_cluster.yml.
CERT_ISSUER: letsencrypt
# --- Storage ----------------------------------------------------------
#
# /media holds uploaded application icons and flow backgrounds — the things
# that make the login page look like yours rather than stock. Small, but
# the one piece of Authentik's state that isn't in Postgres, so it has to
# be copied across at cutover (docs/authentik-migration.md) or the migration
# is visibly incomplete.
#
# ReadWriteOnce on k3s's local-path provisioner, and both the server and
# worker pods mount it. That works without any affinity rules of our own:
# local-path PVs carry node affinity, so once the volume binds to whichever
# worker the first pod lands on, the scheduler is obliged to place the
# second pod on that same node — and RWO permits multiple pods per node.
# The cost is that both pods are pinned to one Pi and neither can be
# rescheduled while it's down. Acceptable for a homelab SSO that is already
# a single logical instance; the fix, if it ever matters, is
# ReadWriteMany-capable storage, not a second PVC.
MEDIA_STORAGE_SIZE: 2Gi
MEDIA_STORAGE_CLASS: local-path
# --- Sizing -----------------------------------------------------------
#
# Pi 4, 3.8Gi usable, already running CNPG + Prometheus + MetalLB. The
# worker gets more headroom than the server because it's the half that runs
# migrations on startup and processes outposts/policies in the background;
# it is the one that gets OOM-killed if these are set naively equal.
SERVER_MEMORY_REQUEST: 384Mi
SERVER_MEMORY_LIMIT: 768Mi
WORKER_MEMORY_REQUEST: 512Mi
WORKER_MEMORY_LIMIT: 1Gi

15
src/authentik/ansible/proxmox/README.md Normal file
View file

@ -0,0 +1,15 @@
# Authentik on Proxmox — placeholder
Not deployed on Proxmox. Authentik runs on the k3s cluster
(`../kubernetes/`), having moved off a hand-made Unraid container — see
`docs/authentik-migration.md`.
To add it here later: `vars.yml` of Proxmox-specific overrides, `install.yml`
of install steps, and templates for the app's config and systemd unit, plus
`src/authentik/terraform/` defining its LXC. See `src/forgejo/ansible/proxmox/`
for a worked example, and CLAUDE.md → "Adding a new app" — in particular that
an app must only ever be live on one platform at a time.
This folder is kept empty-but-present on purpose: every app carries the same
`common/` + `ansible/` + `terraform/` shape whether or not each platform is
in use, so adding one later doesn't mean restructuring.

23
src/authentik/ansible/unraid/README.md Normal file
View file

@ -0,0 +1,23 @@
# Authentik on Unraid — deliberately not managed here
Authentik **does** currently run on Unraid, but it was created by hand
through the Unraid UI, not by this repo. There is no `docker-compose.yml`
here and there should not be one: adding it would put Authentik in `nas01`'s
`stacks:` list, and CLAUDE.md → "Adding a new app" is explicit that an app
must never be declared on two platforms at once — they would share a Vault
path and a database, and two live deployments would corrupt each other's
state.
The k3s deployment under `../kubernetes/` is where Authentik is going. See
`docs/authentik-migration.md` for the cutover: dump the database off the
Unraid Postgres, restore onto the cluster's CNPG instance, bring Authentik up
on k3s on the *same* version, verify, then move DNS.
Once DNS has moved and the k3s instance has been trusted for long enough,
stop and delete the hand-made Unraid container. That teardown is a UI action,
not a `state: absent` run — this repo never deployed it, so it has nothing to
tear down.
This folder stays as a placeholder rather than being deleted, per CLAUDE.md:
the shape is the point, and Authentik moving back to Unraid one day should
mean writing a compose file here, not restructuring the app.

38
src/authentik/common/vars.yml Normal file
View file

@ -0,0 +1,38 @@
---
# Portable config — values that hold regardless of where Authentik runs.
# Platform-specific values (namespace, storage class, ingress class, how the
# database is reached) live in ../ansible/<platform>/vars.yml and are merged
# over these at deploy time.
env_defaults:
# >>> CONFIRM BEFORE THE CUTOVER <<<
#
# This must match the version currently running on Unraid at the moment
# the database is dumped. Authentik runs its Django migrations on startup
# against whatever schema it finds, and those migrations are one-way: a
# newer Authentik pointed at an older dump will silently upgrade the
# schema, and there is no downgrade path if the cutover then has to be
# rolled back. Deploy on the *same* version, confirm it works, flip DNS,
# and only then bump this in a separate commit.
#
# The chart version and the app version are the same string upstream (see
# ../ansible/kubernetes/helmchart.yaml.j2, which uses this for both), so
# this is the only place a version is written down.
AUTHENTIK_VERSION: "2026.5.4"
AUTHENTIK_HOST: auth.turtlesystems.uk
AUTHENTIK_URL: "https://auth.turtlesystems.uk"
# Authentik's own HTTP port inside the container. Not the published port —
# on k3s the Ingress terminates and forwards here; there is no host port.
AUTHENTIK_HTTP_PORT: "9000"
DB_NAME: authentik
DB_USER: authentik
# AUTHENTIK_SECRET_KEY and AUTHENTIK_POSTGRESQL__PASSWORD are secrets —
# sourced from Vault (homelab/authentik), never set here.
#
# AUTHENTIK_SECRET_KEY specifically must be carried over from the existing
# Unraid deployment rather than generated fresh: it signs session cookies,
# so a new value logs every user out at the moment DNS flips, and the
# cutover stops looking like the no-op it's meant to be. See
# docs/authentik-migration.md.

11
src/authentik/terraform/README.md Normal file
View file

@ -0,0 +1,11 @@
# Authentik Terraform — placeholder
No Terraform for Authentik. It runs on the k3s cluster, and the k3s cluster
is 4 Raspberry Pis that already exist — there is no guest for Terraform to
provision, the same reason `homelab-utils` has no `src/<app>/terraform/`
entries of its own (CLAUDE.md → "K3s cluster").
Terraform would only come into this if Authentik moved to Proxmox, where it
would need a module defining its LXC (template, cores, memory, disk, IP),
called from `build/config/terraform/main.tf`. See `src/forgejo/terraform/`
for the shape.

94
src/forgejo/ansible/proxmox/README.md Normal file
View file

@ -0,0 +1,94 @@
# ansible/proxmox/ — forgejo
Native install into the Terraform-provisioned LXC (`../../terraform/`). Run by
the shared `lxc_app` role, which merges `../../common/vars.yml` with `vars.yml`
here, fetches `homelab/forgejo` from Vault, provisions the database, then
includes `install.yml`.
| File | What it is |
|---|---|
| `vars.yml` | Proxmox-specific overrides — paths, service user, exact release, LAN address of the shared Postgres |
| `install.yml` | Packages, `git` user, directory layout, binary download, config/unit rendering, dump schedule |
| `templates/app.ini.j2` | Forgejo's config — the equivalent of the `FORGEJO__section__KEY` env vars in `../unraid/docker-compose.yml` |
| `templates/forgejo.service.j2` | systemd unit, replacing Compose's `restart: unless-stopped` |
| `templates/forgejo-dbdump.*` | Nightly `pg_dump`, script + service + timer |
| `templates/pgpass.j2` | Credentials for that dump, mode 0600 |
## How this is backed up
Nothing is bind-mounted in from the NAS. Every path Forgejo writes to —
repositories, LFS, attachments, indexers, queues, sessions — is left at its
default under `APP_DATA_PATH` on the container's own disk, so a vzdump of the
guest is a complete copy of its filesystem state. Proxmox excludes bind mounts
from vzdump, so an NFS-mounted repository directory would have been the one
thing missing from the backup that was made to protect it.
The off-box copy is the vzdump job in
`build/config/ansible/roles/pve_backup`, writing to an NFS storage on the NAS.
**The database is the other half.** Postgres runs elsewhere — the shared
Postgres LXC at 192.168.50.54, a different guest and therefore a different
archive — so a restored container would come back with every repository and no
issues, pull requests, users or permissions. `forgejo-dbdump.timer` runs
`pg_dump` into `FORGEJO_BACKUP_PATH` on the container's own disk, which puts
the dump inside the same vzdump archive as the repositories it belongs to. One
artifact, one restore.
That only holds if the dump finishes before the backup window, so the two
schedules are a pair:
| | Set in | Default |
|---|---|---|
| Database dump | `vars.yml` → `FORGEJO_DB_DUMP_ONCALENDAR` | 01:30 |
| vzdump job | `group_vars/proxmox_nodes.yml` → `pve_backup_schedule` | 02:00 |
Move one and move the other. Retention is independent: `pg_dump` output is
pruned inside the container by `FORGEJO_BACKUP_KEEP_DAYS`, while the archives
themselves are pruned by the storage's `prune-backups` settings — Proxmox
prunes archives, not their contents.
### Restoring
1. Restore the container from the archive (`pct restore`, or the UI).
2. Recreate the database and role if they're gone —
`ansible-playbook playbooks/proxmox.yml -e only_stacks=forgejo` does it
idempotently.
3. Load the newest dump from `/var/lib/forgejo/backups`:
```sh
systemctl stop forgejo
zcat /var/lib/forgejo/backups/forgejo-<stamp>.sql.gz \
| psql -h <postgres-host> -U forgejo -d forgejo
systemctl start forgejo
```
## Upgrading Forgejo
`FORGEJO_VERSION` in `../../common/vars.yml` is `10` — a Docker tag, a rolling
pointer at the latest 10.x, which is all the Compose stack needs. A native
install downloads one specific artifact, so `vars.yml` here carries
`FORGEJO_RELEASE` with the exact version. Bump it and redeploy: the new binary
lands alongside the running one, the `/usr/local/bin/forgejo` symlink flips,
and the service restarts. The old binary stays on disk, so rolling back is
editing `FORGEJO_RELEASE` and redeploying.
## Differences from the Unraid stack
- **SSH port 22, not 2222.** On Unraid, 2222 was a published Docker port
dodging the host's own sshd. Here the container has its own IP and its own
sshd, which Forgejo uses directly (`START_SSH_SERVER = false`) — it manages
the `git` user's `authorized_keys` rather than running its own server. This
changes the SSH clone URLs Forgejo advertises, so existing remotes need
updating after a migration.
- **`INSTALL_LOCK = true`.** `app.ini` is rendered in full from Vault, so the
web installer is skipped entirely. Nothing is left for Forgejo to generate
on first boot — which is why `LFS_JWT_SECRET` has to exist in Vault here
even though the Compose stack lets Forgejo invent one.
- **`DB_HOST` is a LAN address**, not the `shared-postgres` Docker network
alias — there's no `unraid_shared` network to resolve it through.
## One platform at a time
Both platforms use the same Vault path and the same database, so they must not
run at once. This was why `host_vars/forgejo.yml` carried an empty `apps:`
list for so long; nas2 has since been retired, the list is live, and the
bring-up runbook is `docs/forgejo-proxmox.md`.

190
src/forgejo/ansible/proxmox/install.yml Normal file
View file

@ -0,0 +1,190 @@
---
# App-specific install steps, included by the `lxc_app` role after it has
# merged ../../common/vars.yml with ./vars.yml into `app_config` and fetched
# `vault_secrets` from the app's Vault path. The role handles enabling and
# starting the service once this file has put the unit in place.
#
# Template paths are absolute rather than relative: this file is included from
# a role, so a bare `src:` would be looked up against that role's templates/
# directory, not this app's.
# A native install renders app.ini in full and sets INSTALL_LOCK, so nothing
# is left for Forgejo to generate on first boot. A missing key would otherwise
# surface as an undefined-variable error three tasks later.
- name: Check that every required Vault key is present
ansible.builtin.assert:
that:
- vault_secrets.DB_PASSWORD is defined
- vault_secrets.SECRET_KEY is defined
- vault_secrets.INTERNAL_TOKEN is defined
- vault_secrets.JWT_SECRET is defined
- vault_secrets.LFS_JWT_SECRET is defined
fail_msg: >-
Vault path {{ app.vault_path }} is missing one or more of DB_PASSWORD,
SECRET_KEY, INTERNAL_TOKEN, JWT_SECRET, LFS_JWT_SECRET. See
docs/vault-secrets.md — LFS_JWT_SECRET is needed by the native install
even though the Unraid Compose stack lets Forgejo generate it.
quiet: true
- name: Install packages Forgejo needs
ansible.builtin.apt:
name:
- git
- gzip
- ca-certificates
# pg_dump for the nightly database dump at the end of this file. The
# distro client and the shared Postgres LXC (17, from the PGDG archive)
# are close enough that either direction works, but the supported one is
# a client at least as new as the server — check this if the dump ever
# starts failing after a Postgres major upgrade over there.
- postgresql-client
state: present
update_cache: true
cache_valid_time: 3600
- name: Create the Forgejo service group
ansible.builtin.group:
name: "{{ app_config.FORGEJO_USER }}"
system: true
state: present
- name: Create the Forgejo service user
ansible.builtin.user:
name: "{{ app_config.FORGEJO_USER }}"
group: "{{ app_config.FORGEJO_USER }}"
system: true
# A real shell, not nologin: this account is also the SSH login clients
# use for git@host:owner/repo.git.
shell: /bin/bash
home: "{{ app_config.FORGEJO_HOME }}"
create_home: true
comment: Forgejo
- name: Create Forgejo directories
ansible.builtin.file:
path: "{{ item.path }}"
state: directory
owner: "{{ item.owner | default(app_config.FORGEJO_USER) }}"
group: "{{ item.group | default(app_config.FORGEJO_USER) }}"
mode: "{{ item.mode }}"
loop:
- { path: "{{ app_config.FORGEJO_HOME }}", mode: "0750" }
- { path: "{{ app_config.FORGEJO_DATA_PATH }}", mode: "0750" }
- { path: "{{ app_config.FORGEJO_LOG_PATH }}", mode: "0750" }
# Forgejo writes the git user's authorized_keys here itself.
- { path: "{{ app_config.FORGEJO_HOME }}/.ssh", mode: "0700" }
# Nightly database dumps. On the container's own disk on purpose — that is
# what gets them into the vzdump archive.
- { path: "{{ app_config.FORGEJO_BACKUP_PATH }}", mode: "0700" }
# Config is root-owned and group-readable by Forgejo: it holds the
# database password and the three signing secrets, and Forgejo has no
# business rewriting it — everything in it is rendered from Vault below.
- { path: "{{ app_config.FORGEJO_CONFIG_DIR }}", owner: root, mode: "0750" }
- { path: "{{ app_config.FORGEJO_INSTALL_DIR }}", owner: root, group: root, mode: "0755" }
loop_control:
label: "{{ item.path }}"
- name: Set Forgejo release facts
ansible.builtin.set_fact:
forgejo_artifact: >-
forgejo-{{ app_config.FORGEJO_RELEASE }}-linux-{{ app_config.FORGEJO_ARCH }}
forgejo_release_url: >-
https://codeberg.org/forgejo/forgejo/releases/download/v{{ app_config.FORGEJO_RELEASE }}
- name: Download the Forgejo binary
ansible.builtin.get_url:
url: "{{ forgejo_release_url }}/{{ forgejo_artifact }}"
dest: "{{ app_config.FORGEJO_INSTALL_DIR }}/{{ forgejo_artifact }}"
# Checksum from the release's own .sha256 file. Ansible fetches it and
# matches the line for this artifact — which is why `dest` keeps the
# upstream filename rather than being renamed to plain `forgejo`.
checksum: "sha256:{{ forgejo_release_url }}/{{ forgejo_artifact }}.sha256"
owner: root
group: root
mode: "0755"
register: forgejo_binary
# Versioned file plus a symlink, rather than overwriting one path: an upgrade
# lands the new binary alongside the running one and flips the link, so a
# failed download can't leave a half-written executable in place.
- name: Link the current Forgejo binary onto PATH
ansible.builtin.file:
src: "{{ app_config.FORGEJO_INSTALL_DIR }}/{{ forgejo_artifact }}"
dest: /usr/local/bin/forgejo
state: link
register: forgejo_link
- name: Render Forgejo configuration
ansible.builtin.template:
src: "{{ app_local_dir }}/ansible/proxmox/templates/app.ini.j2"
dest: "{{ app_config.FORGEJO_CONFIG_PATH }}"
owner: root
group: "{{ app_config.FORGEJO_USER }}"
mode: "0640"
register: forgejo_config
no_log: true
- name: Install the Forgejo systemd unit
ansible.builtin.template:
src: "{{ app_local_dir }}/ansible/proxmox/templates/forgejo.service.j2"
dest: /etc/systemd/system/forgejo.service
owner: root
group: root
mode: "0644"
register: forgejo_unit
# `lxc_app` starts the service after this file, but starting an already-running
# service is a no-op — it would not pick up a changed app.ini or a new binary.
- name: Restart Forgejo if its binary or configuration changed
ansible.builtin.systemd_service:
name: forgejo
state: restarted
daemon_reload: true
when: forgejo_binary.changed or forgejo_link.changed
or forgejo_config.changed or forgejo_unit.changed
# --- Database dump -------------------------------------------------------
#
# The vzdump job on the Proxmox node captures this container's filesystem, but
# Forgejo's database is not in it — Postgres runs elsewhere, in the shared
# Postgres LXC at 192.168.50.54, which is a *different guest* and so a
# different archive. Restoring the container alone would give back every repository with
# no issues, pull requests, users or permissions. So dump the database into the
# container's own filesystem on a schedule that finishes before the vzdump
# window, and the one archive holds both halves.
- name: Render the Postgres password file for the dump job
ansible.builtin.template:
src: "{{ app_local_dir }}/ansible/proxmox/templates/pgpass.j2"
dest: "{{ app_config.FORGEJO_HOME }}/.pgpass"
owner: "{{ app_config.FORGEJO_USER }}"
group: "{{ app_config.FORGEJO_USER }}"
mode: "0600"
no_log: true
- name: Install the database dump script
ansible.builtin.template:
src: "{{ app_local_dir }}/ansible/proxmox/templates/forgejo-dbdump.sh.j2"
dest: /usr/local/bin/forgejo-dbdump
owner: root
group: root
mode: "0755"
- name: Install the database dump systemd units
ansible.builtin.template:
src: "{{ app_local_dir }}/ansible/proxmox/templates/{{ item }}.j2"
dest: "/etc/systemd/system/{{ item }}"
owner: root
group: root
mode: "0644"
loop:
- forgejo-dbdump.service
- forgejo-dbdump.timer
register: forgejo_dbdump_units
- name: Enable the nightly database dump timer
ansible.builtin.systemd_service:
name: forgejo-dbdump.timer
enabled: true
state: started
daemon_reload: "{{ forgejo_dbdump_units.changed }}"

116
src/forgejo/ansible/proxmox/templates/app.ini.j2 Normal file
View file

@ -0,0 +1,116 @@
{#
Rendered by Ansible from src/forgejo/common/vars.yml merged with
../vars.yml (as `app_config`), plus Vault path homelab/forgejo (as
`vault_secrets`). The native-install equivalent of the
FORGEJO__section__KEY environment variables in ../../unraid/docker-compose.yml.
Do not edit this file on the host — the next deploy overwrites it. It is
root-owned and group-readable by Forgejo precisely so Forgejo cannot rewrite
it behind Ansible's back.
Note what is NOT configured here: every path Forgejo stores state under is
left at its default beneath APP_DATA_PATH — indexers, queues, sessions,
avatars, packages. That is deliberate. All of it sitting on the container's
own disk is what makes a vzdump of this guest a complete copy of Forgejo's
filesystem state.
#}
APP_NAME = Forgejo
RUN_USER = {{ app_config.FORGEJO_USER }}
RUN_MODE = prod
WORK_PATH = {{ app_config.FORGEJO_WORK_PATH }}
[server]
PROTOCOL = http
DOMAIN = {{ app_config.FORGEJO_DOMAIN }}
ROOT_URL = {{ app_config.FORGEJO_ROOT_URL }}
HTTP_ADDR = 0.0.0.0
HTTP_PORT = {{ app_config.FORGEJO_HTTP_PORT }}
APP_DATA_PATH = {{ app_config.FORGEJO_DATA_PATH }}
{# The container's own sshd serves git over SSH, so Forgejo only advertises
the address and manages the git user's authorized_keys. On Unraid this was
Forgejo's built-in server behind a published Docker port. #}
SSH_DOMAIN = {{ app_config.FORGEJO_DOMAIN }}
SSH_PORT = {{ app_config.FORGEJO_SSH_PORT }}
START_SSH_SERVER = false
DISABLE_SSH = false
LFS_START_SERVER = true
LFS_JWT_SECRET = {{ vault_secrets.LFS_JWT_SECRET }}
[database]
DB_TYPE = postgres
HOST = {{ app_config.DB_HOST }}:{{ app_config.DB_PORT }}
NAME = {{ app_config.DB_NAME }}
USER = {{ app_config.DB_USER }}
PASSWD = {{ vault_secrets.DB_PASSWORD }}
{# The shared Postgres publishes a plain port on the LAN and has no TLS
configured — see src/shared/postgres/ansible/unraid/docker-compose.yml.
Change both together if that ever gains a certificate. #}
SSL_MODE = disable
[repository]
ROOT = {{ app_config.FORGEJO_DATA_PATH }}/forgejo-repositories
[security]
{# Fully configured from Vault, so skip the web installer entirely — without
this, a fresh container serves the setup wizard on first boot and would
write its own app.ini over this one. #}
INSTALL_LOCK = true
SECRET_KEY = {{ vault_secrets.SECRET_KEY }}
INTERNAL_TOKEN = {{ vault_secrets.INTERNAL_TOKEN }}
[oauth2]
JWT_SECRET = {{ vault_secrets.JWT_SECRET }}
[oauth2_client]
{# Policy for signing in through an external OIDC provider (Authentik). The
provider itself is NOT configured here and cannot be: Forgejo keeps
authentication sources in its database, added once with
`forgejo admin auth add-oauth` — see step 8 of docs/forgejo-proxmox.md.
This section only decides what happens to a user who has just
authenticated there. #}
{# Create a Forgejo account on first successful OIDC login. Without this,
Authentik authenticates the user and Forgejo then has nobody to log in as.
Gated by the registration settings below, not independent of them. #}
ENABLE_AUTO_REGISTRATION = true
{# Attach an OIDC identity to an existing local account when the email
matches, rather than creating a second one alongside it. This is what lets
the break-glass admin from step 4b become the same account you arrive as
through Authentik, instead of demoting you to a fresh unprivileged user on
first SSO login. `login` would ask the user to confirm by entering their
local password; `auto` links silently, which is only safe because
Authentik is the sole source of verified addresses here. #}
ACCOUNT_LINKING = auto
{# Take the Forgejo username from the provider's preferred_username claim.
The alternative, `userid`, would name accounts after Authentik's opaque
subject UUID. #}
USERNAME = nickname
[service]
{# INSTALL_LOCK skips the wizard, which is also where these would have been
chosen.
These two are a pair and the combination is deliberate. DISABLE_REGISTRATION
cannot stay `true` once SSO is wanted: it blocks OIDC auto-registration as
well as the local signup form, so Authentik logins authenticate correctly
and are then refused an account — which reads like a broken provider rather
than a policy setting. ALLOW_ONLY_EXTERNAL_REGISTRATION restores exactly
the property that was wanted: no self-service signup, accounts only through
a configured provider.
Safe to have in place before the Authentik source exists. With no external
provider configured, "only external registration" permits nothing, so this
is not a window during which the forge is open — flipping the first value
alone would have been. #}
DISABLE_REGISTRATION = false
ALLOW_ONLY_EXTERNAL_REGISTRATION = true
REQUIRE_SIGNIN_VIEW = false
[log]
{# journald, via systemd capturing stdout — `journalctl -u forgejo`. ROOT_PATH
still matters: some subsystems write their own files regardless. #}
MODE = console
LEVEL = info
ROOT_PATH = {{ app_config.FORGEJO_LOG_PATH }}

16
src/forgejo/ansible/proxmox/templates/forgejo-dbdump.service.j2 Normal file
View file

@ -0,0 +1,16 @@
{# Rendered by Ansible — see ../install.yml. #}
[Unit]
Description=Dump the Forgejo database into this container's filesystem
Documentation=file://{{ app_config.FORGEJO_CONFIG_PATH }}
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
User={{ app_config.FORGEJO_USER }}
Group={{ app_config.FORGEJO_USER }}
ExecStart=/usr/local/bin/forgejo-dbdump
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=full
ReadWritePaths={{ app_config.FORGEJO_BACKUP_PATH }}

50
src/forgejo/ansible/proxmox/templates/forgejo-dbdump.sh.j2 Normal file
View file

@ -0,0 +1,50 @@
#!/bin/bash
{# Rendered by Ansible — see ../install.yml. Do not edit on the host. #}
#
# Dumps the Forgejo database into this container's filesystem, where the
# Proxmox vzdump job will pick it up along with everything else. Postgres runs
# elsewhere (shared, on Unraid), so without this a restored container would
# have every repository and no issues, pull requests, users or permissions.
#
# Run by forgejo-dbdump.timer, not by hand — though running it by hand is
# harmless and is the quickest way to check the credentials work.
# pipefail matters more than usual here: pg_dump feeds gzip, and without it a
# failed dump still exits 0 through gzip and gets published as a valid-looking
# but truncated archive.
set -euo pipefail
readonly BACKUP_DIR="{{ app_config.FORGEJO_BACKUP_PATH }}"
readonly KEEP_DAYS="{{ app_config.FORGEJO_BACKUP_KEEP_DAYS }}"
export PGPASSFILE="{{ app_config.FORGEJO_HOME }}/.pgpass"
umask 077
dest="${BACKUP_DIR}/{{ app_config.DB_NAME }}-$(date -u +%Y%m%dT%H%M%SZ).sql.gz"
# Write to .part and rename only on success, so a dump interrupted by the
# vzdump window (or a reboot) never leaves a partial file that looks complete
# to the retention sweep below.
pg_dump \
--host="{{ app_config.DB_HOST }}" \
--port="{{ app_config.DB_PORT }}" \
--username="{{ app_config.DB_USER }}" \
--dbname="{{ app_config.DB_NAME }}" \
--format=plain \
--no-owner \
--no-privileges \
| gzip -9 > "${dest}.part"
mv "${dest}.part" "${dest}"
# Retention is here rather than in the storage's prune settings because these
# live inside the container: Proxmox prunes backup archives, not their
# contents. Only complete dumps are counted, so a stale .part never displaces
# a good one.
find "${BACKUP_DIR}" -maxdepth 1 -type f -name '*.sql.gz' \
-mtime "+${KEEP_DAYS}" -delete
find "${BACKUP_DIR}" -maxdepth 1 -type f -name '*.sql.gz.part' \
-mtime +1 -delete
echo "wrote ${dest}"

16
src/forgejo/ansible/proxmox/templates/forgejo-dbdump.timer.j2 Normal file
View file

@ -0,0 +1,16 @@
{# Rendered by Ansible — see ../install.yml. #}
[Unit]
Description=Nightly Forgejo database dump
[Timer]
{# Must land before the vzdump window on the Proxmox node
(build/config/ansible/inventory/host_vars/pve.yml). The archive is only a
complete restore point if the dump inside it is from the same night. #}
OnCalendar={{ app_config.FORGEJO_DB_DUMP_ONCALENDAR }}
{# Catch up after a reboot rather than skipping a night silently. #}
Persistent=true
{# No randomised delay: the gap to the backup window is the whole point. #}
AccuracySec=1min
[Install]
WantedBy=timers.target

33
src/forgejo/ansible/proxmox/templates/forgejo.service.j2 Normal file
View file

@ -0,0 +1,33 @@
{# Rendered by Ansible — see ../install.yml. The native-install replacement for
Compose's `restart: unless-stopped`. #}
[Unit]
Description=Forgejo
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
User={{ app_config.FORGEJO_USER }}
Group={{ app_config.FORGEJO_USER }}
WorkingDirectory={{ app_config.FORGEJO_WORK_PATH }}
ExecStart=/usr/local/bin/forgejo web --config {{ app_config.FORGEJO_CONFIG_PATH }}
Restart=always
RestartSec=2s
Environment=USER={{ app_config.FORGEJO_USER }}
Environment=HOME={{ app_config.FORGEJO_HOME }}
Environment=GITEA_WORK_DIR={{ app_config.FORGEJO_WORK_PATH }}
# HTTP_PORT is 3000, so no CAP_NET_BIND_SERVICE is needed. Forgejo does need
# to write the git user's authorized_keys and its own data, so the filesystem
# is not made read-only wholesale.
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=full
ProtectHome=false
ReadWritePaths={{ app_config.FORGEJO_HOME }}
{# Restarting mid-push should not orphan git subprocesses. #}
KillMode=control-group
TimeoutStopSec=30s
[Install]
WantedBy=multi-user.target

Some files were not shown because too many files have changed in this diff Show more