From 1250c9cef61f662aad27636ffc831b9c3ea77cfc Mon Sep 17 00:00:00 2001 From: Russell Seymour Date: Mon, 24 Aug 2026 20:52:36 +0100 Subject: [PATCH] Initial checkin of code for managing homelab --- .forgejo/workflows/deploy.yml | 116 +++ .gitignore | 33 + CLAUDE.md | 597 +++++++++++ README.md | 966 ++++++++++++++++++ build/config/ansible/ansible.cfg | 18 + .../ansible/inventory/group_vars/all.yml | 18 + .../inventory/group_vars/k3s_cluster.yml | 205 ++++ .../group_vars/k3s_control_plane.yml | 9 + .../inventory/group_vars/k3s_workers.yml | 3 + .../inventory/group_vars/proxmox_guests.yml | 19 + .../inventory/group_vars/proxmox_nodes.yml | 46 + .../ansible/inventory/host_vars/forgejo.yml | 52 + .../ansible/inventory/host_vars/nas01.yml | 19 + .../ansible/inventory/host_vars/postgres.yml | 28 + build/config/ansible/inventory/hosts.yml | 58 ++ build/config/ansible/inventory/proxmox.yml | 56 + build/config/ansible/playbooks/deploy.yml | 21 + build/config/ansible/playbooks/k3s.yml | 241 +++++ .../ansible/playbooks/k3s_maintenance.yml | 37 + .../config/ansible/playbooks/k3s_ssh_key.yml | 82 ++ build/config/ansible/playbooks/proxmox.yml | 29 + build/config/ansible/playbooks/pve_host.yml | 14 + build/config/ansible/playbooks/unraid.yml | 24 + build/config/ansible/requirements.yml | 15 + .../roles/compose_stack/defaults/main.yml | 9 + .../roles/compose_stack/tasks/deploy.yml | 183 ++++ .../roles/compose_stack/tasks/main.yml | 75 ++ .../roles/compose_stack/tasks/remove.yml | 106 ++ .../roles/compose_stack/templates/env.j2 | 7 + .../ansible/roles/k3s_app/defaults/main.yml | 34 + .../ansible/roles/k3s_app/tasks/deploy.yml | 170 +++ .../ansible/roles/k3s_app/tasks/main.yml | 37 + .../ansible/roles/k3s_app/tasks/remove.yml | 60 ++ .../k3s_app/templates/app-secrets.yaml.j2 | 35 + .../roles/k3s_cert_manager/defaults/main.yml | 73 ++ .../roles/k3s_cert_manager/tasks/main.yml | 75 ++ .../templates/cert-manager.helmchart.yaml.j2 | 59 ++ .../templates/cluster-issuer.yaml.j2 | 30 + .../templates/dns-credentials.yaml.j2 | 23 + .../roles/k3s_maintenance/tasks/main.yml | 100 ++ .../roles/k3s_metallb/defaults/main.yml | 31 + .../ansible/roles/k3s_metallb/tasks/main.yml | 39 + .../templates/metallb-config.yaml.j2 | 26 + .../templates/metallb.helmchart.yaml.j2 | 18 + .../roles/k3s_monitoring/defaults/main.yml | 86 ++ .../roles/k3s_monitoring/tasks/main.yml | 16 + .../kube-prometheus-stack.helmchart.yaml.j2 | 24 + .../ansible/roles/k3s_node/defaults/main.yml | 18 + .../ansible/roles/k3s_node/tasks/agent.yml | 100 ++ .../ansible/roles/k3s_node/tasks/main.yml | 24 + .../ansible/roles/k3s_node/tasks/prep.yml | 80 ++ .../ansible/roles/k3s_node/tasks/server.yml | 103 ++ .../roles/k3s_postgres/defaults/main.yml | 71 ++ .../ansible/roles/k3s_postgres/tasks/main.yml | 91 ++ .../templates/cnpg-operator.helmchart.yaml.j2 | 21 + .../templates/postgres-cluster.yaml.j2 | 104 ++ .../postgres-superuser-secret.yaml.j2 | 49 + .../roles/k3s_traefik/defaults/main.yml | 115 +++ .../ansible/roles/k3s_traefik/tasks/main.yml | 59 ++ .../templates/dashboard-ingressroute.yaml.j2 | 83 ++ .../templates/traefik.helmchartconfig.yaml.j2 | 34 + .../ansible/roles/lxc_app/defaults/main.yml | 2 + .../ansible/roles/lxc_app/tasks/main.yml | 124 +++ .../roles/pve_backup/defaults/main.yml | 61 ++ .../ansible/roles/pve_backup/tasks/main.yml | 161 +++ .../unattended_upgrades/defaults/main.yml | 19 + .../unattended_upgrades/handlers/main.yml | 5 + .../roles/unattended_upgrades/tasks/main.yml | 43 + .../templates/50unattended-upgrades.j2 | 17 + build/config/terraform/.terraform.lock.hcl | 23 + build/config/terraform/README.md | 66 ++ build/config/terraform/main.tf | 102 ++ build/config/terraform/providers.tf | 42 + build/config/terraform/variables.tf | 78 ++ docs/authentik-migration.md | 420 ++++++++ docs/forgejo-proxmox.md | 287 ++++++ docs/postgres-proxmox.md | 237 +++++ docs/vault-secrets.md | 551 ++++++++++ src/arr/ansible/proxmox/README.md | 22 + src/arr/ansible/unraid/.env.example | 101 ++ src/arr/ansible/unraid/docker-compose.yml | 252 +++++ src/arr/ansible/unraid/icon-prowlarr.png | Bin 0 -> 25627 bytes src/arr/ansible/unraid/icon-shelfarr.png | Bin 0 -> 128877 bytes src/arr/ansible/unraid/vars.yml | 59 ++ src/arr/common/vars.yml | 64 ++ src/arr/terraform/README.md | 11 + .../ansible/kubernetes/helmchart.yaml.j2 | 130 +++ .../ansible/kubernetes/media-pvc.yaml.j2 | 33 + src/authentik/ansible/kubernetes/vars.yml | 63 ++ src/authentik/ansible/proxmox/README.md | 15 + src/authentik/ansible/unraid/README.md | 23 + src/authentik/common/vars.yml | 38 + src/authentik/terraform/README.md | 11 + src/forgejo/ansible/proxmox/README.md | 94 ++ src/forgejo/ansible/proxmox/install.yml | 190 ++++ .../ansible/proxmox/templates/app.ini.j2 | 116 +++ .../templates/forgejo-dbdump.service.j2 | 16 + .../proxmox/templates/forgejo-dbdump.sh.j2 | 50 + .../proxmox/templates/forgejo-dbdump.timer.j2 | 16 + .../proxmox/templates/forgejo.service.j2 | 33 + .../ansible/proxmox/templates/pgpass.j2 | 8 + src/forgejo/ansible/proxmox/vars.yml | 50 + src/forgejo/ansible/unraid/.env.example | 28 + src/forgejo/ansible/unraid/docker-compose.yml | 48 + src/forgejo/ansible/unraid/vars.yml | 21 + src/forgejo/common/vars.yml | 15 + src/forgejo/terraform/README.md | 52 + src/forgejo/terraform/main.tf | 116 +++ src/forgejo/terraform/outputs.tf | 25 + src/forgejo/terraform/providers.tf | 13 + src/forgejo/terraform/variables.tf | 149 +++ src/hashicorp-vault/config.hcl | 18 + .../policies/unraid-deploy.hcl | 3 + .../policies/vault-readonly.hcl | 6 + src/hashicorp-vault/policies/vault-write.hcl | 3 + src/scripts/configure_k3s_node.zsh | 15 + .../postgres/ansible/kubernetes/vars.yml | 20 + src/shared/postgres/ansible/proxmox/README.md | 76 ++ .../postgres/ansible/proxmox/install.yml | 157 +++ .../ansible/proxmox/templates/homelab.conf.j2 | 50 + .../ansible/proxmox/templates/pg_hba.conf.j2 | 32 + src/shared/postgres/ansible/proxmox/vars.yml | 75 ++ .../postgres/ansible/unraid/.env.example | 18 + .../ansible/unraid/docker-compose.yml | 32 + src/shared/postgres/ansible/unraid/vars.yml | 16 + src/shared/postgres/common/vars.yml | 8 + src/shared/postgres/terraform/README.md | 103 ++ src/shared/postgres/terraform/main.tf | 147 +++ src/shared/postgres/terraform/outputs.tf | 34 + src/shared/postgres/terraform/providers.tf | 13 + src/shared/postgres/terraform/variables.tf | 207 ++++ 131 files changed, 10107 insertions(+) create mode 100644 .forgejo/workflows/deploy.yml create mode 100644 .gitignore create mode 100644 CLAUDE.md create mode 100644 README.md create mode 100644 build/config/ansible/ansible.cfg create mode 100644 build/config/ansible/inventory/group_vars/all.yml create mode 100644 build/config/ansible/inventory/group_vars/k3s_cluster.yml create mode 100644 build/config/ansible/inventory/group_vars/k3s_control_plane.yml create mode 100644 build/config/ansible/inventory/group_vars/k3s_workers.yml create mode 100644 build/config/ansible/inventory/group_vars/proxmox_guests.yml create mode 100644 build/config/ansible/inventory/group_vars/proxmox_nodes.yml create mode 100644 build/config/ansible/inventory/host_vars/forgejo.yml create mode 100644 build/config/ansible/inventory/host_vars/nas01.yml create mode 100644 build/config/ansible/inventory/host_vars/postgres.yml create mode 100644 build/config/ansible/inventory/hosts.yml create mode 100644 build/config/ansible/inventory/proxmox.yml create mode 100644 build/config/ansible/playbooks/deploy.yml create mode 100644 build/config/ansible/playbooks/k3s.yml create mode 100644 build/config/ansible/playbooks/k3s_maintenance.yml create mode 100644 build/config/ansible/playbooks/k3s_ssh_key.yml create mode 100644 build/config/ansible/playbooks/proxmox.yml create mode 100644 build/config/ansible/playbooks/pve_host.yml create mode 100644 build/config/ansible/playbooks/unraid.yml create mode 100644 build/config/ansible/requirements.yml create mode 100644 build/config/ansible/roles/compose_stack/defaults/main.yml create mode 100644 build/config/ansible/roles/compose_stack/tasks/deploy.yml create mode 100644 build/config/ansible/roles/compose_stack/tasks/main.yml create mode 100644 build/config/ansible/roles/compose_stack/tasks/remove.yml create mode 100644 build/config/ansible/roles/compose_stack/templates/env.j2 create mode 100644 build/config/ansible/roles/k3s_app/defaults/main.yml create mode 100644 build/config/ansible/roles/k3s_app/tasks/deploy.yml create mode 100644 build/config/ansible/roles/k3s_app/tasks/main.yml create mode 100644 build/config/ansible/roles/k3s_app/tasks/remove.yml create mode 100644 build/config/ansible/roles/k3s_app/templates/app-secrets.yaml.j2 create mode 100644 build/config/ansible/roles/k3s_cert_manager/defaults/main.yml create mode 100644 build/config/ansible/roles/k3s_cert_manager/tasks/main.yml create mode 100644 build/config/ansible/roles/k3s_cert_manager/templates/cert-manager.helmchart.yaml.j2 create mode 100644 build/config/ansible/roles/k3s_cert_manager/templates/cluster-issuer.yaml.j2 create mode 100644 build/config/ansible/roles/k3s_cert_manager/templates/dns-credentials.yaml.j2 create mode 100644 build/config/ansible/roles/k3s_maintenance/tasks/main.yml create mode 100644 build/config/ansible/roles/k3s_metallb/defaults/main.yml create mode 100644 build/config/ansible/roles/k3s_metallb/tasks/main.yml create mode 100644 build/config/ansible/roles/k3s_metallb/templates/metallb-config.yaml.j2 create mode 100644 build/config/ansible/roles/k3s_metallb/templates/metallb.helmchart.yaml.j2 create mode 100644 build/config/ansible/roles/k3s_monitoring/defaults/main.yml create mode 100644 build/config/ansible/roles/k3s_monitoring/tasks/main.yml create mode 100644 build/config/ansible/roles/k3s_monitoring/templates/kube-prometheus-stack.helmchart.yaml.j2 create mode 100644 build/config/ansible/roles/k3s_node/defaults/main.yml create mode 100644 build/config/ansible/roles/k3s_node/tasks/agent.yml create mode 100644 build/config/ansible/roles/k3s_node/tasks/main.yml create mode 100644 build/config/ansible/roles/k3s_node/tasks/prep.yml create mode 100644 build/config/ansible/roles/k3s_node/tasks/server.yml create mode 100644 build/config/ansible/roles/k3s_postgres/defaults/main.yml create mode 100644 build/config/ansible/roles/k3s_postgres/tasks/main.yml create mode 100644 build/config/ansible/roles/k3s_postgres/templates/cnpg-operator.helmchart.yaml.j2 create mode 100644 build/config/ansible/roles/k3s_postgres/templates/postgres-cluster.yaml.j2 create mode 100644 build/config/ansible/roles/k3s_postgres/templates/postgres-superuser-secret.yaml.j2 create mode 100644 build/config/ansible/roles/k3s_traefik/defaults/main.yml create mode 100644 build/config/ansible/roles/k3s_traefik/tasks/main.yml create mode 100644 build/config/ansible/roles/k3s_traefik/templates/dashboard-ingressroute.yaml.j2 create mode 100644 build/config/ansible/roles/k3s_traefik/templates/traefik.helmchartconfig.yaml.j2 create mode 100644 build/config/ansible/roles/lxc_app/defaults/main.yml create mode 100644 build/config/ansible/roles/lxc_app/tasks/main.yml create mode 100644 build/config/ansible/roles/pve_backup/defaults/main.yml create mode 100644 build/config/ansible/roles/pve_backup/tasks/main.yml create mode 100644 build/config/ansible/roles/unattended_upgrades/defaults/main.yml create mode 100644 build/config/ansible/roles/unattended_upgrades/handlers/main.yml create mode 100644 build/config/ansible/roles/unattended_upgrades/tasks/main.yml create mode 100644 build/config/ansible/roles/unattended_upgrades/templates/50unattended-upgrades.j2 create mode 100644 build/config/terraform/.terraform.lock.hcl create mode 100644 build/config/terraform/README.md create mode 100644 build/config/terraform/main.tf create mode 100644 build/config/terraform/providers.tf create mode 100644 build/config/terraform/variables.tf create mode 100644 docs/authentik-migration.md create mode 100644 docs/forgejo-proxmox.md create mode 100644 docs/postgres-proxmox.md create mode 100644 docs/vault-secrets.md create mode 100644 src/arr/ansible/proxmox/README.md create mode 100644 src/arr/ansible/unraid/.env.example create mode 100644 src/arr/ansible/unraid/docker-compose.yml create mode 100644 src/arr/ansible/unraid/icon-prowlarr.png create mode 100644 src/arr/ansible/unraid/icon-shelfarr.png create mode 100644 src/arr/ansible/unraid/vars.yml create mode 100644 src/arr/common/vars.yml create mode 100644 src/arr/terraform/README.md create mode 100644 src/authentik/ansible/kubernetes/helmchart.yaml.j2 create mode 100644 src/authentik/ansible/kubernetes/media-pvc.yaml.j2 create mode 100644 src/authentik/ansible/kubernetes/vars.yml create mode 100644 src/authentik/ansible/proxmox/README.md create mode 100644 src/authentik/ansible/unraid/README.md create mode 100644 src/authentik/common/vars.yml create mode 100644 src/authentik/terraform/README.md create mode 100644 src/forgejo/ansible/proxmox/README.md create mode 100644 src/forgejo/ansible/proxmox/install.yml create mode 100644 src/forgejo/ansible/proxmox/templates/app.ini.j2 create mode 100644 src/forgejo/ansible/proxmox/templates/forgejo-dbdump.service.j2 create mode 100644 src/forgejo/ansible/proxmox/templates/forgejo-dbdump.sh.j2 create mode 100644 src/forgejo/ansible/proxmox/templates/forgejo-dbdump.timer.j2 create mode 100644 src/forgejo/ansible/proxmox/templates/forgejo.service.j2 create mode 100644 src/forgejo/ansible/proxmox/templates/pgpass.j2 create mode 100644 src/forgejo/ansible/proxmox/vars.yml create mode 100644 src/forgejo/ansible/unraid/.env.example create mode 100644 src/forgejo/ansible/unraid/docker-compose.yml create mode 100644 src/forgejo/ansible/unraid/vars.yml create mode 100644 src/forgejo/common/vars.yml create mode 100644 src/forgejo/terraform/README.md create mode 100644 src/forgejo/terraform/main.tf create mode 100644 src/forgejo/terraform/outputs.tf create mode 100644 src/forgejo/terraform/providers.tf create mode 100644 src/forgejo/terraform/variables.tf create mode 100644 src/hashicorp-vault/config.hcl create mode 100644 src/hashicorp-vault/policies/unraid-deploy.hcl create mode 100644 src/hashicorp-vault/policies/vault-readonly.hcl create mode 100644 src/hashicorp-vault/policies/vault-write.hcl create mode 100644 src/scripts/configure_k3s_node.zsh create mode 100644 src/shared/postgres/ansible/kubernetes/vars.yml create mode 100644 src/shared/postgres/ansible/proxmox/README.md create mode 100644 src/shared/postgres/ansible/proxmox/install.yml create mode 100644 src/shared/postgres/ansible/proxmox/templates/homelab.conf.j2 create mode 100644 src/shared/postgres/ansible/proxmox/templates/pg_hba.conf.j2 create mode 100644 src/shared/postgres/ansible/proxmox/vars.yml create mode 100644 src/shared/postgres/ansible/unraid/.env.example create mode 100644 src/shared/postgres/ansible/unraid/docker-compose.yml create mode 100644 src/shared/postgres/ansible/unraid/vars.yml create mode 100644 src/shared/postgres/common/vars.yml create mode 100644 src/shared/postgres/terraform/README.md create mode 100644 src/shared/postgres/terraform/main.tf create mode 100644 src/shared/postgres/terraform/outputs.tf create mode 100644 src/shared/postgres/terraform/providers.tf create mode 100644 src/shared/postgres/terraform/variables.tf diff --git a/.forgejo/workflows/deploy.yml b/.forgejo/workflows/deploy.yml new file mode 100644 index 0000000..2ddc8a3 --- /dev/null +++ b/.forgejo/workflows/deploy.yml @@ -0,0 +1,116 @@ +name: deploy + +on: + push: + branches: [main] + paths: + # No build/config/terraform/** — this workflow only deploys to Unraid, + # and `terraform apply` is run by hand. Add it if that ever changes. + - "src/**" + - "build/config/ansible/**" + +jobs: + deploy: + # Requires a self-hosted Forgejo Actions runner on your LAN (labelled + # `unraid-deploy`) with: network access to Vault and to the Unraid hosts' + # SSH + Postgres ports, and Python + pip available. See README.md + # "Bootstrapping" — this workflow can only run once that runner exists, + # which is why the first deploy has to happen by hand. The SSH key + # `ansible_user` (root) authenticates with is fetched from Vault below, + # not stored as a runner or repo secret — see README.md "SSH access". + runs-on: unraid-deploy + env: + VAULT_ADDR: ${{ secrets.VAULT_ADDR }} + VAULT_AUTH_METHOD: approle + VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID }} + VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # need history to diff — a shallow clone can't compute this + + - name: Install Ansible and Python dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install ansible hvac psycopg2-binary + + - name: Install Ansible collections + working-directory: build/config/ansible + run: ansible-galaxy collection install -r requirements.yml + + - name: Fetch SSH deploy key from Vault + run: | + set -eu + python3 - <<'PY' + import os + import hvac + + client = hvac.Client(url=os.environ["VAULT_ADDR"]) + client.auth.approle.login( + role_id=os.environ["VAULT_ROLE_ID"], + secret_id=os.environ["VAULT_SECRET_ID"], + ) + secret = client.secrets.kv.v2.read_secret_version( + path="homelab/ci/ssh", mount_point="kv" + )["data"]["data"] + + key_path = os.path.join(os.environ["RUNNER_TEMP"], "unraid_ansible_key") + with open(key_path, "w") as f: + f.write(secret["PRIVATE_KEY"].rstrip() + "\n") + os.chmod(key_path, 0o600) + PY + echo "ANSIBLE_PRIVATE_KEY_FILE=$RUNNER_TEMP/unraid_ansible_key" >> "$GITHUB_ENV" + + - name: Determine changed stacks + id: changed + run: | + set -eu + # Three possible outcomes, and they are not the same thing: + # all=true deploy every stack on every host + # only_stacks=a,b deploy just those + # skip=true nothing here affects the Unraid deployment + # An empty only_stacks must never be read as "deploy everything" — + # that is exactly what a Proxmox-only or Terraform-only push + # produces. + # + # A change under build/config/ansible/ can affect how every stack + # is deployed (the role, inventory, playbook), so don't try to + # narrow it. Same if the diff can't be computed at all (force-push, + # or a first push where `before` is all-zeros): fall back to a full + # deploy rather than silently deploying nothing. + if ! changed_files=$(git diff --name-only "${{ github.event.before }}" "${{ github.sha }}" 2>/dev/null); then + echo "all=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + if echo "$changed_files" | grep -q '^build/config/ansible/'; then + echo "all=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + # Only changes that affect the Unraid deployment count, since that + # is all this job runs. A stack's common/ and ansible/unraid/ do; + # its terraform/ and ansible/proxmox/ don't. + stacks=$(echo "$changed_files" \ + | { grep '^src/' || true; } \ + | { grep -Ev '/(terraform|ansible/proxmox)/' || true; } \ + | awk -F/ '{ print ($2 == "shared") ? $3 : $2 }' \ + | sort -u | paste -sd, -) + if [ -z "$stacks" ]; then + echo "skip=true" >> "$GITHUB_OUTPUT" + else + echo "only_stacks=$stacks" >> "$GITHUB_OUTPUT" + fi + + # Unraid only. Deploying to Proxmox from CI would need the runner to + # reach the Proxmox API and the guests, and `terraform apply` is + # deliberately kept manual — creating and destroying LXCs on a push is + # a bigger blast radius than restarting a Compose stack. Run + # playbooks/proxmox.yml by hand for now. + - name: Run deploy playbook + if: steps.changed.outputs.skip != 'true' + working-directory: build/config/ansible + run: | + if [ "${{ steps.changed.outputs.all }}" = "true" ]; then + ansible-playbook playbooks/unraid.yml + else + ansible-playbook playbooks/unraid.yml -e only_stacks="${{ steps.changed.outputs.only_stacks }}" + fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e5a72a3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,33 @@ +# Ansible +*.retry +build/config/ansible/inventory/*.retry + +# Rendered secrets — only .env.example is ever committed +**/.env +!**/.env.example + +# Terraform +# .terraform/ holds the provider cache and, after +# `terraform init -backend-config=...`, the Postgres connection string with +# its password — never commit it. .terraform.lock.hcl is deliberately NOT +# ignored: it pins provider versions and belongs in git. +.terraform/ +*.tfstate +*.tfstate.* +*.tfvars +!*.tfvars.example +# A saved plan (`terraform plan -out=…`) is not just a diff: it embeds a copy +# of the state it was made against *and* the values of every input variable, +# `sensitive` ones included — so build/config/terraform/deploy.plan contains +# the Proxmox API token in the clear. Ignored so that a `git add -A` can't +# publish it. +*.plan + +# OS cruft +.DS_Store +Thumbs.db + +# Scratch output that isn't state and isn't secret-free — e.g. the k3s +# kubeconfig playbooks/k3s.yml fetches to local/k3s/.kubeconfig, +# which carries a client cert. +local/ \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..6a70136 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,597 @@ +# Homelab IaC — Project Notes + +Manages apps as infrastructure-as-code across three platforms — Unraid +server(s), Proxmox, and a bare-metal k3s cluster — with secrets sourced from +HashiCorp Vault and CI/CD via Forgejo Actions (self-hosted, on the Forgejo +instance this repo deploys). + +Ansible is the deployment tool on **all three**; what differs is the +deployment primitive and who creates the host: + +| | Unraid | Proxmox | k3s | +|---|---|---|---| +| Provision the host | already exists | Terraform (`bpg/proxmox`) creates an LXC | already exists (4 Raspberry Pis) | +| Install the app | Ansible + `compose_stack` → `docker compose up -d` | Ansible + `lxc_app` → native install + systemd | Ansible + `k3s_app` → manifests into k3s's auto-deploy dir | +| App config source | `common/vars.yml` + `ansible/unraid/vars.yml` | `common/vars.yml` + `ansible/proxmox/vars.yml` | `common/vars.yml` + `ansible/kubernetes/vars.yml` | +| App list lives in | `host_vars/.yml` `stacks:` | `host_vars/.yml` `apps:` | `group_vars/k3s_cluster.yml` `k3s_apps:` | + +The k3s column is the newest and reached that shape late: `homelab-utils` +started as cluster-bootstrap-only, with no `src//` entry, and the third +column was added when Authentik moved onto it (see +`docs/authentik-migration.md`). Two things about it don't reduce to the +table: + +- **The cluster itself has no `src/` entry.** Terraform never provisions it + and it isn't an app, so it gets an inventory group (`k3s_cluster`) and + `roles/k3s_node`, the same way the Proxmox *node* gets `roles/pve_backup` + while its guests get `lxc_app`. +- **Cluster services aren't apps.** `k3s_metallb`, `k3s_monitoring`, + `k3s_postgres` and `k3s_cert_manager` are the cluster's own + infrastructure — one hard-coded role each, config in `group_vars`. Apps go + through the generic `k3s_app` and live in `src/`. Both end up as + `HelmChart` CRs, so they look alike on disk; the distinction is whether + anything would still need it if no app were deployed. See "K3s cluster + (homelab-utils)" below. + +### Hosts + +Two Unraid entries are gone from the original plan: `nas2` was retired, and its +address (192.168.50.2) was reassigned to `turtle-proxmox-01` — so a stale copy +of it fails by reaching the wrong machine rather than by timing out. What +remains: `nas01` (Unraid, 192.168.50.1), `turtle-proxmox-01`/`-02` (Proxmox, +.2/.3), the guests on them from the API inventory, and the four k3s Pis +(.60–.63, MetalLB pool .80–.99). Gateway is 192.168.50.254. + +**Proxmox LXC guests live in 192.168.50.50–.59** — Forgejo .52, shared +Postgres .54. Addresses here are assigned by hand from that band, not derived +from anything, so a new guest's `ip_address` is a question for whoever owns +the network rather than a value to pick. + +The shared Postgres for everything outside the cluster is now the Proxmox LXC +at 192.168.50.54 — see `docs/postgres-proxmox.md`. The Unraid Compose +deployment under `src/shared/postgres/ansible/unraid/` is kept for the +platform-shape reason every unused platform folder is kept, not because it is +running anywhere. + +This repo supersedes the planning done in `../homelab/CLAUDE.md` — that file +has the original decision log if you want the "why" behind the architecture +in more detail. Multi-host support, the shared-Postgres pattern, and the +Forgejo-Actions-hosts-itself bootstrap problem were decided after that doc +was written; this file and `README.md` are the current source of truth. + +## Layout + +- `src//common/vars.yml` — `env_defaults:` that hold regardless of where + the app runs (version, ports, domain, DB name/user). +- `src//ansible/unraid/` — the Compose deployment: `docker-compose.yml`, + `.env.example`, a `vars.yml` of Unraid-specific overrides (appdata paths, + the `shared-postgres` Docker network alias), and optionally an `icon.png` + for the Unraid Docker page. +- `src//ansible/proxmox/` — the native install: `vars.yml` of + Proxmox-specific overrides, `install.yml` of app-specific install steps, + and templates for the app's own config file and systemd unit. Two apps + ship one: `src/forgejo/` (binary + systemd unit, the shape the role was + written for) and `src/shared/postgres/` (distro packaging, so the unit + comes from the package and `install.yml` only reconfigures the cluster). +- `src//ansible/kubernetes/` — the k3s deployment: `vars.yml` of + cluster-specific overrides, and one or more `*.yaml.j2` manifests (normally + a `HelmChart` CR, plus anything the chart won't create — see + `src/authentik/`). "kubernetes" rather than "k3s": the contents are plain + Kubernetes apart from the `HelmChart` CR. +- `src//terraform/` — a module defining this app's LXC (template, cores, + memory, disk, IP), called from `build/config/terraform/main.tf`. +- `src/shared//` — shared services (databases, etc.) used by + multiple apps, same shape. +- `build/config/ansible/` — inventory, playbooks, and the roles. Three of + them deploy *apps*, one per platform, and share the same layering and + `state: present|absent` contract: `compose_stack` (Unraid), `lxc_app` + (Proxmox guests), `k3s_app` (the cluster). `compose_stack` and `k3s_app` + each split into `deploy.yml`/`remove.yml`, dispatched by `main.yml` on the + entry's `state:`. The rest configure infrastructure rather than apps: + `k3s_node` (bootstraps a Pi as k3s server or agent, picked by inventory + group), `k3s_metallb`, `k3s_monitoring`, `k3s_postgres`, + `k3s_cert_manager` (cluster services), `k3s_maintenance` (drain/reboot/ + uncordon), `unattended_upgrades`, and `pve_backup` (the Proxmox node's + backup storage and vzdump schedule). +- `build/config/terraform/` — Proxmox provider + `backend "pg"` state config, + and the module calls saying what infrastructure exists. +- `.forgejo/workflows/` — CI/CD, runs once Forgejo + a self-hosted runner + exist (see "Bootstrapping" in README.md — chicken-and-egg on the first + deploy). + +## Key decisions + +- Compose files are static and reference `${VAR}`; only the rendered `.env` + changes per deploy. Nothing secret is ever committed. +- Config is layered, not duplicated: `common/vars.yml` merged with + `ansible//vars.yml` (platform wins), then Vault secrets on top + (secrets win). Both roles do this the same way, so moving an app between + platforms means writing a new platform vars file, not restating its config. +- One Vault KV path per app (`homelab/`), fetched as a whole dict via + `community.hashi_vault.vault_kv2_get`. The prefix is `homelab/`, not + `unraid/`, because an app's secrets don't change with the platform it lands + on. +- Unraid draws a Compose deployment's stack row and its containers from + unrelated sources, so one committed `src//ansible/unraid/icon.png` is + copied to two places. The **stack row** is Compose Manager serving + `/icon.png` off disk — a file or nothing, no label and no URL, + and only in the maintained fork (Compose Manager Plus); on the original + plugin the copy is inert. The **containers** get icon, WebUI link and + console shell from the dockerMan template that created them, which a Compose + stack doesn't have, so each service carries `net.unraid.docker.*` labels + (6.10+ falls back to them). The label path is a second copy under + `/mnt/user/appdata/icons/` rather than the project-folder one, because the + webgui resolves it on every Docker page load and that shouldn't hit the + flash drive. +- Of those labels only the icon is role-supplied: `compose_stack` injects + `STACK_ICON` *underneath* both vars.yml layers, so an app can override it + with a hosted URL. WebUI and shell stay literal in the compose file — the + WebUI port has to be the published one and the shell is a property of the + base image, so both belong next to `ports:` and `image:` rather than in a + variable. Details, including where SVG does and doesn't work, in README.md → + "Icons on the Unraid Docker page". +- Shared Postgres is provisioned once per host; per-app DB/user creation + is idempotent via `community.postgresql`, not `docker-entrypoint-initdb.d` + (which only runs once on an empty volume — wrong fit for a shared service + apps get added to over time). The Unraid instance was pinned to 13 because + its data directory predated this repo; the Proxmox one is 17 from the PGDG + archive, a fresh cluster with nothing to stay compatible with. +- The Proxmox shared Postgres lives on the `AppData` ZFS pool with a + `proxmox_replication` job to the second node, and **everything is on one + volume** — no `mount_point` splitting PGDATA from the WAL. Unlike Forgejo's + "no bind mounts", which is about what `vzdump` includes, this is a + correctness requirement: replication snapshots each of a guest's volumes + independently, so two volumes replicate as two snapshots taken at different + instants, and the far node can end up with a data directory and a WAL that + disagree. One volume means one atomic snapshot — exactly the + crash-consistent image WAL replay is designed for. It also means running + out of space has one fix and not two: raise `disk_size` on the module call + (256 GiB as of this writing, up from the original 32 once Forgejo's + database made the OS-sized rootfs look optimistic), never add a second + disk. Cheap to do — on ZFS the resize is a refquota change applied to a + running guest — but one-way, since shrinking is a replacement and + `prevent_destroy` blocks it. Runbook in `docs/postgres-proxmox.md`. Two + further consequences worth keeping straight: replication is configured *per guest*, not per pool, so a + container created on `AppData` without a job is a single point of failure + that looks identical in the storage view (hence Terraform owning the job + rather than a hand-run `pvesr`); and it's asynchronous, so a failover loses + up to `replication_schedule` — five minutes — of commits. It is not a + backup; `pve_backup`'s vzdump is. +- `full_page_writes = off` on the Proxmox instance, and only there. It guards + against a torn page, which copy-on-write storage cannot produce, so on ZFS + the full-page images are pure write amplification that also becomes + replication traffic. The same setting on ext4 or xfs risks unrecoverable + corruption — which is why it's in `ansible/proxmox/vars.yml` and must never + migrate to `common/vars.yml`. +- Terraform provisions the guest and stops there — it has no way to deploy an + app into an LXC, so Ansible picks up from the point the container exists. +- Proxmox guests keep all their state on their own rootfs; no bind mounts, no + NFS shares mounted into a container. Counter-intuitive if the goal is "back + the repos up to the NAS", but `vzdump` excludes bind mounts by design, so a + bind-mounted repo directory is the one thing *missing* from the archive. + Everything local means one archive is a complete filesystem restore point. + Getting a copy off-box is the node's job (`pve_backup` → NFS storage on the + NAS + a vzdump schedule), not the container's. +- State an app keeps *outside* its container is that app's problem. Forgejo's + database is on the shared Postgres LXC — a different guest, and so a + different vzdump archive — so a systemd timer in the + container `pg_dump`s it onto the container's own disk, and the vzdump archive + carries both halves. The dump schedule and the vzdump window are a pair — + move one, move the other. +- Proxmox hosts come from the `community.proxmox` dynamic inventory plugin + querying the Proxmox API, so `terraform apply` creating an LXC doesn't also + require an inventory edit. +- The `terraform` tag is what marks a Proxmox guest as this repo's to manage. + The dynamic inventory returns every LXC on the cluster, hand-made ones + included, so `inventory/proxmox.yml` filters on that tag — both Terraform + modules set it (`tags` in their `variables.tf`), nothing else does. It + filters rather than narrowing the `proxmox_guests` condition, so an + unmanaged guest is absent from `all` as well and no future play can reach + it by accident. The failure mode is silent in one direction: a new module + that forgets the tag creates a container Ansible simply never visits, with + no error anywhere — same shape as an omitted `stacks:` entry, and the first + thing to check if a freshly-applied guest is never deployed to. +- Terraform state lives in Postgres (`backend "pg"`), not a local file, so CI + and a workstation see the same state — specifically the **CloudNativePG + cluster on k3s**, not the shared Postgres this configuration provisions on + Proxmox. That would be circular: Terraform would need the database to exist + in order to create the container the database runs in. State goes somewhere + Terraform has no hand in building, which breaks the cycle outright at the + cost of a dependency the rest of the repo avoids — `terraform apply` now + needs the Pis up. Worth it over the alternative (local state, apply, + `init -migrate-state`), which has to be got right exactly once. Bootstrap + ordering is `docs/postgres-proxmox.md`. +- Every app keeps the same `common/` + `ansible/` + `terraform/` shape even + when only one platform is in use, so adding the other later doesn't require + restructuring. +- Removal on Unraid is explicit (`state: absent` on the stack entry), never + implied by deleting the entry. Ansible stores no record of the last run, so + an omitted stack isn't removed — it's just never visited again, and keeps + running unmanaged. The entry stays as a tombstone documenting that the app + was deliberately removed. Volumes, images and the app's database are + further opt-ins (`remove_volumes`, `remove_images`, `remove_database`) + rather than being implied by `state: absent`, and the Vault path is never + touched — a default teardown is meant to be reversible. `lxc_app` has no + counterpart yet; it would need a per-app `uninstall.yml`. +- The k3s cluster's join token is a fixed value from Vault + (`homelab/k3s-homelab-utils` → `K3S_TOKEN`), not the one `k3s server` + generates for itself on first install. Both the control-plane play and the + worker play look it up independently, so a full rebuild — wipe both SD + cards, reinstall — reproduces the same cluster identity instead of needing + a freshly-generated token fetched off the server and redistributed by hand. + Same "generate once, store it, never let a redeploy invent a new one" + reasoning as Forgejo's fixed secrets, applied to cluster identity instead + of app state. +- The SSH key the k3s plays connect with is fetched from Vault + (`homelab/ci/ssh-k3s`) by `playbooks/k3s_ssh_key.yml` and written to + `local/`, which both `k3s.yml` and `k3s_maintenance.yml` import as their + first play — rather than the operator loading it into `ssh-agent` first, as + the Unraid manual flow still expects. Same arrangement CI already uses for + the Unraid key, just moved into the playbook because these two are + manual-only and there's no workflow to hold the fetch step. It's a separate + playbook, not a `pre_tasks` block, because both entry points need it and + neither can assume the other ran. It targets `k3s_cluster` rather than + `localhost` for a non-obvious reason: the implicit localhost isn't a member + of `all`, so it wouldn't inherit `vault_addr`/`vault_kv_mount`/ + `vault_auth_method` from `group_vars/all.yml`. Nothing in it connects to a + Pi — `gather_facts: false` plus `delegate_to: localhost`, necessarily, since + the key isn't on disk yet at that point. Falling back to a local key takes + *two* overrides (`-e k3s_ssh_key_fetch=false` alongside + `-e ansible_ssh_private_key_file=...`): pointing at another key alone + doesn't help if Vault is what's broken, because the fetch play fails before + any other play runs. +- `playbooks/k3s.yml` is deliberately **not** wired into `deploy.yml` or + `.forgejo/workflows/deploy.yml`. Converging 4 physical nodes on every push + is a bigger blast radius than restarting a Compose stack — the same + reasoning that keeps `terraform apply` manual for Proxmox. Run it by hand. +- `playbooks/k3s.yml`'s plays are tagged so one piece can be converged on its + own — `nodes`, `metallb`, `monitoring`, `postgres`, `services` (those three + cluster services together), `upgrades`, `kubeconfig`. Play-level tags, one + per play, not per-task tags inside the roles: the plays already are the + units, and tagging tasks would mean every role carrying tags for a + selectivity only this playbook needs. This is the k3s equivalent of + `only_stacks` for Unraid — a cluster service isn't a `stacks:` entry, so + `only_stacks` can't reach it. The SSH-key import play is tagged `always` + rather than getting a name of its own, since it's the prerequisite for + anything else connecting and has to survive every `--tags` filter. Note + what a tagged service run does *not* cover: the roles only drop manifests + for k3s's controllers to reconcile, so a green playbook means the files + landed, not that the workload came up. +- `k3s_node` is one role, not two, even though a server and an agent install + differently. Both need the same OS prep (cgroups, swap), and which install + path runs is a group var (`k3s_node_role: server`/`agent`, set per + inventory group) rather than a second role — so the shared prep can't drift + between the two nodes types the way copy-pasted tasks eventually would. +- Extra `INSTALL_K3S_EXEC` flags split across two vars, not one: + `k3s_extra_args` (both roles) and `k3s_server_extra_args` (control-plane + only). `k3s agent` doesn't understand server-only flags like `--disable` + and fails to start if handed one, so a flag like `--disable=servicelb` + (which `k3s_metallb` needs to avoid fighting k3s's bundled ServiceLB for + the same IPs) can't safely live in the shared list. +- `k3s_node`'s install tasks track the exec line they last installed a node + with (a small marker file next to k3s's own config) and reinstall — + restarting the `k3s`/`k3s-agent` service, not the node — whenever it + drifts from what `k3s_extra_args`/`k3s_server_extra_args`/`k3s_api_tls_san` + currently say, even if `k3s_version` hasn't changed. Deliberately folded + into the ordinary install task rather than a separate maintenance + playbook: `playbooks/k3s.yml` is already manual-only and documented as + safe to re-run, and unlike the reboots `k3s_maintenance` guards, bouncing + the k3s process doesn't take pods down — containerd keeps them running + underneath it. The worker play still runs with `serial: 1` so at most one + node's kubelet is ever mid-restart at once. +- `k3s server`/`k3s agent` are installed with an explicit `--node-name + {{ inventory_hostname }}`, not left to default to the OS hostname. This is + what lets `roles/k3s_maintenance` address a node by + `inventory_hostname` when draining/uncordoning — the k8s node object and + the Ansible host are guaranteed to be the same string. +- Patching a k3s Pi is split into two roles that don't know about each other + directly, only through a file: `unattended_upgrades` installs updates + hands-off but with `Automatic-Reboot "false"`, and `k3s_maintenance` is the + only thing that actually reboots a node — triggered by the presence of + `/var/run/reboot-required`, which is the OS's own signal, not something + either role invents. Splitting it this way means the risky half (taking a + node out of the cluster) is exactly one thing (`playbooks/k3s_maintenance.yml`, + `serial: 1`), not entangled with the routine half (installing packages), + which runs unattended every day on every node without anyone thinking about + it. +- `k3s_maintenance`'s kubectl calls (`drain`, `wait`, `uncordon`) are + delegated to the control-plane host and run as `k3s kubectl`, k3s's own + bundled client, rather than requiring a kubectl install or a kubeconfig + anywhere else — including on whatever eventually runs this on a schedule. + This also has to work when the node currently being processed *is* the + control plane: delegating a host to itself is just a normal SSH connection, + used right up until the moment that connection reboots out from under it. +- `k3s_monitoring` (a lean kube-prometheus-stack, for OpenLens and similar + tools) is installed as a `HelmChart` custom resource dropped into k3s's own + auto-deploying manifests directory, not run through a `helm` binary or the + `kubernetes.core.helm` collection — k3s ships a helm-controller that + reconciles anything found there, the same mechanism it uses to install its + own bundled Traefik and ServiceLB. That keeps the pattern this repo already + uses everywhere else (Ansible renders a file, something else converges on + it) instead of adding a second, unrelated way to reach the cluster from the + controller. Grafana and Alertmanager are left disabled — OpenLens brings + its own dashboards and this cluster doesn't page anyone — and the + control-plane scrape targets (`kubeControllerManager`, `kubeScheduler`, + `kubeProxy`, `kubeEtcd`) are disabled too, since k3s bundles those into one + static binary instead of exposing them the way the chart expects; leaving + them on just produces permanently-"down" targets, not a working scrape. +- `k3s_metallb` (MetalLB, L2 mode) follows the same `HelmChart` CR pattern as + `k3s_monitoring`, plus a second, plain manifest for its + `IPAddressPool`/`L2Advertisement` config dropped in the same + auto-deploying directory — k3s's deploy controller applies any manifest it + finds there, not only `HelmChart` CRs, and retries one referencing CRDs + that don't exist yet rather than failing outright, so the config doesn't + need to wait on the chart install finishing first. It replaces k3s's + bundled ServiceLB rather than running next to it — both would otherwise + hand out IPs for the same `LoadBalancer` Services — so + `k3s_server_extra_args` in `inventory/group_vars/k3s_cluster.yml` + carries `--disable=servicelb` (server-only — `k3s_extra_args`, applied to + both server and agent, can't carry a server-only flag like `--disable` + without breaking agent installs). The IP pool (`k3s_metallb_address_range`, same + file) has no built-in default; the role fails fast rather than silently + advertising an empty pool. +- `k3s_postgres` (CloudNativePG) is the k3s cluster's own shared Postgres — + a third platform for `src/shared/postgres/`, alongside Unraid and Proxmox, + deployed via Helm rather than Compose or a native install. Chosen over + Bitnami's postgresql-ha (repmgr+pgpool, and Bitnami's free chart/image + catalog was restructured into a "legacy" repo in 2025) and the Zalando + operator (Patroni-based, heavier) because it fits the same `HelmChart` CR + pattern already established: the operator installs as a chart CR, same + shape as `k3s_metallb`/`k3s_monitoring`, and the actual cluster is a plain + `Cluster` CR manifest — same "config manifest that outlives the CRDs it + references" trick as `k3s_metallb`'s `IPAddressPool`. It's a separate + physical instance from the Docker-based shared/postgres (pinned to 13), + but reuses the same Vault path (`homelab/shared/postgres`) rather than a + k3s-specific one — one superuser identity for the "shared postgres" + concept regardless of which platform it's running on, consistent with how + every host already draws from that same path. 1 primary + 1 replica, + scheduled on worker Pis only via `nodeAffinity` — the control plane stays + free of app pods, same boundary `k3s_metallb`/`k3s_monitoring` already + keep. HA replication only for now, no backups. +- The CNPG cluster is published on the LAN through MetalLB at a pinned + `k3s_postgres_loadbalancer_ip`, on top of the three `ClusterIP` Services + CNPG makes for every `Cluster`. Declared inside the `Cluster` CR as a + managed service (`.spec.managed.services.additional`, `selectorType: rw`) + rather than as a Service manifest of this repo's own, so the operator owns + the selector and the address follows a failover instead of needing to be + re-pointed by hand. Pinned rather than auto-assigned because the address is + meant to be written down in config elsewhere; it comes out of the bottom of + `k3s_metallb_address_range` and would need a second `autoAssign: false` + pool if anything else on this cluster ever wants a fixed IP. This is what + closes the reachability half of per-app DB provisioning — the + `community.postgresql` tasks `compose_stack`/`lxc_app` use can now reach + this instance the same way they reach the Unraid one. `roles/k3s_app` is + the other half, and uses exactly those tasks; note the asymmetry it + relies on, which is easy to misread as a mistake. The *provisioning* runs + from the Ansible controller and therefore uses the LoadBalancer address + (`k3s_postgres_loadbalancer_ip`), because `delegate_to: localhost` is off + the cluster and can't route to a ClusterIP. The *app* uses the in-cluster + `shared-postgres-rw` DNS name, because sending pod traffic out to the LAN + and back would put MetalLB's L2 speaker in the path of every query for no + benefit. Two addresses for one database, each correct for its caller. +- `roles/k3s_app` is the cluster's counterpart to `compose_stack` and + `lxc_app`: the generic role that deploys an *app*, driven by `k3s_apps:` + in `group_vars/k3s_cluster.yml`, as distinct from the `k3s_*` roles that + each deploy one cluster service. It renders whatever `*.yaml.j2` the app + ships under `src//ansible/kubernetes/` into k3s's auto-deploying + manifests directory, so adding a manifest to an app is dropping a file + next to the others rather than a role change. Secrets go into a separate + Kubernetes Secret rendered from Vault, never interpolated into the + manifests — the same committed-config/generated-secrets split as Unraid's + static compose file plus rendered `.env`, which is why the manifests can + stay 0644 on the node while the Secret is 0600. +- `k3s_apps:` lives in `group_vars/k3s_cluster.yml`, not `host_vars/`, unlike + both other platforms. An app is deployed to the cluster, not to a node: + the play runs against `k3s_control_plane` only because that's where the + manifests directory is, which is an implementation detail of how k3s is + reached rather than a statement about where the app runs. +- Removal on k3s is one pass, not two. `compose_stack` needs the compose file + still on disk to run `docker compose down`, so `state: absent` and deleting + `src//` have to be separate commits (see "Removing an app" below). + k3s's deploy controller tracks which resources each manifest file created, + so deleting the file *is* the teardown and `src//` can go in the same + commit. The tombstone convention still applies for the same reason as + everywhere else — an entry dropped from `k3s_apps:` is never visited again + and keeps running unmanaged. +- `k3s_cert_manager` is the fourth cluster service and the first that exists + purely for apps rather than for the cluster — nothing in the cluster needs + a certificate, `k3s_app`'s tenants do. It's still a cluster service rather + than an app, on the same reasoning as the shared Postgres: one + `ClusterIssuer` that every app's Ingress annotates itself against, rather + than each app carrying its own ACME account and DNS credentials. DNS-01, + not HTTP-01, because HTTP-01 needs Let's Encrypt to reach the cluster from + the internet on port 80 and this LAN deliberately isn't reachable — which + has the useful side effect that a certificate can be issued *before* DNS + points at the cluster, so a migration's TLS is settled before its cutover. + The solver stanza itself has no default and the role fails fast without + one: it depends on who runs the DNS, which the repo can't know. Same + fail-fast-on-unset treatment as `k3s_metallb_address_range`. +- Apps on k3s reach the outside through the Traefik k3s already bundles, + on the MetalLB address its Service holds, rather than each getting its own + LoadBalancer IP. One entrypoint and host-based routing means adding an app + costs a DNS record, not a pool address — the opposite of the choice made + for `k3s_postgres`, which is pinned to its own IP precisely because + Postgres isn't HTTP and can't be name-routed. +- `k3s_traefik` is the one cluster-service role that installs nothing: k3s + installs Traefik itself, so the role only adjusts it, via a + `HelmChartConfig` merged over k3s's own `HelmChart` rather than an edit to + the `traefik.yaml` k3s rewrites on every server start. Its scope is the + dashboard, which a stock k3s 404s — Traefik still builds it, but the chart + stopped shipping the router that reaches it in v28. The role restores that + router on Traefik's internal `traefik` entrypoint (port 9000, unpublished → + port-forward only), and optionally publishes the dashboard on a hostname + behind an Authentik forward-auth middleware when + `k3s_traefik_dashboard_host` is set. Three things worth keeping straight: + the internal route stays on even when the hostname one exists, because + authenticated access depends on Authentik → CNPG → a healthy cluster, + exactly what you'd open the dashboard to diagnose; the published route is + an `IngressRoute` with an explicit cert-manager `Certificate` rather than + an annotated `Ingress`, because the dashboard is `api@internal` and has no + Service to point an `Ingress` at; and the role fails fast if a hostname is + set without an auth address, since the failure mode is publishing every + router, service and middleware on the cluster to the LAN unauthenticated. + Authentik's provider must be **forward auth (domain level)** — + single-application mode needs `/outpost.goauthentik.io/` routed on the + dashboard's own host, a cross-namespace service reference Traefik rejects + unless `allowCrossNamespace` is on. + +## Adding a new app + +1. `src//common/vars.yml` — `env_defaults:` for the portable config. +2. Whichever platform(s) it targets: + - **Unraid:** `src//ansible/unraid/` with `docker-compose.yml` + (static, `${VAR}`-driven), `vars.yml` of overrides, `.env.example` + documenting every var. + - **Proxmox:** `src//ansible/proxmox/` with `vars.yml`, + `install.yml`, and config/systemd templates; plus + `src//terraform/` defining its LXC, wired into + `build/config/terraform/main.tf`. + - **k3s:** `src//ansible/kubernetes/` with `vars.yml` and at least + one `*.yaml.j2` manifest — normally a `HelmChart` CR, plus anything the + chart won't create for you. See `src/authentik/` for a worked example, + including the two things a chart usually leaves out: a PVC, and secrets + referenced by `secretKeyRef` rather than set as values. +3. Populate its Vault path (`homelab/`) with real secret values. +4. Declare it on the target: + - Unraid → `host_vars/.yml` `stacks:` list. + - Proxmox → `host_vars/.yml` `apps:` list. + - k3s → `group_vars/k3s_cluster.yml` `k3s_apps:` list. + All three take the same entry shape (name, src, vault_path, optional + `db:`, optional `state:`). Never declare the same app on more than one at + once — they share a Vault path and a database, so two live deployments + corrupt each other's state. Moving an app *between* platforms is + therefore a cutover with a verification step in the middle, not an edit; + `docs/authentik-migration.md` is the worked example. +5. Leave the unused platforms' folders as README placeholders rather than + deleting them — the shape is the point. + +## Removing an app + +**Unraid.** Set `state: absent` on its `stacks:` entry and run +`playbooks/unraid.yml` — don't delete the entry, which removes nothing. Then +delete `src//` in a *second* pass, never before: `docker compose down` +needs the compose file to know what it's tearing down. Full detail, including +the opt-in flags for volumes/images/database, is in README.md → "Removing an +app". + +**k3s.** Same `state: absent` tombstone rule, but one pass — deleting the +manifests is the teardown, because k3s's deploy controller garbage-collects +what each file created, so `src//` can go in the same commit. The +database, PVCs and Vault path deliberately survive; there's no +`remove_volumes`/`remove_database` equivalent yet, and adding one is a +"when it's actually wanted" job rather than a guess at the shape. + +**Proxmox.** No teardown path at all — `lxc_app` would need a per-app +`uninstall.yml`, and neither app that ships an `install.yml` has one. Note +this is worse for the shared Postgres than for an ordinary app: tearing it +down means every other app's database as well. + +## K3s cluster (homelab-utils) + +4 Raspberry Pis, `inventory/hosts.yml` → `k3s_cluster` (`k3s_control_plane`: +1 host, `k3s_workers`: 3). No Terraform — the Pis already exist. The cluster +itself has no `src//` entry (it isn't an app), but apps now land on it: +`k3s_apps:` in `group_vars/k3s_cluster.yml`, deployed by `roles/k3s_app` from +`src//ansible/kubernetes/`. `playbooks/k3s.yml` is manual-only (see "Key +decisions" above); run it with: + +```sh +cd build/config/ansible +ansible-playbook playbooks/k3s.yml + +# or one piece of it — see "Key decisions" above for the tag list +ansible-playbook playbooks/k3s.yml --tags postgres + +# apps only, or one app +ansible-playbook playbooks/k3s.yml --tags apps +ansible-playbook playbooks/k3s.yml --tags apps -e only_apps=authentik +``` + +`--tags services` covers the cluster's own infrastructure and `--tags apps` +what runs on top of it; neither implies the other, so "everything except +reinstalling k3s" is `--tags services,apps`. + +Prerequisites, once, before the first run: the `ansible` user exists on every +Pi with NOPASSWD sudo and the **public** half of this repo's k3s SSH key +installed, and `homelab/k3s-homelab-utils` → `K3S_TOKEN` plus +`homelab/ci/ssh-k3s` → `PRIVATE_KEY` are set in Vault. Nothing needs setting +up on the controller beyond `VAULT_ADDR`/`VAULT_TOKEN` — the private key comes +from Vault at run time (see "Key decisions" above). Full detail in README.md → +"K3s (Raspberry Pi)" and `docs/vault-secrets.md`. + +**Rebuilding**: reimage the SD card(s), redo the `ansible` user + SSH key +step, then re-run the playbook — the fixed Vault token means the rebuilt +node(s) rejoin the same cluster identity rather than needing a token hunted +down from a live server. **Adding a fifth Pi**: add it under +`k3s_control_plane` or `k3s_workers` in `hosts.yml`; no `host_vars/` entry +needed, `k3s_node_role` comes from the group. This repo's `k3s_node/server.yml` +assumes a single control-plane node — adding a second one for HA needs +changes there, not just an inventory edit. + +**Patching**: `playbooks/k3s.yml` also applies `roles/unattended_upgrades` to +every node, so updates install themselves daily with no login required — but +never reboot themselves (`Automatic-Reboot "false"`). Run +`playbooks/k3s_maintenance.yml` to actually apply a pending reboot: it checks +`/var/run/reboot-required` per node and, only where set, drains → reboots → +waits for `Ready` → uncordons, `serial: 1` so only one node is ever down. +Manual-only for now, same as `playbooks/k3s.yml` — meant to eventually run on +a schedule from a self-hosted Forgejo Actions runner rather than by hand. + +**Metrics**: `playbooks/k3s.yml` also applies `roles/k3s_monitoring`, a lean +kube-prometheus-stack (no Grafana/Alertmanager) so OpenLens and similar tools +can show node/pod metrics — see "Key decisions" above for why it's a +`HelmChart` CR rather than a `helm` install, and README.md → "Metrics +(Prometheus)" for pointing OpenLens at it. + +**LoadBalancer IPs**: `playbooks/k3s.yml` also applies `roles/k3s_metallb`, +MetalLB in L2 mode, so `type: LoadBalancer` Services get real LAN IPs instead +of staying `ClusterIP`-only. Set the pool it hands out from — +`k3s_metallb_address_range` in `inventory/group_vars/k3s_cluster.yml` — to a +range your DHCP scope and every static assignment in `hosts.yml` steer clear +of before the first run; see "Key decisions" above for why it replaces +ServiceLB rather than running alongside it, and README.md → "LoadBalancer +IPs (MetalLB)" for more. + +There's no *node* teardown path, unlike `state: absent` for apps — removing a +Pi today means wiping its SD card and deleting its inventory entry by hand. + +**Shared Postgres**: `playbooks/k3s.yml` also applies `roles/k3s_postgres`, +CloudNativePG — the cluster's own shared Postgres, a third platform for +`src/shared/postgres/` alongside Unraid and Proxmox. See "Key decisions" +above for why CloudNativePG and the topology, and README.md → "Postgres +(CloudNativePG)" for connecting to it. Per-app database provisioning works +now (`roles/k3s_app`, `db:` on a `k3s_apps:` entry) — note the two-address +asymmetry described in "Key decisions". Still **no backups**: HA replication +only, which protects against a dead SD card and not against a dropped table. +That gap matters more now that Authentik's database lives here rather than +inside whatever covers Unraid's `/mnt/user/appdata`; a CNPG `ScheduledBackup` +is the obvious next piece of work. + +**TLS**: `playbooks/k3s.yml` also applies `roles/k3s_cert_manager` — +cert-manager plus one `ClusterIssuer` that every app's Ingress annotates +itself against. Two values have no default and the role refuses to run +without them: `k3s_cert_manager_acme_email` and `k3s_cert_manager_solver` +(the DNS-01 stanza, which depends on your DNS provider — a commented +Cloudflare example is in `group_vars/k3s_cluster.yml`). Use the Let's Encrypt +staging directory while working a solver out; production allows five failed +validations per hostname per hour. + +**Traefik dashboard**: `playbooks/k3s.yml` also applies `roles/k3s_traefik` — +the only cluster-service role that installs nothing, since k3s installs +Traefik itself. Out of the box the dashboard is reachable only by +port-forward (`kubectl -n kube-system port-forward deploy/traefik 9000:9000`, +then `http://127.0.0.1:9000/dashboard/` — trailing slash required; without +this role a stock k3s 404s there). Publishing it on a hostname behind +Authentik is one commented-out line, `k3s_traefik_dashboard_host` in +`group_vars/k3s_cluster.yml`, plus a domain-level forward-auth Proxy Provider +on the Authentik side. See "Key decisions" above and README.md → "Dashboard +(Traefik)". + +**Apps**: `k3s_apps:` in `group_vars/k3s_cluster.yml`, deployed by +`roles/k3s_app`. Authentik is the first and currently only one — it moved off +a hand-made Unraid container rather than being deployed fresh, so +`src/authentik/ansible/unraid/` is a README explaining why there's no compose +file there. The cutover procedure is `docs/authentik-migration.md`; the part +worth knowing without reading it is that the deployed `AUTHENTIK_VERSION` +must match the version a restored database was dumped from, because +Authentik's migrations run on startup and are one-way. diff --git a/README.md b/README.md new file mode 100644 index 0000000..0fcab00 --- /dev/null +++ b/README.md @@ -0,0 +1,966 @@ +# Homelab IaC + +Deploy apps as code across two platforms: Unraid server(s), where apps run as +Docker Compose stacks, and Proxmox, where Terraform creates an LXC and the +app is installed into it natively. Ansible is the deployment tool on both. +Secrets are pulled from HashiCorp Vault at deploy time; CI/CD is Forgejo +Actions once Forgejo itself is running. + +There's also a third piece of infrastructure alongside those two: a bare-metal +k3s cluster (`homelab-utils`) across 4 Raspberry Pis, bootstrapped by Ansible +too. It doesn't fit the `src//` table above the way Unraid/Proxmox apps +do — Terraform never provisions it, and no `stacks:`/`apps:` list drives +what's on it — so it gets its own inventory group and role instead. It does +run shared services deployed via Helm, though: `k3s_monitoring`, `k3s_metallb`, +and the cluster's own shared Postgres (`k3s_postgres`, a third platform for +`src/shared/postgres/`). See `playbooks/k3s.yml` and "K3s (Raspberry Pi)" +below. + +See `CLAUDE.md` for the short version of the architecture decisions. + +## Layout + +``` +src/ + / + common/vars.yml config that holds on either platform + ansible/ + unraid/ docker-compose.yml + .env.example + platform overrides + (+ optional icon.png for the Unraid Docker page) + proxmox/ install.yml + config/systemd templates + platform overrides + kubernetes/ vars.yml + *.yaml.j2 manifests (normally a HelmChart CR) + terraform/ this app's LXC definition (Proxmox only) + shared// same shape, for services multiple apps use +build/config/ + ansible/ + inventory/ + hosts.yml static: unraid_servers, proxmox_nodes, k3s_cluster + proxmox.yml dynamic: proxmox_guests, from the Proxmox API + group_vars/all.yml Vault connection, Compose Manager project path + group_vars/k3s_*.yml k3s cluster name/version, per-role vars (server/agent) + host_vars/.yml which stacks/apps deploy there + (k3s apps are in group_vars/k3s_cluster.yml instead) + roles/compose_stack/ Unraid: sync compose + render .env + provision DB + up + (deploy.yml / remove.yml, picked by the entry's state:) + roles/lxc_app/ Proxmox: install natively + render config + systemd + roles/k3s_app/ k3s: render app manifests + Secret + provision DB + (deploy.yml / remove.yml, picked by the entry's state:) + roles/pve_backup/ Proxmox node: NAS backup storage + vzdump schedule + roles/k3s_node/ k3s Pi: cgroups/swap prep + install server or agent + roles/k3s_monitoring/ k3s control plane: lean kube-prometheus-stack via HelmChart CR + roles/k3s_metallb/ k3s control plane: MetalLB (LoadBalancer IPs) via HelmChart CR + roles/k3s_postgres/ k3s control plane: shared Postgres (CloudNativePG) via HelmChart CR + roles/k3s_cert_manager/ k3s control plane: cert-manager + ClusterIssuer via HelmChart CR + roles/unattended_upgrades/ any apt host: hands-off patching, no auto-reboot + roles/k3s_maintenance/ k3s Pi: drain + reboot-if-required + uncordon, one at a time + playbooks/ + deploy.yml everything (Unraid + Proxmox — not k3s, see below) + unraid.yml just Unraid + proxmox.yml just the Proxmox guests + pve_host.yml just the Proxmox node (backups) + k3s.yml bootstraps the k3s cluster (manual-only, not in deploy.yml) + k3s_maintenance.yml rolls pending reboots across it, serial: 1 (manual-only) + terraform/ Proxmox provider, pg state backend, LXC module calls +.forgejo/workflows/ CI/CD (needs a self-hosted runner, see below) +docs/vault-secrets.md Vault path/key reference +docs/authentik-migration.md hand-run cutover: Authentik from Unraid to k3s +``` + +Terraform's scope stops at the guest — it creates the LXC and nothing more, +because the Proxmox provider has no way to install an application. Ansible +picks up from there, which is why it's the common tool and Terraform is +Proxmox-only. + +## How config is layered + +An app's settings are split by whether they survive a change of platform: + +- `src//common/vars.yml` — version, ports, domain, database name/user. +- `src//ansible//vars.yml` — the rest. Data paths differ + (`/mnt/user/appdata/forgejo` vs `/var/lib/forgejo`), and so does how the + database is reached (the `shared-postgres` Docker network alias only exists + on Unraid). + +Both roles merge them the same way: common, then platform overrides, then +Vault secrets last. Nothing is restated between platforms. + +## How a deploy works + +**Unraid.** For each stack in a host's `stacks:` list, `compose_stack`: + +1. Copies `src//ansible/unraid/docker-compose.yml` to the host's + Compose Manager project folder, unmodified — it's static and only + references `${VAR}`. +2. Merges the two `vars.yml` layers with the stack's Vault KV path and + renders the project's `.env`. Nothing secret ever touches git. +3. If the stack declares a `db:` block, ensures its database/role exist on + the shared Postgres instance (idempotent — safe to add new apps to the + shared service over time without touching existing data). +4. Copies `icon.png`, if the stack ships one, to both the project folder and + `/mnt/user/appdata/icons/` (see "Icons on the Unraid Docker page" below). +5. Runs `docker compose up -d` for the stack. + +**Proxmox.** `terraform apply` creates the LXC first. Then, for each app in a +guest's `apps:` list, `lxc_app` does the same config merge and database +provisioning, runs that app's `ansible/proxmox/install.yml`, and manages its +systemd unit instead of a Compose project. + +**k3s.** For each app in `k3s_apps:` (`group_vars/k3s_cluster.yml`, not +host_vars — an app belongs to the cluster, not a node), `k3s_app`: + +1. Does the same two-layer config merge, using + `src//ansible/kubernetes/vars.yml` as the platform layer. +2. Provisions the app's database/role on the CloudNativePG cluster if it + declares a `db:` block — the same `community.postgresql` tasks the other + two platforms use, pointed at MetalLB's LoadBalancer address for the CNPG + primary. +3. Renders the app's Vault path into a Kubernetes `Secret` (plus the + `Namespace`), keys passed through verbatim. +4. Renders every `*.yaml.j2` the app ships into + `/var/lib/rancher/k3s/server/manifests/`, where k3s's own deploy and helm + controllers pick them up. + +Nothing here talks to the Kubernetes API — no `helm`, no kubeconfig, no +`kubernetes.core`. Ansible writes files; k3s reconciles them. The consequence +worth remembering: **a green run means the manifests landed, not that the +workload came up.** Check with `kubectl -n get pods`. + +Secrets land on disk in the rendered `.env` (root-readable, on the Unraid +box) or, on k3s, in a 0600 manifest and then in etcd as an ordinary +Kubernetes Secret — this is "fetch from Vault at deploy time," not a +zero-secrets-at-rest model. Fine for a homelab; revisit if that changes. + +## Icons on the Unraid Docker page + +There are two icons per stack, and they come from completely different places. +Commit **one** `src//ansible/unraid/icon.png` and the deploy feeds both. +A stack running more than one container can commit +`icon-.png` alongside it for the extra services — see "More than one +container" below. + +### The stack row + +Compose Manager draws the collapsible header row for the whole project, and +serves its icon straight off disk from `icon.png` in the project folder — +`/boot/config/plugins/compose.manager/projects//icon.png`, the same +directory the compose file and rendered `.env` land in. No label, no +template, no URL: the file is there or the row has no icon. `.jpg`, `.gif`, +`.svg` and an extensionless `icon` also work, since the plugin serves the file +itself rather than handing a path to the webgui — but stick to PNG so the same +file can serve the labels below. + +This is a feature of **Compose Manager Plus** (`mstrhakr/compose_plugin`), the +maintained fork. The original Docker Compose Manager plugin (`dcflachs`, +archived April 2026) has no icon support at all — on that one the copied file +is simply ignored, so the deploy is safe either way, you just won't see it. + +### The containers under it + +Unraid's Docker page gets each *container's* icon, WebUI link and console +shell out of the dockerMan template that created it, under +`/boot/config/plugins/dockerMan/templates-user/`. A Compose stack never +creates one, which is why compose-deployed containers show a question mark and +no WebUI entry. Unraid 6.10+ falls back to Docker labels when there's no +template, so each service sets them itself: + +```yaml +labels: + net.unraid.docker.icon: ${STACK_ICON} + net.unraid.docker.webui: "http://[IP]:${FORGEJO_HTTP_PORT}/" + net.unraid.docker.shell: sh +``` + +They're per **service**, not per stack — a stack with two containers labels +each one separately, and can point them at different icons. + +- **icon** — `${STACK_ICON}` is set by the role, not by hand. The same + committed `icon.png` is copied a second time to + `/mnt/user/appdata/icons/.png` and the label points there; ship no + `icon.png` and the variable renders empty, leaving the placeholder. To use a + hosted image instead, set `STACK_ICON` in the app's + `ansible/unraid/vars.yml` — it's merged over the role's value. + + A second copy rather than reusing the project-folder one because this label + is a path the *webgui* resolves on every Docker page load, and that path + shouldn't run through the flash drive. + + **PNG only here** — unlike the stack row, this goes through the webgui's own + icon handling, where SVG renders as the question-mark fallback and WebP + renders nothing at all. A remote URL is fetched when the page renders, so an + unreachable host breaks the icon — the reason a committed file is the + default. + +- **webui** — `[IP]` is substituted with the host's address. The port must be + the *published* one, so interpolate the stack's own port variable rather + than the container port. Omit the label entirely for something with no web + interface (as `shared/postgres` does) — an empty value still draws a WebUI + entry that goes nowhere. + +- **shell** — `sh` or `bash`, whichever the image actually has. Alpine-based + images (Forgejo) need `sh`; Debian-based ones (Postgres, Shelfarr) can take + `bash`. + +One caveat: editing a compose container's labels through the Unraid UI writes +a template for it, and the template then wins over the labels until you clear +it. Change these in the compose file and redeploy, not in the webgui. + +### More than one container + +`${STACK_ICON}` is a single value — the stack's `icon.png` — so a stack with +two services would otherwise have to give both the same image. Since the +labels are per-service, commit `icon-.png` next to `icon.png` for +each additional container, named for the **compose service**: + +``` +src/arr/ansible/unraid/ + icon.png → ${STACK_ICON} → /mnt/user/appdata/icons/arr.png + icon-prowlarr.png → ${STACK_ICON_PROWLARR} → /mnt/user/appdata/icons/arr-prowlarr.png +``` + +The variable is the service name uppercased with `-` folded to `_` +(`icon-shelfarr-libation.png` → `${STACK_ICON_SHELFARR_LIBATION}`). These get +the appdata copy only — there is one Compose Manager stack row and `icon.png` +already has it — and, like `STACK_ICON`, each can be overridden with a hosted +URL by setting the same name in the app's `ansible/unraid/vars.yml`. + +The variable only exists when the file does. Interpolating +`${STACK_ICON_FOO}` with no `icon-foo.png` committed gets Compose's +unset-variable warning and the question-mark placeholder — so if the icon is +meant to be a URL, set it in `vars.yml` rather than relying on the file. + +## Backups (Proxmox) + +Proxmox guests keep **all** their state on their own rootfs. Nothing is +bind-mounted in from the NAS, which is the opposite of the obvious instinct — +the reason is that `vzdump` deliberately excludes bind mounts, so a +bind-mounted repository directory would be the one thing missing from the +backup that was made to protect it. Keeping everything local means one archive +is a complete copy of the guest's filesystem. + +`playbooks/pve_host.yml` (role: `pve_backup`) configures the off-box copy on +the node: an NFS storage pointing at the NAS, and a scheduled `vzdump` job +writing to it. Retention is the storage's `prune-backups` settings. Point it at +a Proxmox Backup Server instead by setting `pve_backup_storage_type: pbs` in +`host_vars/.yml`. + +**Data that lives outside a container is that app's problem, not vzdump's.** +Forgejo's database is on the shared Postgres over on Unraid, so restoring its +container alone would give back every repository with no issues, pull requests, +users or permissions. The fix is a `pg_dump` on a systemd timer *inside* the +container, writing into its own filesystem — which puts the dump inside the +same archive as the repositories it belongs to. That only works if it finishes +before the backup window, so the two schedules are a pair: + +| | Set in | Default | +|---|---|---| +| Forgejo database dump | `src/forgejo/ansible/proxmox/vars.yml` → `FORGEJO_DB_DUMP_ONCALENDAR` | 01:30 | +| vzdump job | `inventory/host_vars/pve.yml` → `pve_backup_schedule` | 02:00 | + +Any future app with external state should follow the same pattern. Restore +steps are in `src/forgejo/ansible/proxmox/README.md`. + +None of this covers the Unraid side, which has its own backup arrangements +outside this repo. + +## Prerequisites + +### Both platforms + +- Ansible control node needs to be Linux — **on Windows, run it from WSL2**, + not natively. +- If this repo is checked out on a Windows mount (e.g. under `/mnt/c/...`, + which is where a Seafile-synced folder ends up) rather than the native WSL + filesystem, Ansible will warn that `ansible.cfg` is being ignored because + the directory looks world-writable — that's DrvFs reporting `777` on + everything, not a real permissions problem. Either `export + ANSIBLE_CONFIG="$(pwd)/ansible.cfg"` before running (from the + `build/config/ansible/` dir), or fix it permanently by adding to + `/etc/wsl.conf` inside WSL: + ```ini + [automount] + options = "metadata,umask=22,fmask=11" + ``` + then `wsl --shutdown` from PowerShell and reopen the shell. +- `pip install ansible hvac psycopg2-binary` +- `ansible-galaxy collection install -r build/config/ansible/requirements.yml` +- A reachable Vault instance, with secrets populated per + `docs/vault-secrets.md`. + +### Unraid + +- Root SSH enabled on the Unraid box(es) (Settings → Management Access). +- Compose Manager (Docker Compose plugin) installed on Unraid — already + done. +- **Python3 on each Unraid host itself**, not just the controller: the + `compose_stack` role's tasks (`file`, `copy`, `template`, + `docker_compose_v2`) run on the target, and Ansible modules — unlike + `raw`/ad hoc shell — need a Python interpreter there to execute at all. + Stock Unraid doesn't ship one. Installing it by hand into the running + system doesn't survive a reboot, same reason as the SSH-key gotcha above — + Unraid rebuilds its root filesystem from the flash drive on every boot. + Install the **NerdTools** plugin (Community Applications → search + "NerdTools", formerly "NerdPack") and enable `python3` in its package + list — it reinstalls whatever you've selected on every boot itself, same + mechanism Compose Manager already relies on. `ansible.cfg` already sets + `interpreter_python = auto_silent`, so Ansible finds it automatically + wherever NerdTools puts it; no path to hardcode. Verify after installing: + ```sh + ansible unraid_servers -m ping + ``` + +### Proxmox + +- Terraform CLI (>= 1.6) on whatever runs `terraform apply`. +- A Proxmox API token with rights to create containers, stored in Vault at + `homelab/ci/proxmox` (see `docs/vault-secrets.md`). Both the Terraform + provider and Ansible's dynamic inventory authenticate with it. +- A `terraform_state` database and `terraform` role on the **CloudNativePG + cluster on k3s** (192.168.50.81), for `backend "pg"` — not on either shared + Postgres this repo deploys, which would be circular. See "Bootstrapping" + below. +- Container templates downloaded on the Proxmox node for whatever OS the + LXCs are built from. +- An NFS export on the NAS for backup archives, reachable from the Proxmox + node — set it in `inventory/host_vars/pve.yml`. See "Backups (Proxmox)". +- Root SSH from the Ansible controller to the Proxmox node itself, not just + the guests: `playbooks/pve_host.yml` configures the node over SSH. +- Unlike Unraid, nothing special is needed *inside* the guests: they're + ordinary Linux containers with Python already present, so Ansible works + without the NerdTools workaround above. + +### K3s (Raspberry Pi) + +- 4 Raspberry Pis running Ubuntu Server (64-bit), already imaged and on the + network at the addresses in `inventory/hosts.yml` → `k3s_cluster`. +- An `ansible` user on each Pi with **NOPASSWD sudo** and this repo's k3s SSH + key installed as an authorized key — see "SSH access" below. Not automated + by this repo; create it by hand (or via cloud-init at image time) before the + first run of `playbooks/k3s.yml`. +- `homelab/k3s-homelab-utils` → `K3S_TOKEN` set in Vault *before* the first + run — see `docs/vault-secrets.md`. Both the server and every agent read + this same fixed value rather than one generating it and handing it to the + other, which is what makes a rebuild reproducible. +- Nothing else special: `python3` ships with Ubuntu Server, and the k3s + install script (`get.k3s.io`) handles containerd, the systemd unit, and + everything else that isn't Pi-specific. The one Pi-specific thing — + ensuring the memory cgroup controller is on — is handled by + `roles/k3s_node` itself, not a prerequisite here. + +#### Keeping it patched + +`playbooks/k3s.yml` also applies `roles/unattended_upgrades` to every node — +apt updates install themselves on their own daily schedule, no login +required. It sets `Unattended-Upgrade::Automatic-Reboot "false"` though, so a +kernel or containerd update that needs a reboot to take effect just sits +applied-but-inactive until one happens; blindly auto-rebooting a k3s node +takes its pods down with no warning. + +`playbooks/k3s_maintenance.yml` is the other half: it checks +`/var/run/reboot-required` on each node and, only where it's set, cordons and +drains the node, reboots it, waits for it to report `Ready` again, then +uncordons it — one node at a time (`serial: 1`), so patching the cluster never +means dropping every workload at once. Manual-only for now, same as +`playbooks/k3s.yml`: + +```sh +ansible-playbook playbooks/k3s_maintenance.yml +``` + +Eventually this is meant to run on a schedule from a self-hosted Forgejo +Actions runner rather than by hand — not wired up yet, see the header comment +in the playbook. + +#### Metrics (Prometheus) + +`playbooks/k3s.yml` also deploys `roles/k3s_monitoring`: a lean +kube-prometheus-stack (Prometheus + prometheus-operator + node-exporter + +kube-state-metrics — no Grafana, no Alertmanager) so tools like OpenLens can +show node/pod metrics. It's installed as a `HelmChart` custom resource +dropped into k3s's own auto-deploying manifests directory +(`/var/lib/rancher/k3s/server/manifests/`) rather than run through a `helm` +binary — k3s's bundled helm-controller reconciles it the same way it +installs its own Traefik and ServiceLB, so this needs no extra Ansible +collection or kubeconfig on the controller. See +`roles/k3s_monitoring/defaults/main.yml` for the resource sizing (tuned for +a Raspberry Pi 4, not a real node) and why the control-plane component +scrapers (`kubeControllerManager`, `kubeScheduler`, `kubeProxy`, `kubeEtcd`) +are disabled — k3s doesn't expose those the way kube-prometheus-stack +expects, so leaving them on just produces permanently-"down" targets. + +To point OpenLens at it: open the cluster's Settings → Metrics, set +Prometheus to "Auto detect" or explicitly to the `Operator` provider — it +should find the `kube-prometheus-stack-prometheus` service in the +`monitoring` namespace via the API server proxy, the same path OpenLens +already uses to reach the cluster, so nothing needs to be exposed outside +it. + +#### LoadBalancer IPs (MetalLB) + +`playbooks/k3s.yml` also deploys `roles/k3s_metallb`: MetalLB in L2 mode, +handing out real LAN IPs to `type: LoadBalancer` Services instead of the +`ClusterIP`-only world k3s would otherwise leave homelab-utils apps in. It's +installed the same way as monitoring — a `HelmChart` CR for k3s's +helm-controller to reconcile — plus a plain `IPAddressPool`/`L2Advertisement` +manifest dropped in the same directory; k3s's deploy controller applies both +kinds of file and retries the config manifest until the HelmChart's CRDs +exist. + +MetalLB replaces k3s's bundled ServiceLB (Klipper), it doesn't run alongside +it — both would otherwise compete to satisfy the same Services. That's why +`inventory/group_vars/k3s_cluster.yml` sets `k3s_server_extra_args: +['--disable=servicelb']` — a separate, server-only var from `k3s_extra_args`, +since `k3s agent` doesn't understand `--disable` and would fail to start if +it were passed there too. The IP pool itself is +`k3s_metallb_address_range` in the same file — a range on the cluster's LAN +(`192.168.50.0/24`) that DHCP and every static assignment in +`inventory/hosts.yml` steer clear of. There's no built-in default: the role +fails fast if it's still empty. + +Enabling `--disable=servicelb` on an already-running cluster doesn't need a +separate step: `roles/k3s_node` notices its exec line has changed and +reinstalls (restarting just the `k3s` service, not the node) the next time +`playbooks/k3s.yml` runs — see `CLAUDE.md` → "Key decisions". + +#### Postgres (CloudNativePG) + +`playbooks/k3s.yml` also deploys `roles/k3s_postgres`: CloudNativePG (CNPG), +a Postgres operator, installed the same way as monitoring and MetalLB — a +`HelmChart` CR for k3s's helm-controller to reconcile — plus a plain +`Cluster` CR (the actual database) and a `Secret` (superuser credentials), +dropped in the same auto-deploying manifests directory. This is a third +platform for `src/shared/postgres/`, alongside the Unraid Compose stack and +the Proxmox placeholder: `roles/k3s_postgres` layers +`src/shared/postgres/common/vars.yml` with +`src/shared/postgres/ansible/kubernetes/vars.yml` the same way +`compose_stack`/`lxc_app` layer an app's config, and pulls the superuser +password from the same `homelab/shared/postgres` Vault path those platforms +already use (see `docs/vault-secrets.md`) — a separate physical instance, +not a shared login across platforms, just the same identity. + +1 primary + 1 replica, kept off the control-plane Pi via `nodeAffinity` (see +`roles/k3s_postgres/templates/postgres-cluster.yaml.j2`) — CNPG's own pod +anti-affinity then spreads the two across the 3 worker Pis. Storage is k3s's +default `local-path` StorageClass; that's node-local, but resilience here +comes from CNPG's own streaming replication between instances, not from +shared storage, so losing one instance's disk doesn't lose the data as long +as the other instance is up. HA replication only for now — no +`ScheduledBackup` to the NAS or anywhere else yet. + +To connect: `kubectl -n shared-postgres get svc` lists four Services. Three +are CNPG's own, all `ClusterIP` — `shared-postgres-rw` (the current primary), +`-ro` (replicas only) and `-r` (any instance). An app running on the cluster +uses the first of those by DNS and needs nothing else: + +``` +shared-postgres-rw.shared-postgres.svc.cluster.local:5432 +``` + +The fourth, `shared-postgres-lb`, is a `LoadBalancer` on +`k3s_postgres_loadbalancer_ip` (`inventory/group_vars/k3s_cluster.yml` — +inside the MetalLB pool above), which is how the cluster is reached from +outside it: + +```sh +psql -h 192.168.50.81 -U postgres # password: homelab/shared/postgres +``` + +It's declared in the `Cluster` CR as a CNPG *managed service* rather than as +a Service manifest of our own, so the operator keeps its selector pointed at +whichever instance is currently primary — a hand-written Service would need +re-pointing by hand after a failover. Credentials are the +`shared-postgres-superuser-vault` Secret Ansible renders; CNPG separately +auto-creates `shared-postgres-app` for the default `app` database it +bootstraps, which nothing uses yet. + +The `-vault` suffix keeps that Secret clear of `-superuser`, the +name CNPG uses for the superuser secret it generates itself. Name our own +object that and the operator treats it as one it already authored, so a +rotated password reaches the Secret and never reaches the database — the +Secret reads correctly, `psql` from the LAN keeps failing authentication, and +only `select rolpassword is null from pg_authid where rolname='postgres'` +inside the pod shows the disagreement. + +That LoadBalancer is what makes the controller-reachable `host:port` the +`community.postgresql` tasks need — the same ones `compose_stack` and +`lxc_app` use to create an app's database and role — available here too, and +`roles/k3s_app` now uses exactly those tasks for any app declaring a `db:` +block. + +Note the deliberate asymmetry, which reads like a mistake until you know why: +provisioning uses `192.168.50.81` (the LoadBalancer) because it runs on the +Ansible controller, which is off-cluster and can't route to a ClusterIP; +apps use `shared-postgres-rw.shared-postgres.svc.cluster.local` because +sending pod traffic out to the LAN and back would put MetalLB's L2 speaker in +the path of every query for nothing. Two addresses, one database, each +correct for its caller. + +Without `kubectl` to hand, or if MetalLB is having a bad day, the in-cluster +routes still work: + +```sh +kubectl -n shared-postgres port-forward svc/shared-postgres-rw 5432:5432 +kubectl -n shared-postgres exec -it shared-postgres-1 -- psql -U postgres +``` + +**No backups.** HA replication only — a streaming replica on another Pi, +which covers a dead SD card and not a dropped table, a bad migration, or a +mistyped `DROP`. This mattered less when the cluster held nothing; it matters +now that Authentik's database lives here rather than on Unraid. A CNPG +`ScheduledBackup` to the NAS is the obvious next piece of work and isn't +built yet. + +#### TLS (cert-manager) + +`playbooks/k3s.yml` also applies `roles/k3s_cert_manager`: cert-manager plus +one `ClusterIssuer` named `letsencrypt`, both via the usual `HelmChart` CR + +plain manifest pair. An app's Ingress opts in with a single annotation — + +```yaml +annotations: + cert-manager.io/cluster-issuer: letsencrypt +``` + +— and cert-manager creates and renews the certificate on its own. + +Two values have no default and the role refuses to run without them, both in +`inventory/group_vars/k3s_cluster.yml`: + +- `k3s_cert_manager_acme_email` — an ACME account is registered against it. +- `k3s_cert_manager_solver` — the DNS-01 stanza for your DNS provider, + rendered into the issuer as-is. A commented Cloudflare example is in that + file; for anything else take the stanza from + [cert-manager's docs](https://cert-manager.io/docs/configuration/acme/dns01/). + Its API token goes in Vault at `homelab/k3s-cert-manager`. + +DNS-01 rather than HTTP-01 because HTTP-01 needs Let's Encrypt to reach this +cluster from the internet on port 80, which it can't. That has a useful +consequence: a certificate can be issued **before** DNS points at the +cluster, so a migration's TLS is settled before its cutover rather than +after. + +While working out a solver, point `k3s_cert_manager_acme_server` at Let's +Encrypt staging — production allows five failed validations per hostname per +hour, and exhausting it means waiting, not retrying. Check it registered: + +```sh +kubectl get clusterissuer letsencrypt -o jsonpath='{.status.conditions[*].message}' +``` + +#### Dashboard (Traefik) + +Traefik is the one chart here this repo doesn't install — k3s installs it +itself. `roles/k3s_traefik` only adjusts it, through a `HelmChartConfig` +merged over k3s's own `HelmChart`. Editing +`/var/lib/rancher/k3s/server/manifests/traefik.yaml` on the node instead +works until the next server restart rewrites it. + +A stock k3s answers **404** on the dashboard, which looks like a broken +install and isn't: Traefik still builds the dashboard, but the Traefik chart +stopped creating the router that reaches it in v28. The role puts that router +back on Traefik's internal `traefik` entrypoint, which isn't published on the +Service — so it's reachable by port-forward and nothing else: + +```sh +kubectl -n kube-system port-forward deploy/traefik 9000:9000 +# then http://127.0.0.1:9000/dashboard/ — the trailing slash is required +``` + +Two things to check if that still 404s. `curl -i http://127.0.0.1:9000/ping` +returning 200 means the port-forward is fine and only the router is missing +(so the role hasn't run, or its `HelmChartConfig` hasn't reconciled yet — +`kubectl -n kube-system get ingressroute` should list `traefik-dashboard`). +And forward to `deploy/traefik`, not `svc/traefik`: the Service only publishes +80/443, so going through it lands you on the `web` entrypoint, where an +unmatched request also returns 404. + +**Publishing it on a hostname** is one commented line in +`inventory/group_vars/k3s_cluster.yml`: + +```yaml +k3s_traefik_dashboard_host: traefik.turtlesystems.uk +``` + +That renders an `IngressRoute` on `websecure` with a cert-manager certificate +and an Authentik forward-auth middleware in front. `k3s_traefik_dashboard_auth_address` +must be set too — the role refuses to run otherwise rather than publishing an +unauthenticated view of every router, service and middleware on the cluster. +It's already set in that file, so in practice this is the single line above +plus a DNS record pointing at Traefik's MetalLB address +(`kubectl -n kube-system get svc traefik`). + +On the Authentik side, first: a Proxy Provider in **forward auth (domain +level)** mode, assigned to an application, added to the embedded outpost. +Domain level rather than single-application because the latter needs +`/outpost.goauthentik.io/` routed to Authentik on the dashboard's own +hostname, which from `kube-system` is a cross-namespace service reference +Traefik rejects by default. + +An `IngressRoute` rather than an `Ingress`, unlike every app here, because +the dashboard is served by Traefik's internal `api@internal` and has no +Kubernetes Service for an `Ingress` to point at. That's also why its +certificate is an explicit `Certificate` resource — cert-manager watches +`Ingress`, not `IngressRoute`. + +Clearing the hostname again is a real teardown: the role deletes the +manifest, and k3s's deploy controller garbage-collects the route, middleware +and certificate it created. + +The port-forward stays enabled alongside the published route on purpose. Once +the dashboard is behind Authentik it depends on Authentik, which depends on +CNPG, which depends on a healthy cluster — the things you'd open the +dashboard to diagnose. Port-forward talks to the pod and traverses none of +it, so it's the break-glass path, not a leftover. + +#### Apps on the cluster + +Apps go in `k3s_apps:` in `inventory/group_vars/k3s_cluster.yml` — the k3s +equivalent of a host's `stacks:`/`apps:` list, in group_vars because an app +is deployed to the cluster rather than to a node. Same entry shape as the +other platforms (name, src, vault_path, optional `db:`, optional `state:`). + +```sh +ansible-playbook playbooks/k3s.yml --tags apps +ansible-playbook playbooks/k3s.yml --tags apps -e only_apps=authentik +``` + +Ingress goes through the Traefik k3s already bundles, on the MetalLB address +its Service holds (`kubectl -n kube-system get svc traefik`) — so adding an +app costs a DNS record pointed at that one address, not a pool IP each. + +Removal is `state: absent` on the entry, same tombstone rule as Unraid: keep +the entry, don't delete it, or nothing is torn down and the workload keeps +running unmanaged. Unlike Unraid it's a single pass — deleting the manifests +*is* the teardown, because k3s's deploy controller garbage-collects what each +file created — so `src//` can go in the same commit. The database, PVCs +and Vault path deliberately survive. + +**Authentik** is currently the only app, and it got here by migrating off a +hand-made Unraid container rather than being deployed fresh. If you're +repeating that for something else, `docs/authentik-migration.md` is the +worked procedure; the part worth knowing up front is that the deployed +version must match the version a restored database was dumped from, because +Authentik runs its migrations on startup and they don't go backwards. + +## SSH access + +Ansible connects as `ansible_user: root` (set in +`build/config/ansible/inventory/hosts.yml`) over SSH. Password auth would mean either an interactive prompt every run — +which doesn't work from CI at all — or a plaintext password somewhere, so +use a dedicated key pair instead: + +```sh +ssh-keygen -t ed25519 -f ~/.ssh/unraid_ansible -C "ansible@unraid-iac" -N "" +``` + +Install the public half as an authorized key on **every** Unraid host this +repo targets. Don't just `ssh-copy-id` it into `~/.ssh/authorized_keys` — +Unraid boots from the flash drive and `/root` lives on a RAM-backed overlay, +so anything written there is gone on the next reboot. Persist it through +`/boot/config/ssh/root.authorized_keys` instead, which Unraid copies into +place at every boot: + +```sh +cat ~/.ssh/unraid_ansible.pub | ssh root@ \ + 'cat >> /boot/config/ssh/root.authorized_keys' +``` + +For manual/bootstrap runs, load the key into `ssh-agent` and Ansible picks it +up automatically — no config file changes needed: + +```sh +eval "$(ssh-agent -s)" +ssh-add ~/.ssh/unraid_ansible +``` + +For CI, rather than adding yet another place secrets live, the private key +is stored in Vault (`homelab/ci/ssh`, see `docs/vault-secrets.md`) and the +workflow fetches it at the start of each run: + +```sh +vault kv put kv/homelab/ci/ssh PRIVATE_KEY=@~/.ssh/unraid_ansible +``` + +**K3s Pis** work the same way but as a non-root `ansible` user, not root, and +with their own key pair (`~/.ssh/k3s_ansible`, Vault path +`homelab/ci/ssh-k3s`) — see `docs/vault-secrets.md` for the full commands. +Three differences from the Unraid steps above. The public key goes in the +ordinary `~/.ssh/authorized_keys` for the `ansible` user (Ubuntu's root +filesystem isn't rebuilt from flash on every boot, so there's no persistence +quirk to work around), and that user needs NOPASSWD sudo configured in +`/etc/sudoers.d/` — `playbooks/k3s.yml` installs k3s via `become: true`. + +And the private half doesn't need loading into `ssh-agent` at all, even for a +manual run: `playbooks/k3s_ssh_key.yml` fetches it from Vault and writes it to +`local/k3s/homelab-utils.key`, and both `playbooks/k3s.yml` and +`playbooks/k3s_maintenance.yml` import that as their first play. So a k3s run +needs `VAULT_ADDR`/`VAULT_TOKEN` in the environment and nothing else — the +same "Vault is the only place secrets live" arrangement CI already uses for +the Unraid key, rather than a second setup step to remember. To fall back to a +local copy (Vault down, say), skip the fetch **and** point at the key — +overriding the path alone isn't enough, since the fetch play would still fail +before anything else ran: + +```sh +ansible-playbook playbooks/k3s.yml \ + -e k3s_ssh_key_fetch=false \ + -e ansible_ssh_private_key_file=~/.ssh/k3s_ansible +``` + +## Bootstrapping (chicken-and-egg on the very first deploy) + +CI/CD runs on Forgejo Actions — but Forgejo doesn't exist yet on the first +run, and Actions needs a self-hosted runner besides. So the first deploy is +manual: + +1. Fill in `build/config/ansible/inventory/hosts.yml` with nas2's real + address. +2. Set up the SSH key pair per "SSH access" above and install the public key + on nas2 (and nas1, once you migrate anything to it). +3. `vault kv put` the secrets in `docs/vault-secrets.md` for + `homelab/shared/postgres`, `homelab/forgejo`, and `homelab/ci/ssh`. +4. From WSL, with `VAULT_ADDR` / `VAULT_TOKEN` exported (`vault login` first) + and the key loaded in `ssh-agent`: + ```sh + cd build/config/ansible + ansible-playbook playbooks/unraid.yml --limit nas2 + ``` + This brings up shared Postgres, provisions the `forgejo` database/role, + and brings up Forgejo. Use `unraid.yml` rather than `deploy.yml` at this + point — no Proxmox guests exist yet, so the Proxmox play has nothing to + target. +5. Log into Forgejo, create this repo there, push it. +6. Register a self-hosted Forgejo Actions runner reachable to Vault and to + both Unraid hosts (labelled `unraid-deploy` — see + `.forgejo/workflows/deploy.yml`). Give it an AppRole (`VAULT_ROLE_ID` / + `VAULT_SECRET_ID`) scoped to read the `kv/homelab/*` paths it needs + (including `homelab/ci/ssh`), stored as Forgejo Actions secrets alongside + `VAULT_ADDR`. +7. From then on, pushes to `main` touching `src/**` or `build/config/ansible/**` + deploy automatically — and only the stack(s) whose `src//` folder + actually changed (see "Deploying one platform, one host, one stack" + below); a change under `build/config/ansible/` still triggers a full + deploy of every stack on every host, since that's a change to how *all* of + them get deployed. + +### Bringing Proxmox online later + +The shared Postgres and its LXC are the first thing to stand up here, and the +ordering is fiddly enough to have its own runbook: **`docs/postgres-proxmox.md`**. +The short version, and the reason it isn't just "run Terraform": + +Terraform stores its state in Postgres, so provisioning the Postgres LXC with +Terraform would be circular. It's broken by keeping state on the +CloudNativePG cluster on the k3s Pis instead — something this configuration +has no hand in building — which means `terraform apply` now depends on the +cluster being up. That's a cross-platform dependency the rest of the repo +avoids, and it's worth it because the alternative is a local-state-then-migrate +dance that has to be got right exactly once. + +After that, `docs/postgres-proxmox.md` walks through the template download, +`terraform apply -target=module.postgres`, checking the ZFS replication job +actually exists, installing Postgres with +`ansible-playbook playbooks/proxmox.yml -e only_stacks=postgres`, and +`playbooks/pve_host.yml` for node backups. It also covers failing over between +the two nodes and repointing clients afterwards. + +Forgejo comes after all of that, on **turtle-proxmox-02** — its database is +the shared Postgres the runbook above creates, so none of it works until that +one is finished. `docs/forgejo-proxmox.md` walks through the extra Vault key +the native install needs, `terraform apply -target=module.forgejo`, +`ansible-playbook playbooks/proxmox.yml -e only_stacks=forgejo`, and the parts +Ansible deliberately does not do — moving repository data, and the DNS/SSH-port +cutover. + +## Deploying one platform, one host, one stack + +`playbooks/deploy.yml` runs everything; `playbooks/unraid.yml`, +`playbooks/proxmox.yml` and `playbooks/pve_host.yml` do one slice each +(Unraid stacks, Proxmox guests, and the Proxmox node's backup config +respectively). Within any of them, `--limit ` +narrows by host and `-e only_stacks=[,...]` narrows by stack or +app (name = the `name:` in the host's `stacks:`/`apps:` list, e.g. `forgejo` +or `postgres` — not the `src:` path): + +```sh +# everything, both platforms +ansible-playbook playbooks/deploy.yml + +# everything declared for nas2 +ansible-playbook playbooks/unraid.yml --limit nas2 + +# just forgejo on nas2 +ansible-playbook playbooks/unraid.yml --limit nas2 -e only_stacks=forgejo + +# forgejo and its database, skipping any other stack on nas2 +ansible-playbook playbooks/unraid.yml --limit nas2 -e only_stacks=forgejo,postgres +``` + +Note `only_stacks` doesn't resolve dependencies for you — if you limit to +`forgejo` before `postgres` has ever been deployed, the DB provisioning step +will fail because there's nothing to connect to. CI doesn't hit this: it +computes `only_stacks` from which `src/**` paths actually changed in the +push, so an unrelated app's push never touches Postgres or Forgejo. + +`deploy.yml` runs Unraid before Proxmox by convention rather than necessity — +the two platforms no longer depend on each other. Proxmox-side apps provision +their databases against the Proxmox shared Postgres (192.168.50.54), and +Terraform keeps its state on the CloudNativePG cluster on k3s; neither goes +through Unraid. The node's backup config goes last, since a vzdump job pinned +to specific VMIDs needs those guests to exist. Note that `only_stacks` has no +meaning for `pve_host.yml` — it configures a node, not an app, and ignores it. + +`playbooks/k3s.yml` is separate from all of the above — it isn't included in +`deploy.yml` and doesn't take `only_stacks`, because a cluster service there +isn't a `stacks:` entry to filter down to. It has two selectors of its own +instead: `--limit` narrows *which nodes*, and `--tags` narrows *which piece*. + +```sh +# the whole cluster: nodes, then the cluster services, then the kubeconfig +ansible-playbook playbooks/k3s.yml + +# re-converge one worker, e.g. after reimaging it +ansible-playbook playbooks/k3s.yml --limit k3s-node-02 + +# just one cluster service, no node install across 4 Pis +ansible-playbook playbooks/k3s.yml --tags postgres + +# every cluster service, still no node install +ansible-playbook playbooks/k3s.yml --tags services +``` + +The tags, one per play (see the playbook's header comment): `nodes` (both +node plays), `metallb`, `monitoring`, `postgres`, `cert-manager`, `traefik`, +`services` (all five of those together), `apps`, `upgrades`, `kubeconfig`. +`services` and `apps` don't imply each other, so "everything except +reinstalling k3s" is `--tags services,apps`. The SSH-key fetch play is tagged +`always` rather than named, so it survives every filter — it's what any of +the others connect with, not something you'd pick. + +Deploying a single service this way skips the `kubeconfig` play too, so +`local/`'s copy isn't refreshed. That only matters on a cluster rebuild; +add `--tags postgres,kubeconfig` if you want both. + +Worth knowing what a tagged run does and doesn't tell you: every service role +just templates manifests into `/var/lib/rancher/k3s/server/manifests/` +for k3s's own controllers to reconcile, so the playbook finishing means the +files landed, not that the workload is up. Watch that separately, e.g. +`kubectl -n shared-postgres get cluster,pods -w`. + +`playbooks/k3s_maintenance.yml` is separate again — see "Keeping it patched" +above. It ignores `--limit` grouping in one sense worth knowing: `serial: 1` +still applies to whatever `--limit` narrows the run to, so limiting to two +nodes still patches them one at a time, not together. + +## Adding a new app + +1. `src//common/vars.yml` — `env_defaults:` for the config that doesn't + depend on where it runs. +2. Build the platform side(s) you need: + + **Unraid** — `src//ansible/unraid/` containing + `docker-compose.yml` (static, `${VAR}`-driven, joining `unraid_shared` + with `external: true` if it needs the shared Postgres), `vars.yml` of + overrides, and `.env.example` documenting every var with secrets blank. + Optionally an `icon.png` and the `net.unraid.docker.*` labels, so the + Docker page shows something other than a question mark. + + **Proxmox** — `src//ansible/proxmox/` containing `vars.yml`, + `install.yml`, and templates for the app's config file and systemd unit; + plus `src//terraform/` defining its LXC, called from + `build/config/terraform/main.tf`. + +3. Add its `homelab/` path to Vault (`docs/vault-secrets.md`). +4. Declare it on the target host — `stacks:` in an Unraid host's + `host_vars/.yml`, or `apps:` in a Proxmox guest's. Same entry shape + either way (name, src, vault_path, optional `db:` block). +5. Push — or run the relevant playbook manually before CI exists. +6. Leave the platform you didn't build as a README placeholder rather than + deleting the folder. + +## Removing an app (Unraid) + +**Deleting the entry from `stacks:` does not remove anything.** Ansible keeps +no record of what it deployed last run, so an app that disappears from the +list is simply never visited again — its containers keep running on the host, +now unmanaged and invisible to the playbook. This is the one place the Unraid +side doesn't behave like the Terraform side, where deleting a module call does +destroy the LXC. + +Removal is therefore an instruction, not an absence. Set `state: absent` on +the entry and leave it in place: + +```yaml +stacks: + - name: arr + src: arr + vault_path: homelab/arr + state: absent +``` + +The next run of `playbooks/unraid.yml` runs `docker compose down`, then +deletes the project folder (and with it the rendered `.env`). Because a +`host_vars/` edit is a change under `build/config/ansible/`, CI treats it as a +full converge — so pushing that change is enough to action the teardown. + +Keep the entry as a tombstone until you're sure. It's the only record that +the app was deliberately removed rather than never deployed, and flipping +`state:` back to `present` redeploys it. + +### What survives, and how to remove the rest + +Bind mounts are untouched, so `/mnt/user/appdata/` — which for a +SQLite-backed app like Shelfarr *is* the application — survives a teardown and +makes it reversible. External networks (`caddy-net`, `unraid_shared`) are left +alone too; they belong to Unraid or to another stack. Both copies of the +stack's icon *are* deleted without needing to be asked — the project-folder +one goes with the folder, and the one under `/mnt/user/appdata/icons/` is +removed explicitly. Neither is state; they're copies of a file in the repo, +and a redeploy puts them back. + +Three things are destructive enough to stay opt-in, per entry: + +| Key | Effect | +|---|---| +| `remove_volumes: true` | Also delete the project's named volumes. Bind mounts are unaffected either way. | +| `remove_images: local` | Also delete its images (`local` or `all`, as `docker compose down --rmi`). | +| `remove_database: true` | Drop the Postgres database **and** its role. Only for stacks with a `db:` block. | + +Nothing removes the app's `homelab/` path from Vault — that has its own +lifecycle, and destroying it would make the teardown irreversible. Delete it +by hand with `vault kv metadata delete` once you're finished with the app. + +### Order of operations + +`docker compose down` reads the compose file to know what it's removing, so +the teardown has to run **before** you delete `src//` or the host's +project folder. Delete those first and there's nothing left to tell Docker +what belonged to the project — you're cleaning up by hand with `docker rm` and +`docker network rm` instead. So: set `state: absent`, run the playbook, then +delete files in a second pass. + +There's no equivalent on the Proxmox side yet. `lxc_app` only ever installs, +and a symmetric teardown needs a per-app `uninstall.yml` convention that no +app implements — the role is still a skeleton. For now, removing a Proxmox app +means stopping and disabling its systemd unit by hand, or destroying the +container via Terraform. + +## Adding a host + +**Unraid:** add it under `unraid_servers` in +`build/config/ansible/inventory/hosts.yml`, then create +`build/config/ansible/inventory/host_vars/.yml` with its own `stacks:` +list — each host only runs what it's assigned. + +**Proxmox:** define the LXC as a module in `src//terraform/`, call it +from `build/config/terraform/main.tf`, and `terraform apply`. The guest +appears in inventory automatically via the API — no `hosts.yml` edit — but +it still needs a `host_vars/.yml` declaring its `apps:` list. + +**K3s:** add it as a new host under `k3s_control_plane` or `k3s_workers` (per +`inventory/hosts.yml` → `k3s_cluster`) — no `terraform apply`, the Pi has to +physically exist and have SSH access set up first (see "K3s (Raspberry Pi)" +above). No `host_vars/` entry needed beyond that: `roles/k3s_node` reads +`k3s_node_role` off the group, not the host, so which group a new Pi joins is +the only thing that decides whether it becomes another agent or a second +control-plane node — and this repo's role doesn't support the latter (single +server, no HA etcd) without changes to `k3s_node/tasks/server.yml`. diff --git a/build/config/ansible/ansible.cfg b/build/config/ansible/ansible.cfg new file mode 100644 index 0000000..927dbe1 --- /dev/null +++ b/build/config/ansible/ansible.cfg @@ -0,0 +1,18 @@ +[defaults] +# A directory, not a single file: static Unraid hosts (hosts.yml) and the +# Proxmox API-backed dynamic inventory (proxmox.yml) are both sources. +inventory = inventory/ +roles_path = roles +host_key_checking = False +retry_files_enabled = False +interpreter_python = auto_silent + +[inventory] +# `auto` must come before `yaml`: it reads the `plugin:` key in +# inventory/proxmox.yml and dispatches to the named plugin. With `yaml` +# first, that file gets parsed as a static host list instead, and `plugin:` +# is read as a group name. +enable_plugins = auto, yaml, ini, community.proxmox.proxmox + +[ssh_connection] +pipelining = True diff --git a/build/config/ansible/inventory/group_vars/all.yml b/build/config/ansible/inventory/group_vars/all.yml new file mode 100644 index 0000000..5b353b0 --- /dev/null +++ b/build/config/ansible/inventory/group_vars/all.yml @@ -0,0 +1,18 @@ +--- +# HashiCorp Vault connection. VAULT_ADDR/VAULT_TOKEN (or VAULT_ROLE_ID / +# VAULT_SECRET_ID for AppRole) are read from the environment of whatever is +# running ansible-playbook — a local shell during bootstrap, or Forgejo +# Actions secrets once CI is wired up. Never hardcode a token here. +vault_addr: "{{ lookup('env', 'VAULT_ADDR') }}" +vault_kv_mount: kv + +# AppRole auth (used by CI). Leave unset for local token-based auth +# (`vault login` + VAULT_TOKEN env var) during bootstrap. +vault_auth_method: "{{ lookup('env', 'VAULT_AUTH_METHOD') | default('token', true) }}" +vault_role_id: "{{ lookup('env', 'VAULT_ROLE_ID') | default(omit) }}" +vault_secret_id: "{{ lookup('env', 'VAULT_SECRET_ID') | default(omit) }}" + +# Where Compose Manager (the Unraid Docker Compose plugin) expects project +# folders. Verify this against your plugin version — override per-host in +# host_vars if yours differs. +compose_projects_root: /boot/config/plugins/compose.manager/projects diff --git a/build/config/ansible/inventory/group_vars/k3s_cluster.yml b/build/config/ansible/inventory/group_vars/k3s_cluster.yml new file mode 100644 index 0000000..56d9e14 --- /dev/null +++ b/build/config/ansible/inventory/group_vars/k3s_cluster.yml @@ -0,0 +1,205 @@ +--- +# Shared config for the whole k3s_cluster group (both k3s_control_plane and +# k3s_workers) — see roles/k3s_node and playbooks/k3s.yml. + +# Used as the kubeconfig context/cluster name and as the Vault path suffix +# below, so the cluster has one name in every place it shows up. +k3s_cluster_name: homelab-utils + +# Pinned, not "latest" — a rebuild months from now should produce the same +# cluster it does today, same reasoning as pinning an app's image tag in +# common/vars.yml. Bump deliberately; check the current stable release at +# https://github.com/k3s-io/k3s/releases first. +k3s_version: v1.31.4+k3s1 + +# One Vault path for the whole cluster, same "one path per stack" convention +# as everything under homelab/ — see docs/vault-secrets.md. +k3s_vault_path: "homelab/k3s-{{ k3s_cluster_name }}" + +# The SSH key this group is reached with. A different path from k3s_vault_path +# above because it's a different kind of secret: that one is cluster identity +# (the join token), this one is access to the hosts, and it lives alongside +# the Unraid key under homelab/ci/ rather than with the cluster's own data. +# Fetched by playbooks/k3s_ssh_key.yml, which both k3s playbooks import. +k3s_ssh_key_vault_path: homelab/ci/ssh-k3s + +# Where that play writes it, and what every play here then connects with — so +# a manual run needs Vault credentials in the environment and nothing else, +# rather than the key pre-loaded into ssh-agent. Override on the command line +# (-e ansible_ssh_private_key_file=~/.ssh/k3s_ansible) to use a local copy +# instead, e.g. if Vault itself is down. +# +# Under local/, which .gitignore already excludes — same as the kubeconfig +# below, and for the same reason: generated credential, never committed. Built +# from playbook_dir rather than repo_root, unlike k3s_kubeconfig_local_path: +# repo_root is a *play* var set in k3s.yml, so it only resolves inside the two +# plays that set it, whereas this is a connection var and has to resolve in +# every play in the file. +k3s_ssh_key_local_path: "{{ playbook_dir }}/../../../../local/k3s/{{ k3s_cluster_name }}.key" +ansible_ssh_private_key_file: "{{ k3s_ssh_key_local_path }}" + +# Extra INSTALL_K3S_EXEC args appended on every node, both roles. Empty by +# default. Only flags valid on *both* `k3s server` and `k3s agent` belong +# here — see k3s_server_extra_args below for server-only flags like +# `--disable`, which `k3s agent` doesn't understand and will fail to start +# with. +k3s_extra_args: [] + +# Extra INSTALL_K3S_EXEC args appended on the control-plane node only. +# --disable=servicelb turns off k3s's bundled LoadBalancer controller +# (Klipper) in favour of roles/k3s_metallb — the two would otherwise both +# try to satisfy the same LoadBalancer Services. Add e.g. '--disable=traefik' +# here too if homelab-utils apps end up wanting their own ingress instead. +# +# Changing this (or k3s_extra_args) on an already-installed cluster is picked +# up on the next `ansible-playbook playbooks/k3s.yml` run — roles/k3s_node +# tracks the exec line it last installed with and reinstalls (restarting the +# k3s/k3s-agent service, not the node) whenever it drifts from what's +# configured here. See tasks/server.yml and tasks/agent.yml. +k3s_server_extra_args: + - "--disable=servicelb" + +# MetalLB's LoadBalancer IP pool — see roles/k3s_metallb. Must be addresses +# on the LAN (192.168.50.0/24 — see inventory/hosts.yml) that nothing else, +# DHCP included, will ever hand out. Adjust to a range your DHCP scope +# doesn't cover before the first run. +k3s_metallb_address_range: "192.168.50.80-192.168.50.99" + +# The LAN address the shared Postgres primary answers on — see +# roles/k3s_postgres, which declares a LoadBalancer Service for it alongside +# CNPG's built-in ClusterIP ones. Pinned rather than auto-assigned because +# the address gets written down elsewhere (an app's vars.yml, a connection +# string in Vault) instead of only being looked up at runtime, so it mustn't +# move when the Service is recreated. +# +# Must be inside the pool above — MetalLB only assigns from its own pools, +# and a request for an address outside them leaves the Service pending +# forever rather than failing loudly. The caveat of pinning out of the same +# pool MetalLB auto-assigns from: it hands unpinned Services the lowest free +# address, so if some other LoadBalancer Service is created before this one +# and takes .80, this Service is the one left pending. Nothing else here +# requests an IP today. If that changes, the fix is a second +# IPAddressPool with autoAssign: false in roles/k3s_metallb reserved for +# pinned addresses, not a different number here. +k3s_postgres_loadbalancer_ip: "192.168.50.81" + +# Where playbooks/k3s.yml fetches the kubeconfig to, on the Ansible +# controller. Under local/, which .gitignore already excludes — it's a +# generated credential, never committed. +k3s_kubeconfig_local_path: "{{ repo_root }}/local/k3s/{{ k3s_cluster_name }}.kubeconfig" + +# --- cert-manager (roles/k3s_cert_manager) ---------------------------------- +# +# TLS for anything this cluster serves over Traefik. See that role's +# defaults/main.yml for why DNS-01 rather than HTTP-01, and what each of +# these does. + +# Where Let's Encrypt sends expiry warnings. An ACME account is registered +# against it. +k3s_cert_manager_acme_email: russell.seymour@turtlesystems.co.uk + +# >>> SET BEFORE THE FIRST RUN <<< +# +# How cert-manager proves control of the domain, rendered into the +# ClusterIssuer's solvers list as-is. Left empty deliberately — it depends on +# who runs DNS for turtlesystems.uk, which this repo has no way to know. +# roles/k3s_cert_manager fails fast while it's empty, rather than creating an +# issuer that can never satisfy an order. +# +# Cloudflare, as the most common case — the token needs Zone:DNS:Edit on the +# zone, and is stored in Vault (see k3s_cert_manager_vault_path below and +# docs/vault-secrets.md) rather than written here: +# +# k3s_cert_manager_solver: +# dns01: +# cloudflare: +# apiTokenSecretRef: +# name: cert-manager-dns-credentials +# key: CLOUDFLARE_API_TOKEN +# selector: +# dnsZones: +# - turtlesystems.uk +# +# For another provider, take the stanza from +# https://cert-manager.io/docs/configuration/acme/dns01/ and point its secret +# ref at whichever key you stored at that Vault path. +# +# Use the Let's Encrypt staging directory while working this out — override +# k3s_cert_manager_acme_server. Production allows 5 failed validations per +# hostname per hour, and exhausting it means waiting, not retrying. +k3s_cert_manager_solver: + dns01: + cloudflare: + apiTokenSecretRef: + name: cert-manager-dns-credentials + key: CLOUDFLARE_API_TOKEN + selector: + dnsZones: + - turtlesystems.uk + +# --- Traefik (roles/k3s_traefik) -------------------------------------------- +# +# The ingress controller k3s installs for itself; that role installs nothing, +# it only adjusts what's already there. Its defaults are enough for the +# dashboard over `kubectl port-forward` — see that role's defaults/main.yml +# and README.md → "Dashboard (Traefik)". + +# >>> SET WHEN AUTHENTIK IS READY TO PROTECT IT <<< +# +# Uncomment to publish the dashboard on this hostname, behind the forward-auth +# middleware below. Until then the port-forward is the only way in, which is +# the safe default rather than a limitation. Needs a DNS record pointing at +# Traefik's MetalLB address (`kubectl -n kube-system get svc traefik`). +# +# Before uncommenting, in Authentik: create a Proxy Provider in **forward auth +# (domain level)** mode covering turtlesystems.uk, assign it to an +# application, and add that application to the embedded outpost. Domain level +# rather than single-application for a Kubernetes-specific reason — see +# roles/k3s_traefik/defaults/main.yml. +# +# k3s_traefik_dashboard_host: traefik.turtlesystems.uk + +# Authentik's embedded outpost, by in-cluster DNS — the `authentik-server` +# Service in the `authentik` namespace, i.e. the chart's own service, not a +# separately deployed outpost. Unused while the host above is commented out; +# set here anyway so enabling the dashboard is one line rather than two. +# roles/k3s_traefik refuses to run if the host is set and this isn't. +k3s_traefik_dashboard_auth_address: "http://authentik-server.authentik.svc.cluster.local/outpost.goauthentik.io/auth/traefik" + +# --- Apps on the cluster (roles/k3s_app) ------------------------------------ +# +# The k3s equivalent of `stacks:` in host_vars/nas01.yml and `apps:` in a +# Proxmox guest's host_vars — the list of *applications* on this cluster, as +# opposed to the k3s_* cluster services above, which are the cluster's own +# infrastructure. +# +# In group_vars rather than host_vars because an app is deployed to the +# cluster, not to a node: playbooks/k3s.yml runs roles/k3s_app against +# k3s_control_plane only (that's where the manifests directory is), but +# nothing about an app belongs to that Pi specifically. +# +# Same entry shape as the other two platforms — name, src, vault_path, an +# optional `db:`, and an optional `state:`. And the same removal rule as +# Unraid: set `state: absent` to tear an app down and leave the entry here as +# a tombstone; deleting the entry removes nothing, it just stops Ansible +# visiting it. See roles/k3s_app/tasks/remove.yml. +k3s_apps: + - name: authentik + src: authentik + vault_path: homelab/authentik + db: + name: authentik + user: authentik + # The key at homelab/authentik holding this role's password. Named for + # the environment variable Authentik itself reads, because + # roles/k3s_app passes Vault keys through to the app's Secret verbatim + # — so one value serves both the provisioning step and the running app. + password_vault_key: AUTHENTIK_POSTGRESQL__PASSWORD + admin_vault_path: homelab/shared/postgres + # The CNPG LoadBalancer, not the in-cluster -rw Service: these tasks + # run on the Ansible controller (delegate_to: localhost), which is off + # the cluster and can't resolve or route to a ClusterIP. Authentik + # itself uses the ClusterIP — see + # src/authentik/ansible/kubernetes/vars.yml. + provision_host: "{{ k3s_postgres_loadbalancer_ip }}" + provision_port: "5432" diff --git a/build/config/ansible/inventory/group_vars/k3s_control_plane.yml b/build/config/ansible/inventory/group_vars/k3s_control_plane.yml new file mode 100644 index 0000000..d4b1afc --- /dev/null +++ b/build/config/ansible/inventory/group_vars/k3s_control_plane.yml @@ -0,0 +1,9 @@ +--- +# Selects the server branch of roles/k3s_node/tasks/main.yml. +k3s_node_role: server + +# The API server's cert needs this IP in its SAN list so kubectl can reach it +# at the same address agents join through. Defaults to the control-plane +# host's own address; override here if you later put a DNS name or VIP in +# front of it. +k3s_api_tls_san: "{{ ansible_host }}" diff --git a/build/config/ansible/inventory/group_vars/k3s_workers.yml b/build/config/ansible/inventory/group_vars/k3s_workers.yml new file mode 100644 index 0000000..72e5a5d --- /dev/null +++ b/build/config/ansible/inventory/group_vars/k3s_workers.yml @@ -0,0 +1,3 @@ +--- +# Selects the agent branch of roles/k3s_node/tasks/main.yml. +k3s_node_role: agent diff --git a/build/config/ansible/inventory/group_vars/proxmox_guests.yml b/build/config/ansible/inventory/group_vars/proxmox_guests.yml new file mode 100644 index 0000000..f48f891 --- /dev/null +++ b/build/config/ansible/inventory/group_vars/proxmox_guests.yml @@ -0,0 +1,19 @@ +--- +# Connection settings for the LXCs on the Proxmox nodes — the group +# inventory/proxmox.yml composes from the API, targeted by +# playbooks/proxmox.yml. As distinct from proxmox_nodes.yml, which is the +# hosts those guests run on. +# +# In group_vars rather than repeated per guest because it is a property of +# how every container this repo creates is built, not of any one app: the +# Terraform modules install `ssh_public_keys` for root (see +# build/config/terraform/variables.tf) and create no other account, so root +# is the only user Ansible can authenticate as. +# +# Needed explicitly because the other three groups set this in hosts.yml +# (`root` for unraid_servers and proxmox_nodes, `ansible` for k3s_cluster) +# and the guests have no static entry to set it in. Without it Ansible falls +# back to the *controller's* login name, which no container has an account +# for — a "Permission denied (publickey)" that names a user nothing in the +# repo mentions, so there is no obvious thread to pull. +ansible_user: root diff --git a/build/config/ansible/inventory/group_vars/proxmox_nodes.yml b/build/config/ansible/inventory/group_vars/proxmox_nodes.yml new file mode 100644 index 0000000..cc5942d --- /dev/null +++ b/build/config/ansible/inventory/group_vars/proxmox_nodes.yml @@ -0,0 +1,46 @@ +--- +# Configuration for the Proxmox nodes themselves — as distinct from the guests +# on them, which come from the API-backed dynamic inventory and are configured +# by playbooks/proxmox.yml. Applied by playbooks/pve_host.yml. +# +# In group_vars rather than a host_vars file per node: both nodes want the +# same backup storage and the same schedule, and each backs up whichever +# guests are running on it, so there is nothing here that differs between +# them. A node that ever does need to differ can still override in +# host_vars/.yml. + +# Where archives go. An export on the NAS, added to Proxmox as a storage so +# vzdump can write to it — the node mounts it, no guest ever sees it. +# +# nas01 (192.168.50.1). This used to point at nas2, which no longer exists — +# and note the address it used, 192.168.50.2, now belongs to +# turtle-proxmox-01, so a stale copy of this value would have had each node +# quietly trying to write its backups to itself. +pve_backup_storage: nas-backup +pve_backup_storage_type: nfs +pve_backup_nfs_server: 192.168.50.1 +pve_backup_nfs_export: /mnt/user/backups/proxmox + +# Every guest on the node, so anything added later is covered without an edit +# here. Set false and list VMIDs (postgres is 161, forgejo 160 — see the +# `backup_vmids` output in build/config/terraform/main.tf) to be selective. +# +# Applied per node, so with two nodes each one archives whatever is currently +# running on it. That matters for a replicated guest: it is a real guest on +# one node and an inactive replica volume on the other, so it is backed up +# once, by whichever node currently owns it — and after a failover, by the +# other one, without an edit here. +pve_backup_all: true + +# 02:00, half an hour after Forgejo's database dump at 01:30 +# (src/forgejo/ansible/proxmox/vars.yml). The archive is only a complete +# restore point if the dump inside it is from the same night — move one and +# move the other. +pve_backup_schedule: "02:00" + +# Archive retention, applied by Proxmox. Distinct from the retention inside a +# guest: Proxmox prunes archives, not their contents. +pve_backup_prune: + keep-daily: 7 + keep-weekly: 4 + keep-monthly: 6 diff --git a/build/config/ansible/inventory/host_vars/forgejo.yml b/build/config/ansible/inventory/host_vars/forgejo.yml new file mode 100644 index 0000000..30791b0 --- /dev/null +++ b/build/config/ansible/inventory/host_vars/forgejo.yml @@ -0,0 +1,52 @@ +--- +# The Proxmox LXC created by src/forgejo/terraform (VMID 160), on +# turtle-proxmox-02. Named for the container's hostname, which is how it +# appears in the API-backed dynamic inventory. +# +# This was empty for a long time while Forgejo also ran as a Compose stack on +# nas2 — two deployments sharing one Vault path and one database means the +# second to start wins and the other quietly serves stale state. nas2 has +# since been retired, so there is no longer a second copy to collide with and +# the `apps:` list below is live. +# +# WHAT THIS DOES NOT DO: nothing here moves repository data. Ansible +# provisions the database and installs the app; a Forgejo with no +# /var/lib/forgejo/data/forgejo-repositories is a working, empty forge. If the +# nas2 repositories still exist somewhere, restore them into +# `FORGEJO_DATA_PATH` and `chown -R git:git /var/lib/forgejo/data` before +# pointing DNS at this host — the original instructions here were an rsync +# off nas2, which is no longer a machine that can be read from. +# +# Also note SSH clone URLs move from port 2222 to 22 (see FORGEJO_SSH_PORT in +# src/forgejo/ansible/proxmox/vars.yml): existing remotes need editing, and +# git.turtlesystems.uk has to resolve to this container. + +# The address Terraform assigns this guest (build/config/terraform/main.tf). +# Pinned here rather than left to DNS: the API-backed dynamic inventory +# supplies no `ansible_host`, so without this Ansible SSHes to the bare +# inventory name `forgejo`, which the LAN's resolver has no record for at +# all. These addresses are hand-assigned from the 192.168.50.50-.59 band +# anyway, so restating one here duplicates nothing that was ever derived. +ansible_host: 192.168.50.52 + +apps: + - name: forgejo + src: forgejo + vault_path: homelab/forgejo + db: + # The shared Postgres LXC, reached over the LAN. Unlike the Unraid + # host_vars this can't be `{{ ansible_host }}` — that would be this + # container's own address, not the database's. Keep in step with + # DB_HOST in src/forgejo/ansible/proxmox/vars.yml; the authoritative + # copy of both is the `postgres_address` output in + # build/config/terraform/main.tf. + # + # Provisioning runs from the Ansible controller (`delegate_to: + # localhost` in roles/lxc_app), so this address has to be reachable + # from wherever you run the playbook, not just from this container. + provision_host: 192.168.50.54 + provision_port: 5432 + name: forgejo + user: forgejo + password_vault_key: DB_PASSWORD + admin_vault_path: homelab/shared/postgres diff --git a/build/config/ansible/inventory/host_vars/nas01.yml b/build/config/ansible/inventory/host_vars/nas01.yml new file mode 100644 index 0000000..dcacd1a --- /dev/null +++ b/build/config/ansible/inventory/host_vars/nas01.yml @@ -0,0 +1,19 @@ +--- +# Existing Unraid server. Stacks are applied in list order. +# +# Each entry takes an optional `state:` — `present` (the default) deploys it, +# `absent` tears it down. To remove a stack, set `state: absent` and leave the +# entry here; deleting it instead removes nothing, it just stops Ansible +# managing the containers. See README.md → "Removing an app". +stacks: + # The arr ecosystem as one stack: Shelfarr (library) plus Prowlarr (indexer + # manager) in a single Compose project, so they come up together and share a + # network, a rendered .env and a Vault path. + - name: arr + src: arr + vault_path: homelab/arr + # No `db:` block — neither container uses an external database. Shelfarr + # is SQLite-only (config/database.yml pins production to storage/*.sqlite3 + # with no DATABASE_URL support) and Prowlarr keeps its own SQLite under + # /config, so there is nothing to provision on the shared Postgres. Both + # databases live in the appdata bind mounts along with everything else. diff --git a/build/config/ansible/inventory/host_vars/postgres.yml b/build/config/ansible/inventory/host_vars/postgres.yml new file mode 100644 index 0000000..4d15c18 --- /dev/null +++ b/build/config/ansible/inventory/host_vars/postgres.yml @@ -0,0 +1,28 @@ +--- +# The shared Postgres LXC created by src/shared/postgres/terraform (VMID 161, +# 192.168.50.54). Named for the container's hostname, which is how it appears +# in the API-backed dynamic inventory. +# +# This is the one app with no `db:` block, because it *is* the database — the +# per-app databases are created against it by whichever compose_stack / +# lxc_app / k3s_app run declares them, using homelab/shared/postgres as +# `admin_vault_path`. + +# The address Terraform assigns this guest (build/config/terraform/main.tf). +# Pinned here rather than left to DNS: the API-backed dynamic inventory +# supplies no `ansible_host`, so without this Ansible SSHes to the bare +# inventory name and whatever the LAN's resolver happens to return. For +# `postgres` that resolver answers 192.168.50.53 — the *Unraid* Postgres +# container, a different instance of the same service — which fails as a +# connection refused rather than as anything that names the real problem. +# These addresses are hand-assigned from the 192.168.50.50-.59 band anyway, +# so restating one here duplicates nothing that was ever derived. +ansible_host: 192.168.50.54 + +apps: + - name: postgres + src: shared/postgres + vault_path: homelab/shared/postgres + # The packaging's umbrella unit, not postgresql@17-main — see + # POSTGRES_SERVICE_NAME in src/shared/postgres/ansible/proxmox/vars.yml. + service_name: postgresql diff --git a/build/config/ansible/inventory/hosts.yml b/build/config/ansible/inventory/hosts.yml new file mode 100644 index 0000000..99b38d3 --- /dev/null +++ b/build/config/ansible/inventory/hosts.yml @@ -0,0 +1,58 @@ +--- +all: + children: + unraid_servers: + hosts: + nas01: + ansible_host: 192.168.50.1 + vars: + ansible_user: root + + # The Proxmox host itself, not the guests on it — those come from the + # API-backed dynamic inventory (proxmox.yml) as `proxmox_guests`. Node-level + # configuration (backup storage, the vzdump schedule) has to be applied over + # SSH to the node, which the API inventory gives no way to address. + # + # VERIFY: depending on its configuration, the community.proxmox inventory + # plugin can also emit the node as a host. If it does, this static entry and + # the dynamic one merge under the same name — check with + # `ansible-inventory --graph` before assuming which vars won. + proxmox_nodes: + hosts: + turtle-proxmox-01: + ansible_host: 192.168.50.2 + turtle-proxmox-02: + ansible_host: 192.168.50.3 + vars: + ansible_user: root + + # Bare-metal K3s cluster (4 Raspberry Pis) — a third platform alongside + # Unraid and Proxmox, not an app deployed onto either of them. No + # Terraform: the hosts already exist. See playbooks/k3s.yml and + # roles/k3s_node. Split into two groups, not one, so the playbook can + # bootstrap the control plane before any agent tries to join it. + # + # ansible_user is `ansible`, not root: create this user on every Pi with + # NOPASSWD sudo and install the public half of `homelab/ci/ssh-k3s` + # (docs/vault-secrets.md) as its authorized key before the first run — + # see README.md "SSH access". The private half doesn't need setting up on + # the controller: group_vars/k3s_cluster.yml points + # ansible_ssh_private_key_file at a path playbooks/k3s_ssh_key.yml fetches + # from that same Vault entry. + k3s_cluster: + vars: + ansible_user: ansible + ansible_become: true + children: + k3s_control_plane: + hosts: + k3s-ctrl-01: + ansible_host: 192.168.50.60 + k3s_workers: + hosts: + k3s-wkr-01: + ansible_host: 192.168.50.61 + k3s-wkr-02: + ansible_host: 192.168.50.62 + k3s-wkr-03: + ansible_host: 192.168.50.63 diff --git a/build/config/ansible/inventory/proxmox.yml b/build/config/ansible/inventory/proxmox.yml new file mode 100644 index 0000000..ff3a4a0 --- /dev/null +++ b/build/config/ansible/inventory/proxmox.yml @@ -0,0 +1,56 @@ +--- +# Dynamic inventory: asks the Proxmox API which guests exist rather than +# listing them by hand, so LXCs created by `terraform apply` show up without +# a second edit here. +# +# VERIFY BEFORE FIRST USE — two things this config asserts that depend on +# your Proxmox and collection versions: +# +# 1. Collection name. Proxmox content was split out of `community.general` +# into its own `community.proxmox` collection. If your installed +# Ansible predates that split, this is `community.general.proxmox` +# instead — here, in requirements.yml, and in ansible.cfg's +# `enable_plugins`. Check with: +# ansible-doc -t inventory community.proxmox.proxmox +# 2. Auto-generated group names. The plugin builds groups from guest type +# and state; `proxmox_all_lxc` is the conventional one but confirm +# against your own output before relying on it: +# ansible-inventory -i inventory/proxmox.yml --graph +# +plugin: community.proxmox.proxmox + +url: "{{ lookup('env', 'PROXMOX_URL') }}" +user: "{{ lookup('env', 'PROXMOX_USER') }}" + +# API token, not a password — same reasoning as the Unraid SSH key: a +# password would mean an interactive prompt (impossible from CI) or a +# plaintext secret on disk. Store these in Vault under `homelab/ci/proxmox` +# and export them before running; see docs/vault-secrets.md. +token_id: "{{ lookup('env', 'PROXMOX_TOKEN_ID') }}" +token_secret: "{{ lookup('env', 'PROXMOX_TOKEN_SECRET') }}" + +# Homelab Proxmox typically has a self-signed cert. Flip to true once you +# put a real one on it. +validate_certs: false + +want_facts: true + +# Manually-created guests are not this repo's to touch. Both Terraform +# modules tag the LXCs they create with `terraform` (see +# src/*/terraform/variables.tf), so that tag is the line between "managed +# here" and "made by hand on the node". Filtering rather than narrowing +# the group below keeps hand-made guests out of the inventory entirely, +# so no future play can target them by accident either. +# +# `filters` needs a reasonably recent plugin version. If yours rejects the +# option, delete this block and put the same condition on the group: +# proxmox_guests: >- +# proxmox_vmtype == 'lxc' +# and 'terraform' in (proxmox_tags_parsed | default([])) +filters: + - "'terraform' in (proxmox_tags_parsed | default([]))" + +groups: + # The group playbooks/proxmox.yml targets. Containers only — VMs, if you + # ever add any, are not what `lxc_app` knows how to install into. + proxmox_guests: "proxmox_vmtype == 'lxc'" diff --git a/build/config/ansible/playbooks/deploy.yml b/build/config/ansible/playbooks/deploy.yml new file mode 100644 index 0000000..71ed74f --- /dev/null +++ b/build/config/ansible/playbooks/deploy.yml @@ -0,0 +1,21 @@ +--- +# Everything, both platforms. Unraid first only by convention now — the two +# platforms no longer depend on each other. Proxmox-side apps provision their +# databases against the Proxmox shared Postgres (192.168.50.54), and Terraform +# keeps its state on the CloudNativePG cluster on k3s, neither of which is on +# Unraid. +# +# Run a single platform with `ansible-playbook playbooks/unraid.yml` or +# `playbooks/proxmox.yml` instead. `--limit ` and `-e only_stacks=...` +# work the same way against any of the three. +- name: Deploy Unraid stacks + ansible.builtin.import_playbook: unraid.yml + +- name: Deploy Proxmox apps + ansible.builtin.import_playbook: proxmox.yml + +# Last, because the vzdump job can be pinned to specific VMIDs and those guests +# have to exist first. Harmless to re-run; skipped entirely if no node is +# defined in the `proxmox_nodes` group. +- name: Configure Proxmox node backups + ansible.builtin.import_playbook: pve_host.yml diff --git a/build/config/ansible/playbooks/k3s.yml b/build/config/ansible/playbooks/k3s.yml new file mode 100644 index 0000000..afde238 --- /dev/null +++ b/build/config/ansible/playbooks/k3s.yml @@ -0,0 +1,241 @@ +--- +# Bootstraps the homelab-utils k3s cluster from bare Pis: no Terraform, the +# hosts already exist (inventory/hosts.yml → k3s_cluster). Manual-only for +# now — not wired into .forgejo/workflows/deploy.yml, since converging 4 +# physical nodes on every push is a bigger blast radius than restarting a +# Compose stack (same reasoning the repo already applies to `terraform +# apply`). Run by hand: +# +# ansible-playbook playbooks/k3s.yml +# +# Every play below the SSH-key import is tagged so a single piece can be +# converged without touching the rest — deploying one cluster service +# shouldn't mean re-running the node install across 4 Pis: +# +# --tags nodes k3s install/join + exec-line drift (both node plays) +# --tags metallb MetalLB chart + IPAddressPool +# --tags monitoring kube-prometheus-stack chart +# --tags postgres CloudNativePG operator + Cluster +# --tags cert-manager cert-manager chart + ClusterIssuer +# --tags traefik dashboard route on k3s's bundled Traefik +# --tags services all five of the above, no node install +# --tags apps the k3s_apps: list (roles/k3s_app) — apps, not +# cluster services; narrow further with -e only_apps=… +# --tags upgrades unattended-upgrades config +# --tags kubeconfig re-fetch the admin kubeconfig +# +# e.g. `ansible-playbook playbooks/k3s.yml --tags postgres`. The SSH-key +# import is tagged `always` rather than given a name of its own: it's not a +# thing you'd select, it's the prerequisite for any of these connecting at +# all, so it has to survive every --tags filter. +# +# `services` covers the cluster's own infrastructure and `apps` covers what +# runs on top of it; neither implies the other, so a run meaning "everything +# except reinstalling k3s" is `--tags services,apps`. +# +# Prerequisites (see README.md "K3s (Raspberry Pi)"): the `ansible` user +# exists on every Pi with NOPASSWD sudo and this repo's SSH key installed, +# and homelab/k3s-homelab-utils/K3S_TOKEN is already set in Vault — see +# docs/vault-secrets.md. Both plays fetch that same fixed token independently +# rather than one generating it and handing it to the other, which is what +# makes a full rebuild (wipe both SD cards, reinstall) reproduce the same +# cluster identity. + +# First, before anything tries to connect: pull the SSH key the plays below +# authenticate with out of Vault and onto the controller. See +# k3s_ssh_key.yml — it's a separate file because k3s_maintenance.yml imports +# it too. +- ansible.builtin.import_playbook: k3s_ssh_key.yml + tags: always + +- name: Bootstrap the k3s control plane + hosts: k3s_control_plane + gather_facts: true + tags: nodes + roles: + - k3s_node + +# serial: 1 not for joining itself (idempotent, safe in parallel) but for +# roles/k3s_node's exec-line drift detection: if k3s_extra_args changes and +# every worker's k3s-agent restarts to pick it up, one at a time keeps more +# than one node's kubelet from bouncing simultaneously. See tasks/agent.yml. +- name: Join k3s worker nodes + hosts: k3s_workers + gather_facts: true + serial: 1 + tags: nodes + roles: + - k3s_node + +# Against the control plane only — it just drops manifests for k3s's own +# helm-controller and deploy controller to reconcile, so it doesn't need +# every node like the plays above. Runs after workers have joined so the +# resulting speaker DaemonSet schedules across the whole cluster on first +# reconcile, though this doesn't strictly matter: the controller picks up +# new nodes on its own regardless of ordering. Before monitoring so a +# LoadBalancer Service (e.g. exposing Prometheus/Grafana later) has +# somewhere to get an IP from as soon as it's requested. +- name: Deploy MetalLB (LoadBalancer IPs) + hosts: k3s_control_plane + gather_facts: false + tags: + - services + - metallb + roles: + - k3s_metallb + +# Against the control plane only — it just drops a HelmChart manifest for +# k3s's own helm-controller to reconcile, so it doesn't need every node like +# the plays above. Runs after workers have joined so the resulting +# node-exporter DaemonSet schedules across the whole cluster on first +# reconcile, though this doesn't strictly matter: the controller picks up +# new nodes on its own regardless of ordering. +- name: Deploy cluster monitoring (Prometheus) + hosts: k3s_control_plane + gather_facts: false + tags: + - services + - monitoring + roles: + - k3s_monitoring + +# Against the control plane only, same reasoning as monitoring/MetalLB above +# — it just drops manifests for k3s's own controllers to reconcile. Needs +# repo_root (unlike the other k3s roles) because it reads config from +# src/shared/postgres/, the same shared-service config Unraid/Proxmox +# already deploy from — see roles/k3s_postgres/tasks/main.yml. +- name: Deploy shared Postgres (CloudNativePG) + hosts: k3s_control_plane + gather_facts: false + tags: + - services + - postgres + vars: + repo_root: "{{ playbook_dir }}/../../../.." + roles: + - k3s_postgres + +# Against the control plane only, same reasoning as the services above. +# Before the apps play because an app's Ingress annotates itself against the +# ClusterIssuer this creates — not that ordering is load-bearing (k3s's +# deploy controller retries, and cert-manager picks up an Ingress whenever it +# appears), but an app deployed first would sit without a certificate until +# this ran, which reads as a broken deploy rather than a pending one. +- name: Deploy cert-manager (TLS certificates) + hosts: k3s_control_plane + gather_facts: false + tags: + - services + - cert-manager + roles: + - k3s_cert_manager + +# Against the control plane only, same reasoning as the services above. The +# odd one out among them: it installs nothing, because k3s installs Traefik +# itself — it only adjusts what k3s already put there, via a HelmChartConfig. +# After cert-manager because publishing the dashboard on a hostname asks for a +# Certificate from the ClusterIssuer that play creates; ordering isn't +# load-bearing (the deploy controller retries), it just avoids a route sitting +# without a certificate in between. +- name: Configure Traefik (ingress + dashboard) + hosts: k3s_control_plane + gather_facts: false + tags: + - services + - traefik + roles: + - k3s_traefik + +# Apps, as opposed to the cluster services above — see roles/k3s_app for the +# distinction and inventory/group_vars/k3s_cluster.yml for the list. Against +# the control plane because that's where the manifests directory is; nothing +# about an app belongs to that Pi in particular. +# +# Note what a green run here does and doesn't mean, the same caveat every +# service play above carries: the role renders manifests for k3s's +# controllers to reconcile, so success means the files landed and any +# database was provisioned — not that the workload came up. Check with +# `kubectl -n get pods`. +- name: Converge apps on the cluster + hosts: k3s_control_plane + gather_facts: false + tags: apps + vars: + repo_root: "{{ playbook_dir }}/../../../.." + # Comma-separated app names to restrict this run to, e.g. + # `-e only_apps=authentik`. Empty (the default) converges every app in + # k3s_apps. Same knob as only_stacks in playbooks/unraid.yml — --tags + # can select the apps play as a whole, but not one app within it. + only_apps: "" + + tasks: + # "Converge", not "deploy": an entry carrying `state: absent` is torn + # down rather than brought up. Removals have to stay in the list to be + # acted on — see roles/k3s_app/tasks/main.yml. + - name: Converge each declared app + ansible.builtin.include_role: + name: k3s_app + loop: >- + {{ k3s_apps | default([]) if only_apps == '' + else k3s_apps | default([]) + | selectattr('name', 'in', only_apps.split(',')) | list }} + loop_control: + loop_var: app + label: "{{ app.name }}" + +# Every node, both roles — hands-off patching so the Pis don't need logging +# into just to stay updated. Reboots are deliberately not part of this: see +# roles/unattended_upgrades and playbooks/k3s_maintenance.yml. +- name: Configure unattended upgrades + hosts: k3s_cluster + gather_facts: true + tags: upgrades + roles: + - unattended_upgrades + +# Last, and against the control plane specifically (there's only one) — pulls +# the admin kubeconfig k3s wrote for itself back to the controller so +# `kubectl` works from your workstation. +- name: Fetch the cluster kubeconfig + hosts: k3s_control_plane + gather_facts: false + tags: kubeconfig + vars: + repo_root: "{{ playbook_dir }}/../../../.." + + tasks: + - name: Read the cluster's kubeconfig + ansible.builtin.slurp: + src: /etc/rancher/k3s/k3s.yaml + register: k3s_kubeconfig_raw + + - name: Ensure the local kubeconfig directory exists + ansible.builtin.file: + path: "{{ k3s_kubeconfig_local_path | dirname }}" + state: directory + mode: "0700" + delegate_to: localhost + become: false + + # k3s.yaml points at 127.0.0.1 and names everything "default" — both + # correct only on the node itself. Rewritten so the file is usable + # straight off the controller: the server address becomes reachable from + # off-box, and the cluster/context/user names become this cluster's own + # rather than colliding with every other "default" in ~/.kube/config. + - name: Write the rewritten kubeconfig to the controller + ansible.builtin.copy: + dest: "{{ k3s_kubeconfig_local_path }}" + content: >- + {{ (k3s_kubeconfig_raw.content | b64decode) + | replace('127.0.0.1', ansible_host) + | regex_replace('\\bdefault\\b', k3s_cluster_name) }} + mode: "0600" + delegate_to: localhost + become: false + + - name: Show how to use the fetched kubeconfig + ansible.builtin.debug: + msg: >- + Kubeconfig written to {{ k3s_kubeconfig_local_path }}. Use it with + `export KUBECONFIG={{ k3s_kubeconfig_local_path }}`, or merge it + into ~/.kube/config by hand. diff --git a/build/config/ansible/playbooks/k3s_maintenance.yml b/build/config/ansible/playbooks/k3s_maintenance.yml new file mode 100644 index 0000000..06d7157 --- /dev/null +++ b/build/config/ansible/playbooks/k3s_maintenance.yml @@ -0,0 +1,37 @@ +--- +# Rolls a "reboot required" flag left by unattended-upgrades +# (roles/unattended_upgrades, Automatic-Reboot disabled) into an actual +# reboot, one node at a time — cordon, drain, reboot, wait for Ready, +# uncordon — so patching the OS never means dropping every workload on the +# cluster at once. A no-op on any node that isn't carrying a pending reboot. +# +# serial: 1 is the point of this playbook: exactly one node is ever down for +# maintenance at a time, whatever `--limit` narrows the run to. +# +# Host order matters here and isn't left to inventory.yml's own group +# order (control plane, then workers) — reversed below so workers are +# rebooted while the API server is still up on the control plane, and the +# control plane's own reboot (which drops the API for everyone, single-server +# cluster, no HA) happens last rather than first. +# +# Manual-only for now, like playbooks/k3s.yml — run by hand: +# +# ansible-playbook playbooks/k3s_maintenance.yml +# +# Intended to eventually run on a schedule from a self-hosted Forgejo Actions +# runner (a `schedule:`-triggered workflow, same runner as +# .forgejo/workflows/deploy.yml) rather than by hand — not wired up yet, but +# the SSH key import below is what makes that possible without also handing +# the runner an ssh-agent. + +# Same first play as k3s.yml — the key comes from Vault, and this playbook is +# routinely run on its own, so it can't rely on a k3s.yml run having fetched +# it. See k3s_ssh_key.yml. +- ansible.builtin.import_playbook: k3s_ssh_key.yml + +- name: Roll pending reboots across the k3s cluster + hosts: k3s_workers:k3s_control_plane + gather_facts: true + serial: 1 + roles: + - k3s_maintenance diff --git a/build/config/ansible/playbooks/k3s_ssh_key.yml b/build/config/ansible/playbooks/k3s_ssh_key.yml new file mode 100644 index 0000000..4576faf --- /dev/null +++ b/build/config/ansible/playbooks/k3s_ssh_key.yml @@ -0,0 +1,82 @@ +--- +# Materializes the SSH key the k3s plays authenticate with, from Vault +# (homelab/ci/ssh-k3s) onto the controller. Imported as the first play of both +# playbooks/k3s.yml and playbooks/k3s_maintenance.yml — a separate file rather +# than copied into each, since both target k3s_cluster and both would +# otherwise depend on a file only the other one creates. +# +# This is the same shape CI already uses for the Unraid key: +# .forgejo/workflows/deploy.yml fetches homelab/ci/ssh, writes it to disk and +# points ANSIBLE_PRIVATE_KEY_FILE at it. Vault is the one place secrets live, +# so a manual k3s run shouldn't need the key hand-loaded into ssh-agent first +# — which was the only way it worked before. +# +# hosts: k3s_cluster, not localhost, and deliberately so: the implicit +# localhost is not a member of `all`, so it doesn't inherit group_vars/all.yml +# — where vault_addr, vault_kv_mount and vault_auth_method live. Targeting the +# group picks those up along with k3s_ssh_key_vault_path/_local_path from +# group_vars/k3s_cluster.yml. Nothing here connects to a Pi (gather_facts is +# off and every task is delegated or connectionless), which is the whole +# point: at this stage the key isn't on disk yet. +# +# become: false on the delegated tasks because the k3s_cluster group sets +# ansible_become: true for the Pis (inventory/hosts.yml) — without it these +# would try to sudo on the controller. + +- name: Fetch the k3s SSH deploy key from Vault + hosts: k3s_cluster + gather_facts: false + + tasks: + # The escape hatch, and the reason it's a flag rather than just pointing + # ansible_ssh_private_key_file somewhere else: if Vault is unreachable, + # overriding the key path alone doesn't help — this play would still fail + # before the first real one runs. Skipping it is the only thing that lets + # a run proceed on a local key. Both overrides together: + # + # ansible-playbook playbooks/k3s.yml \ + # -e k3s_ssh_key_fetch=false \ + # -e ansible_ssh_private_key_file=~/.ssh/k3s_ansible + - name: Fetch and write the key + when: k3s_ssh_key_fetch | default(true) | bool + block: + # run_once because the key is per-cluster, not per-host: one Vault + # read for the whole group. Facts set by a run_once task apply to + # every host in the play, but nothing outside this play needs it. + - name: Look up the k3s SSH deploy key from Vault + ansible.builtin.set_fact: + k3s_ssh_key_secret: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + k3s_ssh_key_vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + run_once: true + no_log: true + + - name: Ensure the local key directory exists + ansible.builtin.file: + path: "{{ k3s_ssh_key_local_path | dirname }}" + state: directory + mode: "0700" + delegate_to: localhost + become: false + run_once: true + + # trim + an explicit trailing newline: OpenSSH rejects a key file + # whose final line isn't terminated, and `vault kv put + # PRIVATE_KEY=@file` is as likely to have stored one with trailing + # whitespace as not. 0600 for the same reason — ssh refuses a key file + # that's group- or world-readable. Both fail at connection time rather + # than here, which is a much less obvious error to read. + - name: Write the private key to the controller + ansible.builtin.copy: + dest: "{{ k3s_ssh_key_local_path }}" + content: "{{ k3s_ssh_key_secret.PRIVATE_KEY | trim }}\n" + mode: "0600" + delegate_to: localhost + become: false + run_once: true + no_log: true diff --git a/build/config/ansible/playbooks/proxmox.yml b/build/config/ansible/playbooks/proxmox.yml new file mode 100644 index 0000000..4cdbc73 --- /dev/null +++ b/build/config/ansible/playbooks/proxmox.yml @@ -0,0 +1,29 @@ +--- +# Installs apps natively into Proxmox LXCs. The LXCs themselves are created +# by Terraform (build/config/terraform) — run that first; this playbook +# assumes the guests already exist and are reachable. +# +# Hosts come from the community.proxmox inventory plugin +# (inventory/proxmox.yml), which queries the Proxmox API rather than listing +# guests by hand. `apps` is declared per guest in host_vars/.yml, +# mirroring how `stacks` works for Unraid hosts. +- name: Install apps into Proxmox LXCs + hosts: proxmox_guests + gather_facts: true + vars: + repo_root: "{{ playbook_dir }}/../../../.." + # Comma-separated app names to restrict this run to. Shares the + # `only_stacks` name with playbooks/unraid.yml so a single `-e` narrows + # both platforms in one invocation of deploy.yml. + only_stacks: "" + + tasks: + - name: Install each declared app + ansible.builtin.include_role: + name: lxc_app + loop: >- + {{ apps if only_stacks == '' + else apps | selectattr('name', 'in', only_stacks.split(',')) | list }} + loop_control: + loop_var: app + label: "{{ app.name }}" diff --git a/build/config/ansible/playbooks/pve_host.yml b/build/config/ansible/playbooks/pve_host.yml new file mode 100644 index 0000000..b2d81fe --- /dev/null +++ b/build/config/ansible/playbooks/pve_host.yml @@ -0,0 +1,14 @@ +--- +# Configures the Proxmox node itself — as distinct from the guests on it, +# which playbooks/proxmox.yml handles. Right now that means backups: the NAS +# storage archives are written to, and the vzdump schedule that fills it. +# +# Targets the static `proxmox_nodes` group in inventory/hosts.yml rather than +# the API-backed dynamic inventory, which enumerates guests. If no node is +# defined there yet, this play matches nothing and is skipped. +- name: Configure Proxmox nodes + hosts: proxmox_nodes + gather_facts: true + + roles: + - pve_backup diff --git a/build/config/ansible/playbooks/unraid.yml b/build/config/ansible/playbooks/unraid.yml new file mode 100644 index 0000000..df844ff --- /dev/null +++ b/build/config/ansible/playbooks/unraid.yml @@ -0,0 +1,24 @@ +--- +- name: Converge Unraid Docker Compose stacks + hosts: unraid_servers + gather_facts: true + vars: + repo_root: "{{ playbook_dir }}/../../../.." + # Comma-separated stack names to restrict this run to, e.g. + # `-e only_stacks=forgejo` or `-e only_stacks=forgejo,postgres`. + # Empty (the default) converges every stack declared for the host. + only_stacks: "" + + tasks: + # "Converge", not "deploy": an entry carrying `state: absent` is torn down + # rather than brought up. Removals have to stay in the list to be acted + # on — see the header comment in roles/compose_stack/tasks/main.yml. + - name: Converge each declared stack + ansible.builtin.include_role: + name: compose_stack + loop: >- + {{ stacks if only_stacks == '' + else stacks | selectattr('name', 'in', only_stacks.split(',')) | list }} + loop_control: + loop_var: stack + label: "{{ stack.name }}" diff --git a/build/config/ansible/requirements.yml b/build/config/ansible/requirements.yml new file mode 100644 index 0000000..25cf287 --- /dev/null +++ b/build/config/ansible/requirements.yml @@ -0,0 +1,15 @@ +--- +collections: + - name: community.docker + version: ">=3.10.0" + - name: community.hashi_vault + version: ">=6.2.0" + - name: community.postgresql + version: ">=3.0.0" + - name: ansible.posix + version: ">=1.5.0" + # Proxmox dynamic inventory (inventory/proxmox.yml). If your Ansible + # predates the split of Proxmox content out of community.general, drop + # this and use community.general instead — see the note in + # inventory/proxmox.yml. + - name: community.proxmox diff --git a/build/config/ansible/roles/compose_stack/defaults/main.yml b/build/config/ansible/roles/compose_stack/defaults/main.yml new file mode 100644 index 0000000..291c157 --- /dev/null +++ b/build/config/ansible/roles/compose_stack/defaults/main.yml @@ -0,0 +1,9 @@ +--- +compose_projects_root: /boot/config/plugins/compose.manager/projects +vault_kv_mount: kv + +# Where a stack's committed icon.png is copied to on the Unraid host, for the +# `net.unraid.docker.icon` label to point at. Under appdata rather than +# /boot — it's regenerated content, not configuration worth putting on the +# flash drive's write budget. +unraid_icons_root: /mnt/user/appdata/icons diff --git a/build/config/ansible/roles/compose_stack/tasks/deploy.yml b/build/config/ansible/roles/compose_stack/tasks/deploy.yml new file mode 100644 index 0000000..6093dcb --- /dev/null +++ b/build/config/ansible/roles/compose_stack/tasks/deploy.yml @@ -0,0 +1,183 @@ +--- +# Deploys one stack to the current Unraid host: sync the compose file, render +# its .env from Vault, provision its database if it declares one, then +# `docker compose up -d`. Compose files stay static/generic — only the +# rendered .env differs per run, which is what keeps re-runs idempotent. +# +# Included from main.yml when the stack's `state:` is `present`; the facts it +# relies on (stack_local_dir, stack_remote_dir) are set there. + +- name: Load portable stack variables + ansible.builtin.include_vars: + file: "{{ stack_local_dir }}/common/vars.yml" + name: stack_common_vars + +- name: Load Unraid-specific stack variables + ansible.builtin.include_vars: + file: "{{ stack_local_dir }}/ansible/unraid/vars.yml" + name: stack_platform_vars + +- name: Ensure remote project directory exists + ansible.builtin.file: + path: "{{ stack_remote_dir }}" + state: directory + mode: "0750" + +- name: Copy docker-compose.yml + ansible.builtin.copy: + src: "{{ stack_local_dir }}/ansible/unraid/docker-compose.yml" + dest: "{{ stack_remote_dir }}/docker-compose.yml" + mode: "0640" + +# --- Unraid Docker page presentation --------------------------------------- +# +# The committed icon.png has two destinations, because Unraid draws the stack +# row and the containers under it from entirely different sources: +# +# stack row — Compose Manager serves /icon.png directly +# (icon.php, which also accepts .jpg/.gif/.svg or a bare +# `icon`). No label, no template, no URL: a file or nothing. +# Present only in the maintained fork, Compose Manager Plus; +# the original plugin has no icon support at all, and there +# the copy is inert rather than harmful. +# containers — the Docker page reads a container's icon, WebUI link and +# console shell from the dockerMan template that created it. +# A Compose stack has no template, so 6.10+ falls back to the +# `net.unraid.docker.*` labels the compose files set, and +# STACK_ICON below is what the icon label resolves to. +# +# Hence the appdata copy as well: the label is a path the webgui resolves at +# page-render time, and pointing it into /boot to reuse the project-dir copy +# would put the flash drive in the path of every Docker page load. +# +# Only the second of those is per-container, so a stack running more than one +# service can also commit icon-.png files. Those get the appdata copy +# only — there is exactly one stack row and it already has its icon. See +# main.yml, which is where the names are resolved. +- name: Sync stack icons to the Unraid host + when: stack_icon_dest | length > 0 or stack_service_icon_names | length > 0 + block: + - name: Ensure the icon directory exists + ansible.builtin.file: + path: "{{ unraid_icons_root }}" + state: directory + mode: "0755" + + - name: Copy stack icon for the container labels + ansible.builtin.copy: + src: "{{ stack_local_dir }}/ansible/unraid/icon.png" + dest: "{{ stack_icon_dest }}" + mode: "0644" + when: stack_icon_dest | length > 0 + + # Into the project folder on the flash drive, next to the compose file. + # `copy` is checksum-based, so a redeploy that hasn't changed the icon + # writes nothing — this costs one flash write per icon, not one per run. + - name: Copy stack icon for the Compose Manager stack row + ansible.builtin.copy: + src: "{{ stack_local_dir }}/ansible/unraid/icon.png" + dest: "{{ stack_remote_dir }}/icon.png" + mode: "0644" + when: stack_icon_dest | length > 0 + + # Appdata only, and no flash-drive copy: these label individual containers, + # which is the one thing the Compose Manager stack row isn't. + - name: Copy per-service icons for the container labels + ansible.builtin.copy: + src: "{{ stack_local_dir }}/ansible/unraid/icon-{{ item }}.png" + dest: "{{ unraid_icons_root }}/{{ stack.name }}-{{ item }}.png" + mode: "0644" + loop: "{{ stack_service_icon_names }}" + +- name: Look up stack secrets from Vault + ansible.builtin.set_fact: + vault_secrets: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + stack.vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + no_log: true + +- name: Render .env file + ansible.builtin.template: + src: env.j2 + dest: "{{ stack_remote_dir }}/.env" + mode: "0600" + vars: + # The role-computed icon paths first, then portable values, then + # Unraid-specific overrides on top; Vault secrets win over all three (see + # env.j2). STACK_ICON and the per-service STACK_ICON_ keys sit at + # the bottom so either vars.yml layer can replace one with a hosted URL, + # and STACK_ICON is always defined so Compose never warns about an unset + # variable in the label block. + # + # STACK_ICON_ only exists when the matching icon-.png is + # committed. A compose file that interpolates one without the file present + # gets Compose's empty-value warning and the placeholder icon — set it in + # vars.yml instead if the icon is meant to be a URL. + env_defaults: >- + {{ {'STACK_ICON': stack_icon_dest} + | combine(stack_service_icons) + | combine(stack_common_vars.env_defaults | default({})) + | combine(stack_platform_vars.env_defaults | default({})) }} + no_log: true + +- name: Provision application database + when: stack.db is defined + no_log: true + block: + - name: Look up Postgres superuser credentials from Vault + ansible.builtin.set_fact: + pg_admin_secrets: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + stack.db.admin_vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + + - name: Ensure application database role exists + community.postgresql.postgresql_user: + name: "{{ stack.db.user }}" + password: "{{ vault_secrets[stack.db.password_vault_key] }}" + login_host: "{{ stack.db.provision_host }}" + login_port: "{{ stack.db.provision_port }}" + login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}" + login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}" + state: present + delegate_to: localhost + become: false + + # Role first, then the database with `owner:` — see the same pair of + # tasks in roles/lxc_app for the full reasoning. The short version is + # that `priv: ALL` on postgresql_user was removed from + # community.postgresql in 4.0.0, and ownership is what the replacement + # should have been anyway. + # + # This instance is pinned to Postgres 13, so unlike the Proxmox one it + # was never *also* broken by the Postgres 15 `public` schema change — + # the removal of `priv` is the only thing that forces the edit here. + # Kept identical to the other two roles regardless: the point of the + # three app roles sharing this block is that an app moving between + # platforms gets the same database either way. + - name: Ensure application database exists + community.postgresql.postgresql_db: + name: "{{ stack.db.name }}" + owner: "{{ stack.db.user }}" + login_host: "{{ stack.db.provision_host }}" + login_port: "{{ stack.db.provision_port }}" + login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}" + login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}" + state: present + delegate_to: localhost + become: false + +- name: Deploy stack with Docker Compose + community.docker.docker_compose_v2: + project_src: "{{ stack_remote_dir }}" + state: present + pull: policy diff --git a/build/config/ansible/roles/compose_stack/tasks/main.yml b/build/config/ansible/roles/compose_stack/tasks/main.yml new file mode 100644 index 0000000..0e58c46 --- /dev/null +++ b/build/config/ansible/roles/compose_stack/tasks/main.yml @@ -0,0 +1,75 @@ +--- +# Converges one stack (one loop iteration of `stack` from playbooks/unraid.yml) +# to the state its host_vars entry asks for. +# +# `state: present` (the default) deploys; `state: absent` tears down. Removal +# has to be asked for explicitly because Ansible keeps no record of what it +# deployed last run — deleting a stack from a host's `stacks:` list only stops +# the loop visiting it, leaving the containers running and unmanaged on the +# host forever. So the entry stays put as a tombstone, with `state: absent`, +# until the teardown has actually been applied. +# +# Unraid-only by design. Proxmox guests install apps natively rather than as +# Compose stacks — see the `lxc_app` role. + +- name: Set stack facts + ansible.builtin.set_fact: + stack_local_dir: "{{ repo_root }}/src/{{ stack.src }}" + stack_remote_dir: "{{ compose_projects_root }}/{{ stack.name }}" + stack_state: "{{ stack.state | default('present') }}" + +- name: Validate requested stack state + ansible.builtin.assert: + that: stack_state in ['present', 'absent'] + fail_msg: >- + Stack '{{ stack.name }}' has state '{{ stack_state }}'; expected + 'present' or 'absent'. + quiet: true + +# Which icons this stack ships, worked out from the repo checkout on the +# controller rather than from anything on the host — `is exists` and the +# `fileglob` lookup both evaluate locally. Computed here rather than in +# deploy.yml because both branches need it: deploy copies these files out, +# remove deletes them again. +# +# Two naming conventions, because Unraid has two icon consumers (see +# deploy.yml for the full explanation): +# +# icon.png the stack — Compose Manager's project row, plus the +# `net.unraid.docker.icon` label of whichever service +# interpolates ${STACK_ICON}. +# icon-.png one container — becomes ${STACK_ICON_}, so a +# multi-container stack can label each of its services +# with its own image instead of sharing the stack's. +# +# `` is the compose service name; the variable is it uppercased with +# `-` folded to `_` (icon-shelfarr-libation.png → STACK_ICON_SHELFARR_LIBATION). +- name: Set stack icon facts + ansible.builtin.set_fact: + # Empty when the stack ships no icon.png — an empty label is what the + # webgui already assumes, so it falls back to the placeholder as before. + stack_icon_dest: >- + {{ ((stack_local_dir ~ '/ansible/unraid/icon.png') is exists) + | ternary(unraid_icons_root ~ '/' ~ stack.name ~ '.png', '') }} + stack_service_icon_names: "{{ _service_icon_names }}" + stack_service_icons: >- + {{ dict(_service_icon_names + | map('upper') | map('replace', '-', '_') + | map('regex_replace', '^(.+)$', 'STACK_ICON_\1') + | zip(_service_icon_names + | map('regex_replace', '^(.+)$', + unraid_icons_root ~ '/' ~ stack.name ~ '-\1.png'))) }} + vars: + _service_icon_names: >- + {{ query('fileglob', stack_local_dir ~ '/ansible/unraid/icon-*.png') + | map('basename') + | map('regex_replace', '^icon-(.+)\.png$', '\1') + | map('lower') | list }} + +- name: Deploy stack + ansible.builtin.include_tasks: deploy.yml + when: stack_state == 'present' + +- name: Remove stack + ansible.builtin.include_tasks: remove.yml + when: stack_state == 'absent' diff --git a/build/config/ansible/roles/compose_stack/tasks/remove.yml b/build/config/ansible/roles/compose_stack/tasks/remove.yml new file mode 100644 index 0000000..0f73bab --- /dev/null +++ b/build/config/ansible/roles/compose_stack/tasks/remove.yml @@ -0,0 +1,106 @@ +--- +# Tears one stack down off the current Unraid host: `docker compose down`, +# then delete the project folder, then optionally drop its database. +# +# Included from main.yml when the stack's `state:` is `absent`. +# +# Ordering matters and is not obvious: `docker compose down` reads the compose +# file to know what it is removing, so the project folder has to survive until +# after that step. Delete the folder (or the stack's `src/` tree) first and +# there is nothing left to tell Docker what belonged to the project — the +# containers and networks have to be cleaned up by hand instead. +# +# What `down` takes with it, and what it deliberately doesn't: +# - removed: the containers, and any network the project itself created +# - kept: bind mounts. Appdata under /mnt/user/appdata/ survives, so +# a removal is reversible by flipping `state:` back to `present`. +# - kept: external networks (caddy-net, unraid_shared) — they belong to +# another stack or to Unraid, not to this project. +# - opt-in: named volumes (`remove_volumes: true`) and images +# (`remove_images: local` or `all`). +# +# Not touched at all: the stack's `homelab/` path in Vault. That has its +# own lifecycle and no reason to be destroyed by a redeployable teardown. + +- name: Check whether the stack's project directory is still present + ansible.builtin.stat: + path: "{{ stack_remote_dir }}/docker-compose.yml" + register: stack_compose_file + +- name: Tear down stack with Docker Compose + community.docker.docker_compose_v2: + project_src: "{{ stack_remote_dir }}" + state: absent + # Named volumes and images are destructive beyond "stop running this + # here", so they stay opt-in per stack rather than being implied by + # `state: absent`. + remove_volumes: "{{ stack.remove_volumes | default(false) }}" + remove_images: "{{ stack.remove_images | default(omit) }}" + remove_orphans: true + # Absent already — a re-run after a successful removal, or a stack that was + # never deployed to this host. Both are the desired end state, not an error. + when: stack_compose_file.stat.exists + +- name: Remove stack project directory + ansible.builtin.file: + path: "{{ stack_remote_dir }}" + state: absent + +# Unlike appdata, these are not state — they're copies of PNGs committed in +# the repo, put there purely so the Docker page had something to render. +# Nothing is lost by deleting them and a redeploy puts them back, so they +# aren't opt-in the way volumes and images are. +# +# Listed from the repo checkout (main.yml) rather than globbed on the host: +# `{{ stack.name }}-*.png` would also match the icons of any stack whose name +# starts with this one's. +- name: Remove stack icons + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: "{{ [unraid_icons_root ~ '/' ~ stack.name ~ '.png'] + + stack_service_icons.values() | list }}" + +- name: Drop application database + when: + - stack.db is defined + - stack.remove_database | default(false) + no_log: true + block: + # Only the Postgres superuser credentials are needed here. The stack's own + # Vault secrets are not read, so a teardown still works after its + # `homelab/` path has been deleted. + - name: Look up Postgres superuser credentials from Vault + ansible.builtin.set_fact: + pg_admin_secrets: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + stack.db.admin_vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + + # Database before role: Postgres refuses to drop a role that still owns + # objects, and the role owns this database. + - name: Drop application database + community.postgresql.postgresql_db: + name: "{{ stack.db.name }}" + login_host: "{{ stack.db.provision_host }}" + login_port: "{{ stack.db.provision_port }}" + login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}" + login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}" + state: absent + delegate_to: localhost + become: false + + - name: Drop application database role + community.postgresql.postgresql_user: + name: "{{ stack.db.user }}" + login_host: "{{ stack.db.provision_host }}" + login_port: "{{ stack.db.provision_port }}" + login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}" + login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}" + state: absent + delegate_to: localhost + become: false diff --git a/build/config/ansible/roles/compose_stack/templates/env.j2 b/build/config/ansible/roles/compose_stack/templates/env.j2 new file mode 100644 index 0000000..108e556 --- /dev/null +++ b/build/config/ansible/roles/compose_stack/templates/env.j2 @@ -0,0 +1,7 @@ +{# Non-secret defaults first, then Vault secrets — secrets win on key clashes. #} +{% for key, value in (env_defaults | default({})).items() %} +{{ key }}="{{ value }}" +{% endfor %} +{% for key, value in (vault_secrets | default({})).items() %} +{{ key }}="{{ value }}" +{% endfor %} diff --git a/build/config/ansible/roles/k3s_app/defaults/main.yml b/build/config/ansible/roles/k3s_app/defaults/main.yml new file mode 100644 index 0000000..36fccbd --- /dev/null +++ b/build/config/ansible/roles/k3s_app/defaults/main.yml @@ -0,0 +1,34 @@ +--- +# Ansible-side knobs for roles/k3s_app — the k3s equivalent of +# compose_stack (Unraid) and lxc_app (Proxmox): the generic role that +# deploys one *application* onto the homelab-utils cluster, as opposed to +# the k3s_* roles that deploy one *cluster service*. +# +# The distinction is worth stating, because the file layout looks similar. +# roles/k3s_metallb, k3s_monitoring, k3s_postgres and k3s_cert_manager each +# hard-code a single thing the cluster itself needs, with their config in +# defaults/ and group_vars. This role hard-codes nothing: it's driven by the +# `k3s_apps:` list in inventory/group_vars/k3s_cluster.yml and reads each +# app's config out of src//, the same src/ tree Unraid and Proxmox +# already deploy from. An app that moves between platforms gets a new +# platform vars file, not a rewrite — see CLAUDE.md → "How config is +# layered". +# +# Everything reaches the cluster the same way every other k3s role does: +# rendering a file into k3s's auto-deploying manifests directory and letting +# k3s's own controllers reconcile it. No helm binary, no kubeconfig, no +# kubernetes.core collection on the controller. + +# Where k3s watches for manifests to apply. Anything written here is applied +# by k3s's deploy controller, and — importantly for remove.yml — anything +# *deleted* from here has its resources torn down again, because the +# controller tracks what each file created via an Addon CR. +k3s_manifests_dir: /var/lib/rancher/k3s/server/manifests + +vault_kv_mount: kv + +# Which directory under src//ansible/ holds the Kubernetes deployment. +# "kubernetes" rather than "k3s": what's in there is plain Kubernetes plus +# k3s's HelmChart CR, and an app moved to a different distribution would +# keep most of it. +k3s_app_platform_dir: kubernetes diff --git a/build/config/ansible/roles/k3s_app/tasks/deploy.yml b/build/config/ansible/roles/k3s_app/tasks/deploy.yml new file mode 100644 index 0000000..6b0211f --- /dev/null +++ b/build/config/ansible/roles/k3s_app/tasks/deploy.yml @@ -0,0 +1,170 @@ +--- +# Deploys one app to the k3s cluster: layer its config, provision its +# database if it declares one, render its secrets into a Secret, then render +# whatever manifests the app ships into k3s's auto-deploying directory. +# +# The shape deliberately mirrors compose_stack/deploy.yml, because the +# separation it enforces is the same one: config that's committed, secrets +# that never are. On Unraid the split is "static docker-compose.yml +# referencing ${VAR}" + "rendered .env"; here it's "manifests carrying only +# non-secret config" + "a rendered Secret the manifests reference by name". +# In both cases the committed half is safe to read and the generated half +# never lands in git. +# +# Included from main.yml when the app's `state:` is `present`; the facts it +# relies on (app_local_dir) are set there. + +- name: Load portable app variables + ansible.builtin.include_vars: + file: "{{ app_local_dir }}/common/vars.yml" + name: app_common_vars + +- name: Load Kubernetes-specific app variables + ansible.builtin.include_vars: + file: "{{ app_local_dir }}/ansible/{{ k3s_app_platform_dir }}/vars.yml" + name: app_platform_vars + +# Portable values first, platform overrides on top — same precedence as +# compose_stack's env_defaults merge, and the same reason: an app's ports and +# database name don't change with the platform, its storage class and +# ingress class do. Vault secrets are *not* merged in here; they go to the +# Secret in a separate task below, so a manifest template can never +# accidentally interpolate one into a world-readable file. +- name: Merge app configuration + ansible.builtin.set_fact: + app_config: >- + {{ (app_common_vars.env_defaults | default({})) + | combine(app_platform_vars.env_defaults | default({})) }} + +- name: Look up app secrets from Vault + ansible.builtin.set_fact: + app_vault_secrets: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + app.vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + no_log: true + +# --- Database --------------------------------------------------------------- +# +# Identical to compose_stack's block, pointed at a different Postgres. That +# it *can* be identical is the whole point of roles/k3s_postgres publishing +# the CNPG primary on a MetalLB LoadBalancer: `community.postgresql` runs on +# the controller (delegate_to: localhost) and needs a real host:port, which +# an in-cluster ClusterIP Service isn't. See CLAUDE.md → "Key decisions" — +# this is the deploy path that entry says is missing. +# +# provision_host is the LoadBalancer address, not the -rw ClusterIP; the app +# itself still talks to the ClusterIP by DNS (see the app's vars.yml), so +# the LAN-facing address is only ever used by the controller at deploy time. +- name: Provision application database + when: app.db is defined + no_log: true + block: + - name: Look up Postgres superuser credentials from Vault + ansible.builtin.set_fact: + pg_admin_secrets: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + app.db.admin_vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + + - name: Ensure application database role exists + community.postgresql.postgresql_user: + name: "{{ app.db.user }}" + password: "{{ app_vault_secrets[app.db.password_vault_key] }}" + login_host: "{{ app.db.provision_host }}" + login_port: "{{ app.db.provision_port }}" + login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}" + login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}" + state: present + delegate_to: localhost + become: false + + # After the role, not before, and with owner: — unlike compose_stack, + # which creates the database first and then grants on it. The difference + # is that this database may be restored into from a pg_dump taken + # elsewhere (see docs/authentik-migration.md): a dump recreates objects + # with their original ownership, which only resolves if the owning role + # already exists and owns the database. Creating it owner-less and + # granting after works for an empty database and quietly leaves a + # restored one owned by postgres. + - name: Ensure application database exists + community.postgresql.postgresql_db: + name: "{{ app.db.name }}" + owner: "{{ app.db.user }}" + login_host: "{{ app.db.provision_host }}" + login_port: "{{ app.db.provision_port }}" + login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}" + login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}" + state: present + delegate_to: localhost + become: false + +# --- Manifests -------------------------------------------------------------- + +# Namespace + Secret in one file, so the namespace an app's manifests target +# is guaranteed to be created by something even if the app ships only a +# HelmChart CR (whose createNamespace: true fires too late for a Secret the +# chart's pods mount). Ordering between files doesn't otherwise matter here: +# k3s's deploy controller retries a manifest whose namespace or CRDs don't +# exist yet rather than failing once and giving up — the same property +# roles/k3s_postgres and roles/k3s_metallb already rely on. +# +# 0600 and no_log because this one carries every value from the app's Vault +# path. Note this is on-disk protection on the node only: the Secret's +# contents are then base64 in etcd like any Kubernetes Secret, which is the +# same trust boundary the rest of this cluster already assumes. +- name: Render the app namespace and secrets manifest + ansible.builtin.template: + src: app-secrets.yaml.j2 + dest: "{{ k3s_manifests_dir }}/{{ app.name }}-secrets.yaml" + owner: root + group: root + mode: "0600" + become: true + no_log: true + +# Whatever the app ships — a HelmChart CR, an Ingress, a PVC, a Certificate. +# Enumerated from the repo checkout on the controller (fileglob evaluates +# locally), so adding a manifest to an app means dropping a .yaml.j2 next to +# the others, with no role change. +# +# Prefixed with the app name on the node, because every app's manifests share +# one flat directory there. Name the files for their content and not for the +# app (helmchart.yaml.j2, not authentik.helmchart.yaml.j2) — the prefix is +# added here, and an app-named file stutters into authentik-authentik.yaml. +- name: Find the app's Kubernetes manifest templates + ansible.builtin.set_fact: + app_manifest_templates: >- + {{ query('fileglob', + app_local_dir ~ '/ansible/' ~ k3s_app_platform_dir ~ '/*.yaml.j2') + | sort }} + +- name: Fail fast if the app ships no manifests + ansible.builtin.assert: + that: app_manifest_templates | length > 0 + fail_msg: >- + App '{{ app.name }}' has no *.yaml.j2 under + src/{{ app.src }}/ansible/{{ k3s_app_platform_dir }}/ — nothing to + deploy. An app on this platform needs at least one manifest (normally + a HelmChart CR). + quiet: true + +- name: Render the app's Kubernetes manifests + ansible.builtin.template: + src: "{{ item }}" + dest: >- + {{ k3s_manifests_dir }}/{{ app.name }}-{{ + item | basename | regex_replace('\.j2$', '') }} + owner: root + group: root + mode: "0644" + loop: "{{ app_manifest_templates }}" + become: true diff --git a/build/config/ansible/roles/k3s_app/tasks/main.yml b/build/config/ansible/roles/k3s_app/tasks/main.yml new file mode 100644 index 0000000..353ba8b --- /dev/null +++ b/build/config/ansible/roles/k3s_app/tasks/main.yml @@ -0,0 +1,37 @@ +--- +# Converges one app (one loop iteration of `app` from playbooks/k3s.yml) to +# the state its k3s_apps: entry asks for. +# +# `state: present` (the default) deploys; `state: absent` tears down. Same +# explicit-removal rule as compose_stack, and for the same reason: Ansible +# keeps no record of what it deployed last run, so deleting an app from +# `k3s_apps:` only stops the loop visiting it — the workload keeps running, +# unmanaged, until something says `absent`. The entry stays as a tombstone. +# +# Unlike compose_stack, removal here does tear the workload down completely +# on the first pass, because k3s's deploy controller owns the resources a +# manifest created and garbage-collects them when the file goes away. What +# it does *not* touch, by the same "a default teardown should be +# reversible" reasoning compose_stack applies: the app's database, its +# PersistentVolumeClaims, and its Vault path. + +- name: Set app facts + ansible.builtin.set_fact: + app_local_dir: "{{ repo_root }}/src/{{ app.src }}" + app_state: "{{ app.state | default('present') }}" + +- name: Validate requested app state + ansible.builtin.assert: + that: app_state in ['present', 'absent'] + fail_msg: >- + App '{{ app.name }}' has state '{{ app_state }}'; expected 'present' + or 'absent'. + quiet: true + +- name: Deploy app + ansible.builtin.include_tasks: deploy.yml + when: app_state == 'present' + +- name: Remove app + ansible.builtin.include_tasks: remove.yml + when: app_state == 'absent' diff --git a/build/config/ansible/roles/k3s_app/tasks/remove.yml b/build/config/ansible/roles/k3s_app/tasks/remove.yml new file mode 100644 index 0000000..01f6f85 --- /dev/null +++ b/build/config/ansible/roles/k3s_app/tasks/remove.yml @@ -0,0 +1,60 @@ +--- +# Tears one app off the cluster: delete the manifests it was deployed from +# and let k3s's deploy controller garbage-collect what they created. +# +# This is the one place k3s is *less* work than Compose. `compose_stack` +# has to run `docker compose down` and therefore needs the compose file to +# still be on disk to know what it's tearing down (which is why removing an +# app there is a two-pass job — see CLAUDE.md → "Removing an app"). Here the +# deploy controller already tracks which resources each manifest file +# created, via the Addon CR it writes alongside them, so deleting the file +# is the teardown. src// can be deleted in the same commit. +# +# Deliberately *not* removed, same reasoning as compose_stack's opt-in +# flags — a default teardown should be reversible: +# +# - the app's database and role on the shared Postgres +# - its PersistentVolumeClaims (the app's manifests own those; if the +# chart's PVCs carry a Helm ownership annotation they go with the +# HelmChart CR, so check `kubectl -n get pvc` after) +# - its Vault path +# - the namespace, which the secrets manifest below creates but the +# controller will only remove if nothing else landed in it +# +# There's no `remove_database`/`remove_volumes` equivalent yet. Add one the +# day it's actually wanted rather than guessing at the shape now. + +- name: Find the app's Kubernetes manifest templates + ansible.builtin.set_fact: + app_manifest_templates: >- + {{ query('fileglob', + app_local_dir ~ '/ansible/' ~ k3s_app_platform_dir ~ '/*.yaml.j2') + | sort }} + +# The secrets manifest first, then the app's own. Order is cosmetic — the +# controller reconciles each file's removal independently — but removing the +# workload's namespace/Secret last would leave pods briefly running without +# the credentials they were started with, and losing them noisily in a log +# is worse than losing them quietly. +- name: Remove the app's Kubernetes manifests + ansible.builtin.file: + path: >- + {{ k3s_manifests_dir }}/{{ app.name }}-{{ + item | basename | regex_replace('\.j2$', '') }} + state: absent + loop: "{{ app_manifest_templates }}" + become: true + +- name: Remove the app namespace and secrets manifest + ansible.builtin.file: + path: "{{ k3s_manifests_dir }}/{{ app.name }}-secrets.yaml" + state: absent + become: true + +- name: Report what removal left behind + ansible.builtin.debug: + msg: >- + App '{{ app.name }}' manifests removed; k3s will garbage-collect the + resources they created. Its database, PVCs and Vault path were left + alone on purpose — remove those by hand if the teardown is meant to be + permanent. diff --git a/build/config/ansible/roles/k3s_app/templates/app-secrets.yaml.j2 b/build/config/ansible/roles/k3s_app/templates/app-secrets.yaml.j2 new file mode 100644 index 0000000..8926c09 --- /dev/null +++ b/build/config/ansible/roles/k3s_app/templates/app-secrets.yaml.j2 @@ -0,0 +1,35 @@ +{# + Managed by Ansible (roles/k3s_app) — do not edit on the node. + + The k3s counterpart of compose_stack's rendered .env: every key from the + app's Vault path, verbatim, in one Secret the app's manifests reference by + name. Keys are not renamed or filtered on the way through, which is what + keeps this role generic — an app decides what its environment looks like by + choosing its Vault keys, exactly as it does on Unraid, and the role stays + ignorant of any particular app's variables. + + The intended consumer is an envFrom/secretRef in the app's own manifest, + so the values become environment variables without ever being named in a + committed file. See src/authentik/ansible/kubernetes/ for the worked + example. + + Namespace lives here rather than in the app's manifests so it's guaranteed + to exist before anything mounts this Secret — a HelmChart CR's + createNamespace: true happens when the chart installs, which is after the + helm-controller job needs somewhere to put it. +-#} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ app_config.K8S_NAMESPACE }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ app.name }}-secrets + namespace: {{ app_config.K8S_NAMESPACE }} +type: Opaque +stringData: +{% for key, value in (app_vault_secrets | default({})) | dictsort %} + {{ key }}: {{ value | string | to_json }} +{% endfor %} diff --git a/build/config/ansible/roles/k3s_cert_manager/defaults/main.yml b/build/config/ansible/roles/k3s_cert_manager/defaults/main.yml new file mode 100644 index 0000000..d760e5f --- /dev/null +++ b/build/config/ansible/roles/k3s_cert_manager/defaults/main.yml @@ -0,0 +1,73 @@ +--- +# Ansible-side knobs for roles/k3s_cert_manager — cert-manager, which issues +# and renews the TLS certificates Ingresses on this cluster serve. Installed +# via k3s's own bundled helm-controller, same mechanism as roles/k3s_metallb, +# k3s_monitoring and k3s_postgres (see tasks/main.yml) — no helm binary or +# extra Ansible collection needed on the controller. +# +# This is the fourth cluster service, and the first one that exists purely +# for apps rather than for the cluster: nothing here needs a certificate, +# roles/k3s_app's tenants do. It's still a cluster service rather than an +# app, because a certificate issuer is shared infrastructure in the same way +# the shared Postgres is — one ClusterIssuer, every app's Ingress annotates +# itself against it. + +k3s_cert_manager_namespace: cert-manager + +k3s_cert_manager_chart_repo: https://charts.jetstack.io + +# Pinned, same reasoning as every other chart version here — a rebuild +# months from now should reproduce today's install, not whatever's newest. +# Bump deliberately; check the current release first at +# https://github.com/cert-manager/cert-manager/releases. +k3s_cert_manager_chart_version: "v1.21.1" + +# The name every app's Ingress annotates itself with +# (cert-manager.io/cluster-issuer: ). A ClusterIssuer rather than a +# per-namespace Issuer, so an app doesn't need its own copy of the ACME +# account and DNS credentials in its own namespace. +k3s_cert_manager_issuer_name: letsencrypt + +# Let's Encrypt's production directory. Swap for the staging URL +# (https://acme-staging-v02.api.letsencrypt.org/directory) while working out +# a solver configuration: production has a hard rate limit of 5 failed +# validations per account/hostname/hour, and burning it means waiting rather +# than retrying. Changing this changes which ACME account the issuer uses, so +# existing certificates are re-issued rather than renewed. +k3s_cert_manager_acme_server: https://acme-v02.api.letsencrypt.org/directory + +# The address Let's Encrypt sends expiry warnings to. No default — an ACME +# account is registered against it, so a wrong value is worth failing on +# rather than guessing. Set in inventory/group_vars/k3s_cluster.yml. +k3s_cert_manager_acme_email: "" + +# Where the ACME account's private key is kept. cert-manager generates it on +# first registration; it is *not* the certificate key, and losing it means +# re-registering rather than losing certificates. +k3s_cert_manager_acme_key_secret: letsencrypt-account-key + +# --- Solver ----------------------------------------------------------------- +# +# How cert-manager proves control of the domain. DNS-01 by default, not +# HTTP-01, because HTTP-01 requires Let's Encrypt to reach this cluster from +# the internet on port 80 — true only if the LAN is port-forwarded, and this +# cluster deliberately isn't. DNS-01 needs no inbound path at all, and is the +# only option that can issue wildcards. +# +# The provider block is left open rather than hard-coding Cloudflare, since +# it's the one part of this role that depends on who runs the DNS. Set +# k3s_cert_manager_solver in inventory/group_vars/k3s_cluster.yml to whatever +# your provider's stanza looks like in cert-manager's docs +# (https://cert-manager.io/docs/configuration/acme/dns01/); it's rendered +# into the ClusterIssuer's solvers list as-is. Any secret it references +# should name the Secret this role renders from Vault (below), so no token is +# ever committed. tasks/main.yml fails fast while this is empty. +k3s_cert_manager_solver: {} + +# The Vault path holding the DNS provider's API credentials. Every key at +# this path becomes a key in a Secret named after it in the cert-manager +# namespace, the same "Vault keys pass through verbatim" rule roles/k3s_app +# uses — so the solver above references whichever key name you stored. +# Leave empty for a solver that needs no credentials (e.g. HTTP-01). +k3s_cert_manager_vault_path: homelab/k3s-cert-manager +k3s_cert_manager_credentials_secret: cert-manager-dns-credentials diff --git a/build/config/ansible/roles/k3s_cert_manager/tasks/main.yml b/build/config/ansible/roles/k3s_cert_manager/tasks/main.yml new file mode 100644 index 0000000..378c622 --- /dev/null +++ b/build/config/ansible/roles/k3s_cert_manager/tasks/main.yml @@ -0,0 +1,75 @@ +--- +# Drops a HelmChart CR for cert-manager, plus a ClusterIssuer and the DNS +# credentials it solves with, into k3s's auto-deploying manifests directory; +# k3s's bundled helm-controller and deploy controller reconcile them — same +# mechanism and same two-manifest shape as roles/k3s_metallb (chart CR + +# plain config manifest referencing CRDs the chart hasn't installed yet). +# +# The ClusterIssuer is exactly that case: cert-manager.io/v1 doesn't exist +# until the chart has installed, so this manifest is unappliable at the +# moment it's written. That's fine and deliberate — k3s's deploy controller +# retries a manifest referencing not-yet-existing CRDs until they show up, +# rather than failing once and giving up. + +- name: Fail fast if cert-manager is not configured + ansible.builtin.assert: + that: + - k3s_cert_manager_acme_email | length > 0 + - k3s_cert_manager_solver | length > 0 + fail_msg: >- + k3s_cert_manager_acme_email and/or k3s_cert_manager_solver are unset — + set both in inventory/group_vars/k3s_cluster.yml before running + playbooks/k3s.yml. An ACME account is registered against the email, and + without a solver the ClusterIssuer would be created but never able to + prove domain control, leaving every Certificate pending indefinitely + rather than failing loudly. + quiet: true + run_once: true + +- name: Deploy the cert-manager HelmChart manifest + ansible.builtin.template: + src: cert-manager.helmchart.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/cert-manager.yaml + owner: root + group: root + mode: "0644" + become: true + +# Before the ClusterIssuer, because that's what references it. Ordering +# between files isn't actually enforced (see the header) — this is for the +# reader, not the controller. +- name: Look up the DNS provider credentials from Vault + when: k3s_cert_manager_vault_path | length > 0 + ansible.builtin.set_fact: + cert_manager_vault_secrets: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + k3s_cert_manager_vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + no_log: true + +# Separate file and mode from everything else here, same split +# roles/k3s_postgres makes: this is the only manifest carrying a credential, +# so it's the only one that isn't safe at 0644. +- name: Deploy the DNS provider credentials Secret + when: k3s_cert_manager_vault_path | length > 0 + ansible.builtin.template: + src: dns-credentials.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/cert-manager-dns-credentials.yaml + owner: root + group: root + mode: "0600" + become: true + no_log: true + +- name: Deploy the ClusterIssuer manifest + ansible.builtin.template: + src: cluster-issuer.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/cert-manager-cluster-issuer.yaml + owner: root + group: root + mode: "0644" + become: true diff --git a/build/config/ansible/roles/k3s_cert_manager/templates/cert-manager.helmchart.yaml.j2 b/build/config/ansible/roles/k3s_cert_manager/templates/cert-manager.helmchart.yaml.j2 new file mode 100644 index 0000000..a83ce1a --- /dev/null +++ b/build/config/ansible/roles/k3s_cert_manager/templates/cert-manager.helmchart.yaml.j2 @@ -0,0 +1,59 @@ +{# + Managed by Ansible (roles/k3s_cert_manager) — do not edit on the node. + + A HelmChart CR, same pattern as roles/k3s_metallb and roles/k3s_monitoring: + k3s's bundled helm-controller reconciles it, the same mechanism it uses to + install its own Traefik. +-#} +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: cert-manager + namespace: kube-system +spec: + chart: cert-manager + repo: {{ k3s_cert_manager_chart_repo }} + version: "{{ k3s_cert_manager_chart_version }}" + targetNamespace: {{ k3s_cert_manager_namespace }} + createNamespace: true + valuesContent: |- + # The chart installs its own CRDs. Without this they'd have to be applied + # separately before the chart, which the helm-controller gives no ordering + # hook for — and the ClusterIssuer this role also renders would then never + # become appliable. + crds: + enabled: true + + # Sized for a Raspberry Pi 4, same reasoning as k3s_monitoring_values and + # k3s_postgres_resources. cert-manager is idle almost all of the time — + # it wakes to renew a certificate every 60 days — so these are set for + # "doesn't get OOM-killed during a renewal", not for throughput. + resources: + requests: + cpu: 10m + memory: 64Mi + limits: + memory: 128Mi + webhook: + resources: + requests: + cpu: 10m + memory: 64Mi + limits: + memory: 128Mi + cainjector: + resources: + requests: + cpu: 10m + memory: 96Mi + limits: + memory: 192Mi + + # DNS-01 self-check queries the authoritative nameservers directly rather + # than going through the cluster's resolver. Without this, CoreDNS + # forwards to whatever the LAN's DNS is, and a split-horizon setup — + # where the internal view of the zone doesn't carry the _acme-challenge + # TXT record the public view does — makes cert-manager wait for a record + # it will never see, until the order times out. + dns01RecursiveNameservers: "1.1.1.1:53,9.9.9.9:53" + dns01RecursiveNameserversOnly: true diff --git a/build/config/ansible/roles/k3s_cert_manager/templates/cluster-issuer.yaml.j2 b/build/config/ansible/roles/k3s_cert_manager/templates/cluster-issuer.yaml.j2 new file mode 100644 index 0000000..d433719 --- /dev/null +++ b/build/config/ansible/roles/k3s_cert_manager/templates/cluster-issuer.yaml.j2 @@ -0,0 +1,30 @@ +{# + Managed by Ansible (roles/k3s_cert_manager) — do not edit on the node. + + Plain manifest, not a HelmChart — same pattern as + roles/k3s_metallb/templates/metallb-config.yaml.j2 and + roles/k3s_postgres/templates/postgres-cluster.yaml.j2: this references a + CRD (cert-manager.io) that only exists once the chart alongside it has + installed, and k3s's deploy controller retries rather than giving up. + + ClusterIssuer rather than Issuer so it's usable from every namespace — + an app's Ingress just annotates itself with + `cert-manager.io/cluster-issuer: {{ k3s_cert_manager_issuer_name }}` and + needs no ACME account or DNS credentials of its own. +-#} +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: {{ k3s_cert_manager_issuer_name }} +spec: + acme: + server: {{ k3s_cert_manager_acme_server }} + email: {{ k3s_cert_manager_acme_email }} + # Where cert-manager keeps the ACME *account* key it generates on first + # registration — not any certificate's key. In the cert-manager namespace + # because a ClusterIssuer's secrets always resolve there, regardless of + # which namespace the Certificate using it lives in. + privateKeySecretRef: + name: {{ k3s_cert_manager_acme_key_secret }} + solvers: +{{ [k3s_cert_manager_solver] | to_nice_yaml(indent=2) | indent(6, first=true) }} diff --git a/build/config/ansible/roles/k3s_cert_manager/templates/dns-credentials.yaml.j2 b/build/config/ansible/roles/k3s_cert_manager/templates/dns-credentials.yaml.j2 new file mode 100644 index 0000000..8713644 --- /dev/null +++ b/build/config/ansible/roles/k3s_cert_manager/templates/dns-credentials.yaml.j2 @@ -0,0 +1,23 @@ +{# + Managed by Ansible (roles/k3s_cert_manager) — do not edit on the node. + + The DNS provider API credentials the ACME DNS-01 solver authenticates with, + passed through from Vault verbatim — same rule roles/k3s_app's Secret + follows, so the key names in k3s_cert_manager_solver are whatever you chose + when populating the Vault path, and this role stays ignorant of which + provider is in use. + + In the cert-manager namespace because a ClusterIssuer resolves every Secret + it references there, never in the namespace of the Certificate being + issued. +-#} +apiVersion: v1 +kind: Secret +metadata: + name: {{ k3s_cert_manager_credentials_secret }} + namespace: {{ k3s_cert_manager_namespace }} +type: Opaque +stringData: +{% for key, value in (cert_manager_vault_secrets | default({})) | dictsort %} + {{ key }}: {{ value | string | to_json }} +{% endfor %} diff --git a/build/config/ansible/roles/k3s_maintenance/tasks/main.yml b/build/config/ansible/roles/k3s_maintenance/tasks/main.yml new file mode 100644 index 0000000..57fe07e --- /dev/null +++ b/build/config/ansible/roles/k3s_maintenance/tasks/main.yml @@ -0,0 +1,100 @@ +--- +# Rolling "reboot this node if unattended-upgrades left one pending" for one +# k3s node — one iteration of the serial: 1 loop in +# playbooks/k3s_maintenance.yml. roles/unattended_upgrades installs updates on +# its own schedule with Automatic-Reboot disabled, so a kernel/library update +# can sit applied-but-inactive on a node indefinitely; this is what actually +# reboots it — cordoned and drained first, so workloads move off before the +# node disappears rather than during. +# +# kubectl commands are delegated to the control-plane node and run as +# `k3s kubectl`, k3s's own bundled client — no separate kubectl install or +# local kubeconfig needed. Works the same whether the node having its turn +# right now *is* the control plane: delegating to itself, over SSH, before it +# reboots itself. + +- name: Sanity-check there is exactly one control-plane node + ansible.builtin.assert: + that: groups['k3s_control_plane'] | length == 1 + fail_msg: >- + k3s_maintenance delegates kubectl to groups['k3s_control_plane'][0] — + it assumes a single control-plane node. Update this role before adding + a second one for HA. + quiet: true + +- name: Check whether a reboot is required + ansible.builtin.stat: + path: /var/run/reboot-required + register: k3s_reboot_required + +- name: Node is up to date — nothing to do + ansible.builtin.debug: + msg: "{{ inventory_hostname }}: no reboot required, skipping." + when: not k3s_reboot_required.stat.exists + +- name: Reboot this node if unattended-upgrades left one pending + when: k3s_reboot_required.stat.exists + block: + # --force: this is a homelab, not a cluster with a policy against bare + # pods — better to evict them than have a stray one block every rebuild. + # --delete-emptydir-data: emptyDir contents are expected to be + # disposable; anything that isn't shouldn't be using emptyDir. + - name: Cordon and drain the node + ansible.builtin.command: + argv: + - k3s + - kubectl + - drain + - "{{ inventory_hostname }}" + - --ignore-daemonsets + - --delete-emptydir-data + - --force + - --timeout=120s + delegate_to: "{{ groups['k3s_control_plane'][0] }}" + become: false + changed_when: true + + - name: Reboot the node + ansible.builtin.reboot: + reboot_timeout: 300 + + # Polls rather than trusting the reboot handshake alone — the node can be + # reachable over SSH before k3s (and, if this is the control-plane node + # itself, the API server it just took down with it) has finished coming + # back up. + - name: Wait for the node to report Ready again + ansible.builtin.command: + argv: + - k3s + - kubectl + - wait + - --for=condition=Ready + - "node/{{ inventory_hostname }}" + - --timeout=20s + delegate_to: "{{ groups['k3s_control_plane'][0] }}" + become: false + register: k3s_node_ready + changed_when: false + failed_when: false + until: k3s_node_ready.rc == 0 + retries: 12 + delay: 15 + + - name: Fail if the node never came back Ready + ansible.builtin.fail: + msg: >- + {{ inventory_hostname }} rebooted but never reported Ready again — + it's left cordoned; check it by hand before re-running this against + the rest of the cluster. + when: k3s_node_ready.rc != 0 + + - name: Uncordon the node + ansible.builtin.command: + argv: [k3s, kubectl, uncordon, "{{ inventory_hostname }}"] + delegate_to: "{{ groups['k3s_control_plane'][0] }}" + become: false + changed_when: true + + - name: Node rebooted and rejoined the cluster + ansible.builtin.debug: + msg: "{{ inventory_hostname }}: rebooted, drained and uncordoned cleanly." diff --git a/build/config/ansible/roles/k3s_metallb/defaults/main.yml b/build/config/ansible/roles/k3s_metallb/defaults/main.yml new file mode 100644 index 0000000..45f4f83 --- /dev/null +++ b/build/config/ansible/roles/k3s_metallb/defaults/main.yml @@ -0,0 +1,31 @@ +--- +# Ansible-side knobs for roles/k3s_metallb — MetalLB, the LoadBalancer +# implementation for the homelab-utils cluster. Installed via k3s's own +# bundled helm-controller, same mechanism as roles/k3s_monitoring (see +# tasks/main.yml) — no helm binary or extra Ansible collection needed on the +# controller. +# +# MetalLB replaces k3s's bundled ServiceLB (Klipper), it doesn't sit +# alongside it — both would otherwise try to satisfy the same LoadBalancer +# Services and fight over IPs. That's why `k3s_extra_args` in +# inventory/group_vars/k3s_cluster.yml carries `--disable=servicelb`. + +k3s_metallb_namespace: metallb-system + +k3s_metallb_chart_repo: https://metallb.github.io/metallb + +# Pinned, same reasoning as k3s_monitoring_chart_version — a rebuild months +# from now should reproduce today's stack, not whatever's newest at the +# time. Bump deliberately; check the current release first at +# https://github.com/metallb/metallb/releases. +k3s_metallb_chart_version: "0.14.9" + +# The pool of IPs MetalLB hands out to LoadBalancer-type Services, as a +# "first-last" range or CIDR (e.g. "192.168.50.240-192.168.50.250"). Must be +# addresses on the cluster's LAN that nothing else — DHCP, static +# assignments, the hosts in inventory/hosts.yml — will ever claim: MetalLB +# doesn't coordinate with your router, it just hands out whatever's in this +# range. No default on purpose; set it in +# inventory/group_vars/k3s_cluster.yml before the first run. tasks/main.yml +# fails fast if it's still empty. +k3s_metallb_address_range: "" diff --git a/build/config/ansible/roles/k3s_metallb/tasks/main.yml b/build/config/ansible/roles/k3s_metallb/tasks/main.yml new file mode 100644 index 0000000..f79b717 --- /dev/null +++ b/build/config/ansible/roles/k3s_metallb/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# Drops a HelmChart CR for MetalLB, plus its IPAddressPool/L2Advertisement +# config, into k3s's auto-deploying manifests directory; k3s's bundled +# helm-controller and deploy controller reconcile them — same mechanism as +# roles/k3s_monitoring (see its tasks/main.yml for why this needs no helm +# binary or kubeconfig on the controller). +# +# The config manifest references CRDs that only exist once the HelmChart +# above has actually installed the chart, so it's templated in the same +# pass rather than gated behind a "wait for CRDs" step: k3s's deploy +# controller retries a manifest referencing not-yet-existing CRDs until they +# show up, instead of failing once and giving up. + +- name: Fail fast if no MetalLB address range is configured + ansible.builtin.assert: + that: + - k3s_metallb_address_range | length > 0 + fail_msg: >- + k3s_metallb_address_range is empty — set it in + inventory/group_vars/k3s_cluster.yml before running playbooks/k3s.yml. + run_once: true + +- name: Deploy the MetalLB HelmChart manifest + ansible.builtin.template: + src: metallb.helmchart.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/metallb.yaml + owner: root + group: root + mode: "0644" + become: true + +- name: Deploy the MetalLB address pool config + ansible.builtin.template: + src: metallb-config.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/metallb-config.yaml + owner: root + group: root + mode: "0644" + become: true diff --git a/build/config/ansible/roles/k3s_metallb/templates/metallb-config.yaml.j2 b/build/config/ansible/roles/k3s_metallb/templates/metallb-config.yaml.j2 new file mode 100644 index 0000000..6cf85af --- /dev/null +++ b/build/config/ansible/roles/k3s_metallb/templates/metallb-config.yaml.j2 @@ -0,0 +1,26 @@ +# Managed by Ansible (roles/k3s_metallb) — do not edit on the node. +# +# Plain manifests, not a HelmChart — these are MetalLB's own CRs, not chart +# values. k3s's deploy controller applies anything dropped in this +# directory, not only HelmChart CRs, and retries until the CRDs the +# HelmChart alongside this file installs actually exist. +# +# L2 (ARP/NDP) mode, not BGP — the simplest fit for a flat home LAN with no +# BGP-speaking router. One pool, one advertisement, both named "default". +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: default + namespace: {{ k3s_metallb_namespace }} +spec: + addresses: + - {{ k3s_metallb_address_range }} +--- +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: default + namespace: {{ k3s_metallb_namespace }} +spec: + ipAddressPools: + - default diff --git a/build/config/ansible/roles/k3s_metallb/templates/metallb.helmchart.yaml.j2 b/build/config/ansible/roles/k3s_metallb/templates/metallb.helmchart.yaml.j2 new file mode 100644 index 0000000..630d2a0 --- /dev/null +++ b/build/config/ansible/roles/k3s_metallb/templates/metallb.helmchart.yaml.j2 @@ -0,0 +1,18 @@ +# Managed by Ansible (roles/k3s_metallb) — do not edit on the node. +# +# A HelmChart CR, same pattern as +# roles/k3s_monitoring/templates/kube-prometheus-stack.helmchart.yaml.j2 — +# k3s's bundled helm-controller reconciles it, the same mechanism it uses to +# install its own bundled Traefik and ServiceLB. No helm binary, kubeconfig, +# or extra Ansible collection needed on the controller. +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: metallb + namespace: kube-system +spec: + chart: metallb + repo: {{ k3s_metallb_chart_repo }} + version: "{{ k3s_metallb_chart_version }}" + targetNamespace: {{ k3s_metallb_namespace }} + createNamespace: true diff --git a/build/config/ansible/roles/k3s_monitoring/defaults/main.yml b/build/config/ansible/roles/k3s_monitoring/defaults/main.yml new file mode 100644 index 0000000..909dd99 --- /dev/null +++ b/build/config/ansible/roles/k3s_monitoring/defaults/main.yml @@ -0,0 +1,86 @@ +--- +# Ansible-side knobs for roles/k3s_monitoring — the kube-prometheus-stack +# Helm chart, deployed via k3s's own bundled helm-controller rather than a +# helm binary or the kubernetes.core collection (see tasks/main.yml). Lean by +# design: OpenLens's Metrics feature reads straight off the Prometheus API +# through the k8s API server proxy, so there's no need for Grafana or +# Alertmanager in-cluster — both are disabled below, mainly to save RAM on +# the Pis. + +k3s_monitoring_namespace: monitoring + +k3s_monitoring_chart_repo: https://prometheus-community.github.io/helm-charts + +# Pinned, same reasoning as k3s_version in inventory/group_vars/k3s_cluster.yml +# — a rebuild months from now should reproduce today's stack, not whatever's +# newest at the time. Bump deliberately; check the current release first at +# https://github.com/prometheus-community/helm-charts/releases (tags look +# like kube-prometheus-stack-X.Y.Z). +k3s_monitoring_chart_version: "88.3.0" + +# Handed to the chart verbatim as valuesContent (see templates/). Notes on +# the less obvious choices: +# - grafana/alertmanager: off. OpenLens supplies its own dashboards and +# this cluster doesn't page anyone, so both would just be extra Pi RAM. +# - kubeControllerManager/kubeScheduler/kubeProxy/kubeEtcd: off. k3s bundles +# the control plane inside one static binary instead of exposing these as +# separate systemd units on their usual ports, so kube-prometheus-stack's +# default scrape targets for them sit permanently "down" — a known false +# alarm on k3s, not a sign anything's actually broken. kubelet (and the +# cAdvisor/node metrics it serves) is the one k3s does expose normally, +# and it's also the one OpenLens's node/pod metrics actually need. +# - prometheus retention/storage: short retention, no PVC template (so it +# runs on emptyDir). This stack exists to answer "what are the nodes +# doing right now", not to keep months of history, so losing the TSDB on +# a pod restart is an acceptable trade for not standing up persistent +# storage across 4 SD cards. +# - every component's resources: sized for a Raspberry Pi 4, not a +# datacenter node. +k3s_monitoring_values: + grafana: + enabled: false + alertmanager: + enabled: false + kubeApiServer: + enabled: true + kubeControllerManager: + enabled: false + kubeScheduler: + enabled: false + kubeProxy: + enabled: false + kubeEtcd: + enabled: false + kubelet: + enabled: true + prometheusOperator: + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + memory: 128Mi + prometheus: + prometheusSpec: + retention: 3d + scrapeInterval: 30s + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 512Mi + kube-state-metrics: + resources: + requests: + cpu: 20m + memory: 32Mi + limits: + memory: 64Mi + prometheus-node-exporter: + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + memory: 32Mi diff --git a/build/config/ansible/roles/k3s_monitoring/tasks/main.yml b/build/config/ansible/roles/k3s_monitoring/tasks/main.yml new file mode 100644 index 0000000..fc05a3c --- /dev/null +++ b/build/config/ansible/roles/k3s_monitoring/tasks/main.yml @@ -0,0 +1,16 @@ +--- +# Drops a HelmChart CR into k3s's auto-deploying manifests directory on the +# control-plane node; k3s's bundled helm-controller reconciles it, the same +# mechanism it uses to install its own bundled Traefik and ServiceLB. See +# templates/kube-prometheus-stack.helmchart.yaml.j2 for why this needs no +# helm binary or extra Ansible collection, and README.md "Metrics +# (Prometheus)" for how to point OpenLens at the result. + +- name: Deploy the kube-prometheus-stack HelmChart manifest + ansible.builtin.template: + src: kube-prometheus-stack.helmchart.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/kube-prometheus-stack.yaml + owner: root + group: root + mode: "0644" + become: true diff --git a/build/config/ansible/roles/k3s_monitoring/templates/kube-prometheus-stack.helmchart.yaml.j2 b/build/config/ansible/roles/k3s_monitoring/templates/kube-prometheus-stack.helmchart.yaml.j2 new file mode 100644 index 0000000..25ed6f7 --- /dev/null +++ b/build/config/ansible/roles/k3s_monitoring/templates/kube-prometheus-stack.helmchart.yaml.j2 @@ -0,0 +1,24 @@ +--- +# Managed by Ansible (roles/k3s_monitoring) — do not edit on the node. +# +# A HelmChart CR, not a `helm install`: k3s ships its own helm-controller +# that watches /var/lib/rancher/k3s/server/manifests/ (this file's +# destination) the same way it watches for any other auto-deploying +# manifest, and reconciles HelmChart resources found there — the same +# mechanism k3s uses to install its own bundled Traefik and ServiceLB. That +# means no helm binary, kubeconfig, or extra Ansible collection is needed on +# the controller; re-running the playbook just re-templates this file and +# the controller reconciles the diff. +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: kube-prometheus-stack + namespace: kube-system +spec: + chart: kube-prometheus-stack + repo: {{ k3s_monitoring_chart_repo }} + version: "{{ k3s_monitoring_chart_version }}" + targetNamespace: {{ k3s_monitoring_namespace }} + createNamespace: true + valuesContent: | +{{ k3s_monitoring_values | to_nice_yaml(indent=2) | indent(4, first=true) }} diff --git a/build/config/ansible/roles/k3s_node/defaults/main.yml b/build/config/ansible/roles/k3s_node/defaults/main.yml new file mode 100644 index 0000000..ce38dd6 --- /dev/null +++ b/build/config/ansible/roles/k3s_node/defaults/main.yml @@ -0,0 +1,18 @@ +--- +# Ubuntu Server's boot partition. Same path on Raspberry Pi OS, so this +# doesn't need to change if the Pis are ever reimaged to that instead. +k3s_boot_cmdline_path: /boot/firmware/cmdline.txt + +# k3s (like any Kubernetes distro) needs the memory cgroup controller, which +# isn't always on by default on a Pi kernel. Ubuntu's Pi images usually ship +# these already — this task is a safety net for that assumption, not the +# primary mechanism, and a no-op on most runs. See tasks/prep.yml. +k3s_cgroup_params: + - cgroup_memory=1 + - cgroup_enable=memory + +# Safety-net defaults for inventory/group_vars/k3s_cluster.yml's extra-args +# vars (see tasks/server.yml, tasks/agent.yml) — empty so the role doesn't +# break if a var is left undefined there. +k3s_extra_args: [] +k3s_server_extra_args: [] diff --git a/build/config/ansible/roles/k3s_node/tasks/agent.yml b/build/config/ansible/roles/k3s_node/tasks/agent.yml new file mode 100644 index 0000000..b2c7014 --- /dev/null +++ b/build/config/ansible/roles/k3s_node/tasks/agent.yml @@ -0,0 +1,100 @@ +--- +# Joins this node as a k3s agent (worker). Only ever runs after server.yml has +# succeeded somewhere — playbooks/k3s.yml targets k3s_control_plane before +# k3s_workers — so K3S_URL below always points at an already-live API server. +# +# --node-name pins the k8s node object to the Ansible inventory_hostname — +# see the matching comment in server.yml, same reason. + +- name: Bootstrap the k3s agent + no_log: true # K3S_TOKEN passes through this block + block: + - name: Look up the k3s cluster secrets + ansible.builtin.set_fact: + k3s_secrets: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + k3s_vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + + - name: Compute the desired k3s agent exec line + ansible.builtin.set_fact: + k3s_agent_exec: "agent --node-name {{ inventory_hostname }} {{ k3s_extra_args | join(' ') }}" + + # Compared against what k3s was last installed with (see the copy task + # below) so a change to k3s_extra_args gets applied on the next run + # instead of silently sitting unused: the version check below has no way + # to notice an exec-line-only change. Missing file (first install) counts + # as changed. + - name: Read the exec line k3s was last installed with + ansible.builtin.slurp: + src: /etc/rancher/k3s/.ansible_install_exec + register: k3s_installed_exec_raw + failed_when: false + check_mode: false + + - name: Determine whether the exec line has changed + ansible.builtin.set_fact: + k3s_exec_changed: >- + {{ k3s_installed_exec_raw.content is not defined + or (k3s_installed_exec_raw.content | b64decode) != k3s_agent_exec }} + + - name: Check the installed k3s version + ansible.builtin.command: k3s --version + register: k3s_installed_version + changed_when: false + failed_when: false + check_mode: false + + # The installer is safe to re-run — it's a no-op if the requested version + # and exec line are already active — but skipping it when neither changed + # avoids restarting the agent (and briefly dropping the node's kubelet) + # on every playbook run. Restarting the k3s-agent process itself (as + # opposed to rebooting the node, which roles/k3s_maintenance handles + # separately) doesn't touch already-running pods — containerd keeps them + # up underneath it. playbooks/k3s.yml still runs this play with + # serial: 1, so at most one node's kubelet is ever bouncing at a time. + - name: Install/upgrade k3s agent + ansible.builtin.shell: curl -sfL https://get.k3s.io | sh - + environment: + INSTALL_K3S_VERSION: "{{ k3s_version }}" + INSTALL_K3S_EXEC: "{{ k3s_agent_exec }}" + # The one control-plane node, addressed by its inventory IP rather + # than delegating a lookup to it — simple and correct as long as this + # cluster stays single-server. Revisit if it ever gets HA control + # plane nodes. + K3S_URL: "https://{{ hostvars[groups['k3s_control_plane'][0]].ansible_host }}:6443" + K3S_TOKEN: "{{ k3s_secrets.K3S_TOKEN }}" + when: >- + k3s_installed_version.rc != 0 + or k3s_version not in k3s_installed_version.stdout + or k3s_exec_changed + +- name: Ensure the k3s-agent service is enabled and running + ansible.builtin.systemd_service: + name: k3s-agent + enabled: true + state: started + +# k3s creates /etc/rancher/k3s itself once it has something to put there +# (a config.yaml, the generated kubeconfig on the server) — an agent with no +# extra config doesn't necessarily end up with anything else prompting that, +# so this marker can't assume the directory already exists. +- name: Ensure /etc/rancher/k3s exists + ansible.builtin.file: + path: /etc/rancher/k3s + state: directory + owner: root + group: root + mode: "0755" + +- name: Record the exec line k3s was installed with + ansible.builtin.copy: + dest: /etc/rancher/k3s/.ansible_install_exec + content: "{{ k3s_agent_exec }}" + owner: root + group: root + mode: "0600" diff --git a/build/config/ansible/roles/k3s_node/tasks/main.yml b/build/config/ansible/roles/k3s_node/tasks/main.yml new file mode 100644 index 0000000..ecbd30d --- /dev/null +++ b/build/config/ansible/roles/k3s_node/tasks/main.yml @@ -0,0 +1,24 @@ +--- +# Converges one node of the cluster: the same prep either way, then branch on +# which k3s role it plays. k3s_node_role is set per group, not per host — see +# inventory/group_vars/k3s_control_plane.yml (server) and k3s_workers.yml +# (agent) — so adding a fifth Pi later is a hosts.yml edit, not a role change. + +- name: Validate k3s_node_role + ansible.builtin.assert: + that: k3s_node_role in ['server', 'agent'] + fail_msg: >- + k3s_node_role is '{{ k3s_node_role }}'; expected 'server' or 'agent'. + Check inventory/group_vars/k3s_control_plane.yml and k3s_workers.yml. + quiet: true + +- name: Prepare the node for k3s + ansible.builtin.include_tasks: prep.yml + +- name: Install the k3s server (control plane) + ansible.builtin.include_tasks: server.yml + when: k3s_node_role == 'server' + +- name: Install the k3s agent (worker) + ansible.builtin.include_tasks: agent.yml + when: k3s_node_role == 'agent' diff --git a/build/config/ansible/roles/k3s_node/tasks/prep.yml b/build/config/ansible/roles/k3s_node/tasks/prep.yml new file mode 100644 index 0000000..47913b2 --- /dev/null +++ b/build/config/ansible/roles/k3s_node/tasks/prep.yml @@ -0,0 +1,80 @@ +--- +# OS-level prerequisites, identical for a server and an agent node. Runs +# before either install so a from-scratch Pi (fresh Ubuntu Server image, SSH +# + the `ansible` user already set up) can go straight to a working cluster +# in one playbook run — the "easy to rebuild" part of the design. + +- name: Read the current kernel boot parameters + ansible.builtin.command: cat {{ k3s_boot_cmdline_path }} + register: k3s_cmdline_current + changed_when: false + check_mode: false + +- name: Work out which cgroup parameters are missing + ansible.builtin.set_fact: + k3s_cmdline_missing: >- + {{ k3s_cgroup_params | reject('in', k3s_cmdline_current.stdout) | list }} + +# cmdline.txt is one line, space-separated — rewritten whole rather than +# appended in place, since there's no line-based anchor to insert after. +# Existing file mode is left alone (no `mode:` here) rather than guessed at. +- name: Add missing cgroup parameters to the boot command line + ansible.builtin.copy: + dest: "{{ k3s_boot_cmdline_path }}" + content: >- + {{ (k3s_cmdline_current.stdout.split() + k3s_cmdline_missing) | join(' ') }} + owner: root + group: root + register: k3s_cmdline_updated + when: k3s_cmdline_missing | length > 0 + +- name: Reboot to apply updated boot parameters + ansible.builtin.reboot: + reboot_timeout: 300 + when: k3s_cmdline_updated is changed + +- name: Check active swap devices + ansible.builtin.command: swapon --summary + register: k3s_swap_active + changed_when: false + check_mode: false + +- name: Turn off active swap + ansible.builtin.command: swapoff -a + when: k3s_swap_active.stdout | trim | length > 0 + +- name: Comment out swap entries in fstab + ansible.builtin.replace: + path: /etc/fstab + regexp: '^([^#\n]*\sswap\s.*)$' + replace: '# \1' + +# Best-effort: Ubuntu's zram-backed swap ships under different unit names +# across releases, and most won't be present at all. A missing unit is not a +# failure here — only an already-active one that we failed to disable would +# leave swap coming back on the next boot, and swapoff -a above already +# handles the running instance for this boot. +# +# `failed_when: false` rather than `ignore_errors: true`: systemd_service +# raises "Could not find the requested service" as a hard module failure +# when the unit is absent, and that's reported here as a genuine task +# failure regardless of ignore_errors — failed_when overrides the result +# directly instead of trying to catch it after the fact. +- name: Disable Ubuntu's zram-backed swap, if present + ansible.builtin.systemd_service: + name: "{{ item }}" + enabled: false + state: stopped + loop: + - zram-config.service + - systemd-zram-setup@zram0.service + register: k3s_zram_disable + failed_when: false + changed_when: k3s_zram_disable is changed + +- name: Ensure curl is installed + ansible.builtin.apt: + name: curl + state: present + update_cache: true + cache_valid_time: 3600 diff --git a/build/config/ansible/roles/k3s_node/tasks/server.yml b/build/config/ansible/roles/k3s_node/tasks/server.yml new file mode 100644 index 0000000..3c838e0 --- /dev/null +++ b/build/config/ansible/roles/k3s_node/tasks/server.yml @@ -0,0 +1,103 @@ +--- +# Bootstraps this node as the (single) k3s control plane. playbooks/k3s.yml +# runs the k3s_control_plane play before k3s_workers, so by the time +# agent.yml runs anywhere, K3S_URL below already points at something live. +# +# The token comes from Vault as a fixed, pre-shared value rather than letting +# k3s generate one on first install — see docs/vault-secrets.md +# (homelab/k3s-homelab-utils). That's what makes a full rebuild (wipe both SD +# cards, reinstall) reproduce the same cluster identity instead of needing the +# new token hunted down and re-distributed by hand. +# +# --node-name pins the k8s node object to the Ansible inventory_hostname +# rather than whatever the OS hostname happens to be — roles/k3s_maintenance +# addresses nodes by inventory_hostname when draining/uncordoning, and that +# only works if the two names are guaranteed to match. + +- name: Bootstrap the k3s server + no_log: true # K3S_TOKEN passes through this block + block: + - name: Look up the k3s cluster secrets + ansible.builtin.set_fact: + k3s_secrets: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + k3s_vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + + - name: Compute the desired k3s server exec line + ansible.builtin.set_fact: + k3s_server_exec: >- + server --node-name {{ inventory_hostname }} --tls-san {{ k3s_api_tls_san }} + --write-kubeconfig-mode 644 {{ (k3s_extra_args + k3s_server_extra_args) | join(' ') }} + + # Compared against what k3s was last installed with (see the copy task + # below) so a change to k3s_extra_args/k3s_server_extra_args — or + # k3s_api_tls_san — gets applied on the next run instead of silently + # sitting unused: the version check below has no way to notice an + # exec-line-only change. Missing file (first install) counts as changed. + - name: Read the exec line k3s was last installed with + ansible.builtin.slurp: + src: /etc/rancher/k3s/.ansible_install_exec + register: k3s_installed_exec_raw + failed_when: false + check_mode: false + + - name: Determine whether the exec line has changed + ansible.builtin.set_fact: + k3s_exec_changed: >- + {{ k3s_installed_exec_raw.content is not defined + or (k3s_installed_exec_raw.content | b64decode) != k3s_server_exec }} + + - name: Check the installed k3s version + ansible.builtin.command: k3s --version + register: k3s_installed_version + changed_when: false + failed_when: false + check_mode: false + + # The installer is safe to re-run — it's a no-op if the requested version + # and exec line are already active — but skipping it when neither changed + # avoids restarting the service (and briefly dropping the API) on every + # playbook run. Restarting the k3s process itself (as opposed to + # rebooting the node, which roles/k3s_maintenance handles separately) is + # a brief control-plane/kubelet blip, not a pod outage — containerd keeps + # every already-running pod up underneath it. + - name: Install/upgrade k3s server + ansible.builtin.shell: curl -sfL https://get.k3s.io | sh - + environment: + INSTALL_K3S_VERSION: "{{ k3s_version }}" + INSTALL_K3S_EXEC: "{{ k3s_server_exec }}" + K3S_TOKEN: "{{ k3s_secrets.K3S_TOKEN }}" + when: >- + k3s_installed_version.rc != 0 + or k3s_version not in k3s_installed_version.stdout + or k3s_exec_changed + +- name: Ensure the k3s service is enabled and running + ansible.builtin.systemd_service: + name: k3s + enabled: true + state: started + +# k3s creates /etc/rancher/k3s itself once it has something to put there +# (the generated kubeconfig, on the server) — defensive rather than relied +# on, so this marker doesn't assume the directory already exists. +- name: Ensure /etc/rancher/k3s exists + ansible.builtin.file: + path: /etc/rancher/k3s + state: directory + owner: root + group: root + mode: "0755" + +- name: Record the exec line k3s was installed with + ansible.builtin.copy: + dest: /etc/rancher/k3s/.ansible_install_exec + content: "{{ k3s_server_exec }}" + owner: root + group: root + mode: "0600" diff --git a/build/config/ansible/roles/k3s_postgres/defaults/main.yml b/build/config/ansible/roles/k3s_postgres/defaults/main.yml new file mode 100644 index 0000000..2094fd7 --- /dev/null +++ b/build/config/ansible/roles/k3s_postgres/defaults/main.yml @@ -0,0 +1,71 @@ +--- +# Ansible-side knobs for roles/k3s_postgres — CloudNativePG (CNPG), the +# Postgres operator for the homelab-utils cluster's shared database. +# Installed via k3s's own bundled helm-controller, same mechanism as +# roles/k3s_metallb and roles/k3s_monitoring (see tasks/main.yml) — no helm +# binary or extra Ansible collection needed on the controller. +# +# Chosen over Bitnami's postgresql-ha (repmgr+pgpool, more moving parts per +# pod, and Bitnami's free chart/image catalog was restructured into a +# "legacy" repo in 2025 — a risky long-term dependency) and the Zalando +# operator (mature but Patroni-based, heavier, and its own config +# conventions fit less naturally into this repo's lean HelmChart-CR-only +# pattern). CNPG installs as one operator chart plus one plain Cluster CR — +# the same "HelmChart + plain config manifest" shape already used for +# MetalLB (chart + IPAddressPool). +# +# The app-portable half of this config (instance count, storage size/class) +# lives with the app it belongs to instead: see +# src/shared/postgres/ansible/kubernetes/vars.yml, loaded into `pg_config` +# by tasks/main.yml — same common/vars.yml + platform-vars.yml split +# compose_stack and lxc_app use for every other app. + +k3s_postgres_operator_namespace: cnpg-system + +k3s_postgres_chart_repo: https://cloudnative-pg.github.io/charts + +# Pinned, same reasoning as k3s_monitoring_chart_version/k3s_metallb_chart_version +# — a rebuild months from now should reproduce today's operator, not +# whatever's newest at the time. Bump deliberately; check the current +# release first at https://github.com/cloudnative-pg/charts/releases. +k3s_postgres_chart_version: "0.29.0" + +# Where the actual Postgres Cluster (not the operator) lives. +k3s_postgres_namespace: shared-postgres + +# Postgres major/minor pinned via the operand image, independent of the +# chart/operator version above. This is a separate physical instance from +# the Docker-based shared/postgres (pinned to 13 there — see +# src/shared/postgres/common/vars.yml), so there's no need to match; pin to +# a current stable major instead. Multi-arch (amd64/arm64) upstream, same +# as every other image this cluster runs. +k3s_postgres_image: ghcr.io/cloudnative-pg/postgresql:18.4 + +# Same Vault path the Unraid/Proxmox shared-postgres instances already use +# (see docs/vault-secrets.md) — a separate physical instance, but one +# superuser identity for the "shared postgres" concept everywhere it runs. +k3s_postgres_vault_path: homelab/shared/postgres + +# The LAN address the primary is published on, via the LoadBalancer Service +# declared in templates/postgres-cluster.yaml.j2. Must be inside +# k3s_metallb_address_range — MetalLB only assigns from its own pools, and a +# request for anything outside them leaves the Service pending forever rather +# than failing loudly. +# +# Empty here on purpose, same reasoning as k3s_metallb_address_range's own +# default: the address is a property of the LAN, not of this role, so it's set +# in inventory/group_vars/k3s_cluster.yml next to the pool it has to fall +# inside. Unlike the pool, leaving it empty is survivable — the Service is +# still created and MetalLB assigns the next free address; you just don't know +# which one until you look, which is fine for in-cluster clients and no good +# for anything that has to write the address down. +k3s_postgres_loadbalancer_ip: "" + +# Resource requests/limits per instance — sized for a Raspberry Pi 4, not a +# datacenter node, same reasoning as k3s_monitoring_values. +k3s_postgres_resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 512Mi diff --git a/build/config/ansible/roles/k3s_postgres/tasks/main.yml b/build/config/ansible/roles/k3s_postgres/tasks/main.yml new file mode 100644 index 0000000..867300a --- /dev/null +++ b/build/config/ansible/roles/k3s_postgres/tasks/main.yml @@ -0,0 +1,91 @@ +--- +# Deploys the shared Postgres cluster onto k3s: install CloudNativePG (the +# operator) via a HelmChart CR, then define the actual cluster as a plain +# Cluster CR — same two-manifest pattern as roles/k3s_metallb (chart CR + +# plain config manifest), for the same reason: k3s's deploy controller +# retries a manifest referencing CRDs the HelmChart above hasn't installed +# yet instead of failing outright, so the Cluster CR doesn't need to wait on +# the operator chart finishing first. +# +# Config is layered the same way compose_stack/lxc_app layer an app's +# common/vars.yml + platform vars.yml — src/shared/postgres/ is the same +# shared service already used on Unraid/Proxmox, just deployed a third way +# here. Secrets come from the same Vault path those platforms already use +# (homelab/shared/postgres) — this is a separate physical instance, but +# reuses the path rather than inventing a k3s-specific one, same "one path +# per app-concept" convention as every other stack. + +- name: Set shared-postgres source facts + ansible.builtin.set_fact: + pg_local_dir: "{{ repo_root }}/src/shared/postgres" + +- name: Load portable shared-postgres variables + ansible.builtin.include_vars: + file: "{{ pg_local_dir }}/common/vars.yml" + name: pg_common_vars + +- name: Load Kubernetes-specific shared-postgres variables + ansible.builtin.include_vars: + file: "{{ pg_local_dir }}/ansible/kubernetes/vars.yml" + name: pg_platform_vars + +- name: Merge shared-postgres configuration + ansible.builtin.set_fact: + pg_config: >- + {{ (pg_common_vars.env_defaults | default({})) + | combine(pg_platform_vars.env_defaults | default({})) }} + +- name: Look up shared-postgres superuser credentials from Vault + ansible.builtin.set_fact: + pg_vault_secrets: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + k3s_postgres_vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + no_log: true + +- name: Deploy the CloudNativePG operator HelmChart manifest + ansible.builtin.template: + src: cnpg-operator.helmchart.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/cnpg-operator.yaml + owner: root + group: root + mode: "0644" + become: true + +# Namespace + Cluster CR, no secrets — safe to render even though the +# Cluster CR references the Secret below, for the same "later file, retried +# reconcile" reasoning as the Secret referencing this file's Namespace. +- name: Deploy the shared-postgres Cluster manifest + ansible.builtin.template: + src: postgres-cluster.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/shared-postgres-cluster.yaml + owner: root + group: root + mode: "0644" + become: true + +# Separate file (and mode) from the Cluster manifest above purely because +# this one carries the Vault-sourced password — everything else here is safe +# at 0644, this one isn't. +# +# The dest filename deliberately doesn't track the Secret's own name, which +# gained a `-vault` suffix (see the template's header for why). k3s's deploy +# controller records which resources each manifest *file* created and prunes +# the ones that disappear from it, so renaming the Secret inside this same +# file makes the old `shared-postgres-superuser` object get garbage-collected +# on the next run. Renaming the file too would orphan it instead: Ansible +# doesn't remove files it no longer writes, so the old manifest would sit +# there keeping the stale Secret alive. +- name: Deploy the shared-postgres superuser Secret + ansible.builtin.template: + src: postgres-superuser-secret.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/shared-postgres-superuser-secret.yaml + owner: root + group: root + mode: "0600" + become: true + no_log: true diff --git a/build/config/ansible/roles/k3s_postgres/templates/cnpg-operator.helmchart.yaml.j2 b/build/config/ansible/roles/k3s_postgres/templates/cnpg-operator.helmchart.yaml.j2 new file mode 100644 index 0000000..053a2ae --- /dev/null +++ b/build/config/ansible/roles/k3s_postgres/templates/cnpg-operator.helmchart.yaml.j2 @@ -0,0 +1,21 @@ +# Managed by Ansible (roles/k3s_postgres) — do not edit on the node. +# +# A HelmChart CR, same pattern as +# roles/k3s_metallb/templates/metallb.helmchart.yaml.j2 and +# roles/k3s_monitoring — k3s's bundled helm-controller reconciles it, the +# same mechanism it uses to install its own bundled Traefik and ServiceLB. +# No helm binary, kubeconfig, or extra Ansible collection needed on the +# controller. This installs the CloudNativePG operator and its CRDs only — +# the actual Postgres cluster is templates/postgres-cluster.yaml.j2, a plain +# manifest applied once the CRD it references exists (see that file). +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: cloudnative-pg + namespace: kube-system +spec: + chart: cloudnative-pg + repo: {{ k3s_postgres_chart_repo }} + version: "{{ k3s_postgres_chart_version }}" + targetNamespace: {{ k3s_postgres_operator_namespace }} + createNamespace: true diff --git a/build/config/ansible/roles/k3s_postgres/templates/postgres-cluster.yaml.j2 b/build/config/ansible/roles/k3s_postgres/templates/postgres-cluster.yaml.j2 new file mode 100644 index 0000000..2f82be7 --- /dev/null +++ b/build/config/ansible/roles/k3s_postgres/templates/postgres-cluster.yaml.j2 @@ -0,0 +1,104 @@ +# Managed by Ansible (roles/k3s_postgres) — do not edit on the node. +# +# Plain manifests, not a HelmChart — same pattern as +# roles/k3s_metallb/templates/metallb-config.yaml.j2: the Cluster CR below +# references a CRD (postgresql.cnpg.io) that only exists once the +# cloudnative-pg HelmChart (cnpg-operator.yaml, same manifests directory) has +# actually installed the operator, so it's applied here rather than gated +# behind a "wait for CRDs" step — k3s's deploy controller retries a manifest +# referencing not-yet-existing CRDs until they show up, instead of failing +# once and giving up. templates/postgres-superuser-secret.yaml.j2 relies on +# the Namespace created here the same way, for the same reason. +apiVersion: v1 +kind: Namespace +metadata: + name: {{ k3s_postgres_namespace }} +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: shared-postgres + namespace: {{ k3s_postgres_namespace }} +spec: + # 1 primary + 1 replica (src/shared/postgres/ansible/kubernetes/vars.yml). + # CNPG's own default pod anti-affinity (preferred, topology key hostname) + # spreads them across distinct nodes; the nodeAffinity below narrows which + # nodes those can be. + instances: {{ pg_config.POSTGRES_INSTANCES }} + imageName: {{ k3s_postgres_image }} + + # Password-based superuser login, from the Secret Ansible renders + # alongside this file. Off by default in CNPG — without this, the operator + # ignores the secret's content and sets the postgres user's password to + # NULL, disabling remote login as it. + # + # The name must stay clear of `-superuser` (i.e. + # `shared-postgres-superuser`), which is what CNPG calls the secret it + # generates for itself when this stanza is absent. Claiming that name for + # our own object stops password changes reaching the database — see + # postgres-superuser-secret.yaml.j2 for the full symptom. + enableSuperuserAccess: true + superuserSecret: + name: shared-postgres-superuser-vault + + # A LoadBalancer Service for the primary, on top of the ClusterIP -rw/-ro/-r + # Services CNPG creates for every Cluster. Declared here as a CNPG *managed + # service* rather than as a Service manifest of our own: selectorType: rw + # means the operator maintains the selector, so the LB follows a failover to + # the other instance the same way the built-in -rw Service does. A + # hand-written Service would need its selector re-pointed by hand after + # every promotion. + # + # This is what makes the cluster reachable from the Ansible controller at a + # fixed host:port, which is the thing per-app database provisioning needs — + # the community.postgresql tasks compose_stack/lxc_app use can't talk to a + # ClusterIP, and `kubectl port-forward` isn't a stable address. It does put + # the superuser on the LAN: acceptable here for the same reason the Unraid + # instance's published port is, and the LAN is the boundary either way. + managed: + services: + additional: + - selectorType: rw + serviceTemplate: + metadata: + name: shared-postgres-lb +{% if k3s_postgres_loadbalancer_ip %} + # Pinned rather than left to MetalLB's next-free pick, because + # this address ends up in config elsewhere (inventory, an app's + # vars.yml) and shouldn't move when the Service is recreated. + annotations: + metallb.universe.tf/loadBalancerIPs: "{{ k3s_postgres_loadbalancer_ip }}" +{% endif %} + spec: + type: LoadBalancer + # No ports: — the operator fills in Postgres's own (5432) from + # the same template it builds the -rw Service from; naming them + # here would only risk disagreeing with it. + + # POSTGRES_PORT (common/vars.yml) is 5432, the same port CNPG always + # listens on inside the pod/Service — nothing to override here. + storage: + size: {{ pg_config.POSTGRES_STORAGE_SIZE }} + storageClass: {{ pg_config.POSTGRES_STORAGE_CLASS }} + + resources: +{{ k3s_postgres_resources | to_nice_yaml(indent=2) | indent(4, first=true) }} + + # Workers only — k3s-ctrl-01 stays free of app pods, same boundary the + # cluster already keeps for monitoring/MetalLB. + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: DoesNotExist + + # No bootstrap: stanza — CNPG's default initdb bootstrap creates a + # `postgres` superuser (above) plus a default `app` database owned by an + # auto-generated `app` role/secret. That default app database is unused + # today: no app is deployed onto k3s yet, and real per-app database + # provisioning here is a deferred design problem (see CLAUDE.md → "Key + # decisions") — the community.postgresql approach compose_stack/lxc_app + # use needs a controller-reachable host:port, which this cluster's + # in-cluster -rw Service isn't without further work. diff --git a/build/config/ansible/roles/k3s_postgres/templates/postgres-superuser-secret.yaml.j2 b/build/config/ansible/roles/k3s_postgres/templates/postgres-superuser-secret.yaml.j2 new file mode 100644 index 0000000..38c3aed --- /dev/null +++ b/build/config/ansible/roles/k3s_postgres/templates/postgres-superuser-secret.yaml.j2 @@ -0,0 +1,49 @@ +# Managed by Ansible (roles/k3s_postgres) — do not edit on the node. +# +# Superuser credentials for the shared-postgres Cluster +# (postgres-cluster.yaml.j2), sourced from the same Vault path +# (homelab/shared/postgres) the Unraid/Proxmox shared-postgres instances +# already use — see docs/vault-secrets.md. CNPG requires +# `type: kubernetes.io/basic-auth` with username/password keys, and +# `enableSuperuserAccess: true` on the Cluster (set there) for this secret to +# actually be usable for password login rather than just ignored. +# +# The `-vault` suffix is load-bearing, and removing it reintroduces a bug +# that is very hard to read off the symptoms. CNPG generates its own +# superuser secret named `-superuser` when the Cluster doesn't name +# one — for a Cluster called `shared-postgres` that is exactly +# `shared-postgres-superuser`, the name this file used to claim. Writing our +# own object at the operator's reserved name meant the operator treated it as +# one it had already authored and reconciled, so a password change in Vault +# propagated as far as this Secret and stopped there: the role in the +# database kept whatever was set at bootstrap. Every check short of querying +# pg_authid agrees the change landed, while `psql` from the LAN keeps +# returning "password authentication failed for user postgres". Any name +# outside the `-*` pattern the operator owns avoids it. +# +# Namespace comes from postgres-cluster.yaml.j2, a separate file in the same +# manifests directory — see that file's header for why applying this before +# the Namespace exists isn't a problem. +# +# The username is layered the same way every other value in this repo is — +# Vault on top, common/vars.yml underneath — rather than required from Vault. +# It isn't a secret (only the password is), and the Vault path is allowed to +# carry just POSTGRES_SUPERUSER_PASSWORD; setting POSTGRES_SUPERUSER there +# still wins, which is what docs/vault-secrets.md means by "Vault is +# authoritative since it's paired with the password". +apiVersion: v1 +kind: Secret +metadata: + name: shared-postgres-superuser-vault + namespace: {{ k3s_postgres_namespace }} +type: kubernetes.io/basic-auth +# to_json, not bare interpolation: it emits a double-quoted YAML scalar with +# escaping handled, so a password containing `#`, `:`, a leading `*`/`&`, or +# leading/trailing whitespace survives intact. Unquoted, YAML silently +# reinterprets those (`#` starts a comment, an all-digit password becomes an +# int) and the Secret ends up holding something other than what Vault has — +# which fails authentication while every value still looks right in playbook +# output. Alphanumeric-only passwords hide the problem rather than fix it. +stringData: + username: {{ (pg_vault_secrets.POSTGRES_SUPERUSER | default(pg_config.POSTGRES_SUPERUSER)) | to_json }} + password: {{ pg_vault_secrets.POSTGRES_SUPERUSER_PASSWORD | to_json }} diff --git a/build/config/ansible/roles/k3s_traefik/defaults/main.yml b/build/config/ansible/roles/k3s_traefik/defaults/main.yml new file mode 100644 index 0000000..af3c0ae --- /dev/null +++ b/build/config/ansible/roles/k3s_traefik/defaults/main.yml @@ -0,0 +1,115 @@ +--- +# Ansible-side knobs for roles/k3s_traefik — the ingress controller k3s +# bundles. The odd one out among the k3s_* cluster services: every other one +# installs a chart this repo chose (roles/k3s_metallb, k3s_monitoring, +# k3s_postgres, k3s_cert_manager), whereas Traefik is already installed by +# k3s itself before Ansible ever connects. So this role installs nothing — +# it only adjusts what's there, via a HelmChartConfig (see +# templates/traefik.helmchartconfig.yaml.j2 for why that, and not an edit to +# k3s's own traefik.yaml). +# +# Scope today is the dashboard, which a stock k3s serves 404s for. That's +# not a broken install: Traefik builds the dashboard regardless +# (`api.dashboard` defaults on), but the Traefik chart stopped creating the +# router that reaches it in v28, so there is nothing routing /dashboard/ on +# any entrypoint until something puts that router back. + +# Where k3s installs Traefik, and so where the HelmChartConfig has to live — +# a HelmChartConfig only applies to the HelmChart of the same name in the +# same namespace. +k3s_traefik_namespace: kube-system + +# --- The dashboard, reachable by port-forward -------------------------------- +# +# Re-enables the chart's own dashboard IngressRoute, which binds to Traefik's +# internal `traefik` entrypoint (port 9000). That entrypoint is not published +# on Traefik's Service, so this exposes nothing to the LAN; it makes exactly +# one thing work: +# +# kubectl -n kube-system port-forward deploy/traefik 9000:9000 +# # then http://127.0.0.1:9000/dashboard/ — the trailing slash is required +# +# Left on even when the dashboard is also exposed on a hostname below, and +# that's deliberate rather than redundant. Once the exposed route is behind +# Authentik, the dashboard depends on Authentik being up, which depends on +# CNPG being up, which depends on the cluster being healthy — precisely the +# things you would open the dashboard to diagnose. Port-forward talks to the +# pod and traverses none of it, so it stays as the break-glass path. Don't +# "tidy this up" once the hostname works. +k3s_traefik_dashboard_enabled: true + +# --- The dashboard, exposed on a hostname ------------------------------------ +# +# Empty (the default) means the port-forward above is the only way in, and +# tasks/main.yml renders no IngressRoute at all — it removes one it previously +# rendered, so clearing this is a real teardown rather than a no-op. +# +# Set it to a hostname (e.g. traefik.turtlesystems.uk) to publish the +# dashboard on the `websecure` entrypoint with a cert-manager certificate and +# the forward-auth middleware below in front of it. Doing so requires +# k3s_traefik_dashboard_auth_address to be set as well — the role refuses to +# run otherwise, because the failure mode of getting this wrong is a +# read-only view of every route, service and middleware in the cluster +# published unauthenticated to the LAN. +# +# The hostname needs a DNS record pointing at Traefik's MetalLB address +# (`kubectl -n kube-system get svc traefik`), same as any app's Ingress. +k3s_traefik_dashboard_host: "" + +# The ClusterIssuer roles/k3s_cert_manager creates. Must match +# k3s_cert_manager_issuer_name — stated rather than referenced because that +# lives in the other role's defaults, which aren't in scope here. Same +# convention as CERT_ISSUER in src/authentik/ansible/kubernetes/vars.yml. +# +# An explicit Certificate rather than an Ingress annotation, unlike every app +# on this cluster: cert-manager watches Ingress resources, and the dashboard +# can't be one. It's served by Traefik's internal `api@internal` service, +# which has no Kubernetes Service for an Ingress to point at — hence an +# IngressRoute, which cert-manager doesn't watch. +k3s_traefik_dashboard_cert_issuer: letsencrypt +k3s_traefik_dashboard_tls_secret: traefik-dashboard-tls + +# --- Authentication in front of the exposed dashboard ------------------------ +# +# A Traefik forwardAuth middleware pointing at Authentik, which now runs on +# this cluster (roles/k3s_app, src/authentik). No default, and required as +# soon as k3s_traefik_dashboard_host is set: a default here would be a guess +# at another app's service name, and a wrong guess that happens to be +# unreachable is a 500, not an open dashboard — but one that happens to +# resolve to the wrong thing is worse. Set it explicitly, next to the host, +# in inventory/group_vars/k3s_cluster.yml. +# +# For Authentik's embedded outpost, in-cluster: +# +# http://authentik-server.authentik.svc.cluster.local/outpost.goauthentik.io/auth/traefik +# +# The provider in Authentik must be a Proxy Provider in **forward auth +# (domain level)** mode, not single-application. Single-application mode +# additionally requires /outpost.goauthentik.io/ to be routed to Authentik on +# *this* host, which from an IngressRoute in kube-system means referencing a +# Service in the authentik namespace — a cross-namespace reference Traefik +# rejects unless started with providers.kubernetescrd.allowCrossNamespace, +# which it isn't. Domain-level mode handles the redirect on Authentik's own +# hostname instead and needs no such route, so it's the mode that fits. +k3s_traefik_dashboard_auth_address: "" + +k3s_traefik_dashboard_auth_middleware: dashboard-auth + +# What Authentik sets on the way back through, forwarded to the dashboard. +# The dashboard itself reads none of them — it has no notion of a user — so +# this is really about the headers existing for anything else that reuses +# this middleware later, and about matching Authentik's documented list +# rather than inventing a shorter one. +k3s_traefik_dashboard_auth_response_headers: + - X-authentik-username + - X-authentik-groups + - X-authentik-entitlements + - X-authentik-email + - X-authentik-name + - X-authentik-uid + - X-authentik-jwt + - X-authentik-meta-jwks + - X-authentik-meta-outpost + - X-authentik-meta-provider + - X-authentik-meta-app + - X-authentik-meta-version diff --git a/build/config/ansible/roles/k3s_traefik/tasks/main.yml b/build/config/ansible/roles/k3s_traefik/tasks/main.yml new file mode 100644 index 0000000..8fd2686 --- /dev/null +++ b/build/config/ansible/roles/k3s_traefik/tasks/main.yml @@ -0,0 +1,59 @@ +--- +# Adjusts the Traefik k3s installs for itself, rather than installing +# anything: a HelmChartConfig merged over k3s's own Traefik HelmChart, plus — +# only when the dashboard is being published on a hostname — the +# Certificate/Middleware/IngressRoute trio that puts it behind Authentik. +# See defaults/main.yml and templates/traefik.helmchartconfig.yaml.j2. +# +# Same delivery mechanism as every other k3s_* role: template a file into +# /var/lib/rancher/k3s/server/manifests/ and let k3s's bundled helm-controller +# and deploy controller reconcile it. No helm binary, kubeconfig or extra +# collection on the controller. + +- name: Fail fast if the dashboard would be exposed without authentication + when: k3s_traefik_dashboard_host | length > 0 + ansible.builtin.assert: + that: + - k3s_traefik_dashboard_auth_address | length > 0 + fail_msg: >- + k3s_traefik_dashboard_host is set but k3s_traefik_dashboard_auth_address + is empty — set both in inventory/group_vars/k3s_cluster.yml, or neither. + Publishing the dashboard without the forward-auth middleware would put a + read-only view of every router, service and middleware on this cluster + on the LAN unauthenticated, so this refuses rather than defaulting to an + address that might not be Authentik. + quiet: true + run_once: true + +- name: Deploy the Traefik HelmChartConfig + ansible.builtin.template: + src: traefik.helmchartconfig.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/traefik-config.yaml + owner: root + group: root + mode: "0644" + become: true + +- name: Deploy the exposed dashboard route + when: k3s_traefik_dashboard_host | length > 0 + ansible.builtin.template: + src: dashboard-ingressroute.yaml.j2 + dest: /var/lib/rancher/k3s/server/manifests/traefik-dashboard.yaml + owner: root + group: root + mode: "0644" + become: true + +# Clearing k3s_traefik_dashboard_host is a teardown, not just a stop-managing: +# k3s's deploy controller garbage-collects the objects a manifest created when +# the manifest is removed, so deleting this file withdraws the public route, +# its middleware and its certificate. Same reasoning as roles/k3s_app's +# removal path — with the difference that this one needs no `state: absent` +# tombstone, because the hostname is a single value rather than a list entry +# that could be silently dropped. +- name: Withdraw the exposed dashboard route when no host is configured + when: k3s_traefik_dashboard_host | length == 0 + ansible.builtin.file: + path: /var/lib/rancher/k3s/server/manifests/traefik-dashboard.yaml + state: absent + become: true diff --git a/build/config/ansible/roles/k3s_traefik/templates/dashboard-ingressroute.yaml.j2 b/build/config/ansible/roles/k3s_traefik/templates/dashboard-ingressroute.yaml.j2 new file mode 100644 index 0000000..e869918 --- /dev/null +++ b/build/config/ansible/roles/k3s_traefik/templates/dashboard-ingressroute.yaml.j2 @@ -0,0 +1,83 @@ +{# + Managed by Ansible (roles/k3s_traefik) — do not edit on the node. + + Plain manifests, not a HelmChart — same pattern as + roles/k3s_metallb/templates/metallb-config.yaml.j2 and + roles/k3s_cert_manager/templates/cluster-issuer.yaml.j2: CRs belonging to + charts installed elsewhere, dropped in the auto-deploying directory and + retried by k3s's deploy controller until the CRDs they need exist. + + Rendered only when k3s_traefik_dashboard_host is set; tasks/main.yml + deletes this file when it isn't, and k3s's deploy controller + garbage-collects what the file created. + + traefik.io/v1alpha1, not traefik.containo.us/v1alpha1 — the group changed + with Traefik v3, which is what current k3s bundles. +-#} +{# Explicit Certificate rather than the cert-manager.io/cluster-issuer + annotation every app's Ingress uses: cert-manager watches Ingress + resources, and this route can't be one (see below). DNS-01, so this + issues without the hostname resolving anywhere yet. -#} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: traefik-dashboard + namespace: {{ k3s_traefik_namespace }} +spec: + secretName: {{ k3s_traefik_dashboard_tls_secret }} + issuerRef: + name: {{ k3s_traefik_dashboard_cert_issuer }} + kind: ClusterIssuer + dnsNames: + - {{ k3s_traefik_dashboard_host }} +--- +{# Authentik, in forward-auth (domain level) mode — see defaults/main.yml for + why domain level and not single-application. Fails closed: if Authentik is + down this returns an error rather than passing the request through, which + is the correct direction and the reason the port-forward path stays. -#} +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: {{ k3s_traefik_dashboard_auth_middleware }} + namespace: {{ k3s_traefik_namespace }} +spec: + forwardAuth: + address: {{ k3s_traefik_dashboard_auth_address }} + # Traefik strips X-Forwarded-* from client requests by default; Authentik + # needs them to know which host and scheme the user actually asked for, + # and builds its redirect back out of them. Safe here because the only + # thing that can set them is Traefik itself — nothing reaches this + # middleware without passing through the entrypoint first. + trustForwardHeader: true + authResponseHeaders: +{% for header in k3s_traefik_dashboard_auth_response_headers %} + - {{ header }} +{% endfor %} +--- +{# An IngressRoute rather than an Ingress, and not by preference: the + dashboard is served by api@internal, a Traefik-internal service with no + Kubernetes Service behind it, so there is nothing for an Ingress backend + to name. That single fact is why the Certificate above is explicit and why + the middleware is attached here rather than by annotation. -#} +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: traefik-dashboard + namespace: {{ k3s_traefik_namespace }} +spec: + entryPoints: + - websecure + routes: + # Both prefixes, because the dashboard is a static bundle under + # /dashboard/ that reads its data from /api — serving the first without + # the second gets you a page that loads and then stays empty. + - kind: Rule + match: Host(`{{ k3s_traefik_dashboard_host }}`) && (PathPrefix(`/dashboard`) || PathPrefix(`/api`)) + middlewares: + - name: {{ k3s_traefik_dashboard_auth_middleware }} + namespace: {{ k3s_traefik_namespace }} + services: + - kind: TraefikService + name: api@internal + tls: + secretName: {{ k3s_traefik_dashboard_tls_secret }} diff --git a/build/config/ansible/roles/k3s_traefik/templates/traefik.helmchartconfig.yaml.j2 b/build/config/ansible/roles/k3s_traefik/templates/traefik.helmchartconfig.yaml.j2 new file mode 100644 index 0000000..f70814e --- /dev/null +++ b/build/config/ansible/roles/k3s_traefik/templates/traefik.helmchartconfig.yaml.j2 @@ -0,0 +1,34 @@ +{# + Managed by Ansible (roles/k3s_traefik) — do not edit on the node. + + A HelmChartConfig, not a HelmChart, because this is the one chart on the + cluster this repo doesn't install: k3s installs Traefik itself, rewriting + /var/lib/rancher/k3s/server/manifests/traefik.yaml on every server start + and overwriting whatever is in it. A HelmChartConfig carrying the same + name and namespace as that HelmChart is the supported way in — k3s's + helm-controller merges its valuesContent over the chart's values on the + next reconcile. Editing traefik.yaml directly works until the next + restart, and then silently doesn't. + + Hence also the filename this renders to (traefik-config.yaml): anything + called traefik.yaml here belongs to k3s and would be reclaimed. +-#} +apiVersion: helm.cattle.io/v1 +kind: HelmChartConfig +metadata: + name: traefik + namespace: {{ k3s_traefik_namespace }} +spec: + valuesContent: |- + ingressRoute: + dashboard: + # Traefik builds the dashboard either way — `api.dashboard` is on by + # default. What the chart stopped shipping in v28 is the router that + # reaches it, which is why a stock k3s answers 404 to + # :9000/dashboard/ while :9000/ping happily returns 200. This puts + # that router back, on Traefik's internal `traefik` entrypoint + # (9000), which isn't published on the Service — so it is reachable + # by `kubectl port-forward` and nothing else. See defaults/main.yml + # for why that stays true even once the dashboard is also exposed on + # a hostname. + enabled: {{ k3s_traefik_dashboard_enabled | bool | lower }} diff --git a/build/config/ansible/roles/lxc_app/defaults/main.yml b/build/config/ansible/roles/lxc_app/defaults/main.yml new file mode 100644 index 0000000..a4f0e89 --- /dev/null +++ b/build/config/ansible/roles/lxc_app/defaults/main.yml @@ -0,0 +1,2 @@ +--- +vault_kv_mount: kv diff --git a/build/config/ansible/roles/lxc_app/tasks/main.yml b/build/config/ansible/roles/lxc_app/tasks/main.yml new file mode 100644 index 0000000..b1007e0 --- /dev/null +++ b/build/config/ansible/roles/lxc_app/tasks/main.yml @@ -0,0 +1,124 @@ +--- +# Installs one app (one loop iteration of `app` from playbooks/proxmox.yml) +# natively into a Proxmox LXC: merge portable + Proxmox config, fetch secrets +# from Vault, provision its database if it declares one, run the app's own +# install steps, then manage its systemd unit. +# +# The Proxmox counterpart to `compose_stack`. Everything up to "Run +# app-specific install steps" is deliberately the same shape as that role — +# same vars merge, same Vault lookup, same DB provisioning — so config lives +# in one place regardless of which platform an app lands on. What differs is +# the deployment primitive: a systemd service built from packages/binaries +# rather than `docker compose up -d`. +# +# UNPROVEN: `src/shared/postgres/` and `src/forgejo/` both ship an +# `ansible/proxmox/install.yml` now, but neither has been run end to end +# against a real LXC. The structure mirrors proven code; the details are not +# yet exercised. + +- name: Set app facts + ansible.builtin.set_fact: + app_local_dir: "{{ repo_root }}/src/{{ app.src }}" + +- name: Load portable app variables + ansible.builtin.include_vars: + file: "{{ app_local_dir }}/common/vars.yml" + name: app_common_vars + +- name: Load Proxmox-specific app variables + ansible.builtin.include_vars: + file: "{{ app_local_dir }}/ansible/proxmox/vars.yml" + name: app_platform_vars + +- name: Merge app configuration + ansible.builtin.set_fact: + # Portable values first, Proxmox-specific overrides on top. Available to + # the app's install.yml, alongside `vault_secrets` below. + app_config: >- + {{ (app_common_vars.env_defaults | default({})) + | combine(app_platform_vars.env_defaults | default({})) }} + +- name: Look up app secrets from Vault + ansible.builtin.set_fact: + vault_secrets: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + app.vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + no_log: true + +- name: Provision application database + when: app.db is defined + no_log: true + block: + - name: Look up Postgres superuser credentials from Vault + ansible.builtin.set_fact: + pg_admin_secrets: >- + {{ lookup('community.hashi_vault.vault_kv2_get', + app.db.admin_vault_path, + engine_mount_point=vault_kv_mount, + url=vault_addr, + auth_method=vault_auth_method, + role_id=vault_role_id | default(omit), + secret_id=vault_secret_id | default(omit)).secret }} + + - name: Ensure application database role exists + community.postgresql.postgresql_user: + name: "{{ app.db.user }}" + password: "{{ vault_secrets[app.db.password_vault_key] }}" + login_host: "{{ app.db.provision_host }}" + login_port: "{{ app.db.provision_port }}" + login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}" + login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}" + state: present + delegate_to: localhost + become: false + + # Role first, then the database with `owner:` — the shape roles/k3s_app + # already uses, and not the `postgresql_user` + `priv: ALL` this role was + # written with. Two independent reasons it had to change: + # + # 1. `priv` was deprecated in community.postgresql 3.x and REMOVED in + # 4.0.0, so the old call is a hard "Unsupported parameters" failure + # on any current collection. requirements.yml asks for >=3.0.0, + # which installs 4.x. + # 2. On this instance it would have been wrong even if it still worked. + # `priv: ALL` grants database-level privileges (CONNECT, CREATE, + # TEMPORARY); since Postgres 15 the `public` schema no longer grants + # CREATE to PUBLIC, so a role holding all of those still cannot + # create a table. The Proxmox cluster is 17 (the Unraid one is 13, + # which is why the same code never failed that way there). Making + # the app role own the database covers it on both: on 15+ `public` + # is owned by `pg_database_owner`, which resolves to whoever owns + # the database. + # + # Ownership also makes a pg_dump restore land correctly, for the reason + # spelled out at the same tasks in roles/k3s_app. + - name: Ensure application database exists + community.postgresql.postgresql_db: + name: "{{ app.db.name }}" + owner: "{{ app.db.user }}" + login_host: "{{ app.db.provision_host }}" + login_port: "{{ app.db.provision_port }}" + login_user: "{{ pg_admin_secrets.POSTGRES_SUPERUSER }}" + login_password: "{{ pg_admin_secrets.POSTGRES_SUPERUSER_PASSWORD }}" + state: present + delegate_to: localhost + become: false + +# Everything genuinely app-specific — fetching the binary or package, creating +# the service user, laying out data directories, rendering the app's own +# config file from `app_config` + `vault_secrets`, and installing a systemd +# unit — lives with the app, not here. See src//ansible/proxmox/. +- name: Run app-specific install steps + ansible.builtin.include_tasks: "{{ app_local_dir }}/ansible/proxmox/install.yml" + +- name: Enable and start the service + ansible.builtin.systemd_service: + name: "{{ app.service_name | default(app.name) }}" + enabled: true + state: started + daemon_reload: true diff --git a/build/config/ansible/roles/pve_backup/defaults/main.yml b/build/config/ansible/roles/pve_backup/defaults/main.yml new file mode 100644 index 0000000..ba8e34e --- /dev/null +++ b/build/config/ansible/roles/pve_backup/defaults/main.yml @@ -0,0 +1,61 @@ +--- +# Backup storage on the NAS, and the vzdump job that writes to it. Override +# per-node in inventory/host_vars/.yml. + +# Name the storage appears under in Proxmox. Also what `pve_backup_job` +# targets. +pve_backup_storage: nas-backup + +# `nfs` or `pbs`. NFS is the plain option and needs nothing on the NAS beyond +# an export; PBS gets deduplication and incremental backups but needs a Proxmox +# Backup Server to exist first. See tasks/main.yml for what each consumes. +pve_backup_storage_type: nfs + +# NFS (pve_backup_storage_type: nfs) +pve_backup_nfs_server: "" +pve_backup_nfs_export: "" +# 4.2 rather than 3: better locking and no separate portmapper. +pve_backup_nfs_options: vers=4.2 + +# PBS (pve_backup_storage_type: pbs) +pve_backup_pbs_server: "" +pve_backup_pbs_datastore: "" +pve_backup_pbs_username: "" +pve_backup_pbs_password: "" +pve_backup_pbs_fingerprint: "" + +# How many archives the storage keeps. Proxmox prunes archives, not their +# contents — anything inside a guest (such as Forgejo's nightly pg_dump output) +# is pruned by that guest, on its own schedule. +pve_backup_prune: + keep-daily: 7 + keep-weekly: 4 + keep-monthly: 6 + +# systemd calendar event. Guests that dump a database into their own filesystem +# first must finish before this — see src/forgejo/ansible/proxmox/README.md. +pve_backup_schedule: "02:00" + +# Stable marker used to find this job again on later runs. The Proxmox API +# assigns job IDs itself, so matching on the comment is what makes this +# idempotent rather than creating a duplicate job every deploy. +pve_backup_comment: managed-by-homelab-iac + +# Which guests to back up. `true` covers everything on the node, including +# guests added later without an edit here; set it false and list VMIDs in +# pve_backup_vmids to be selective. +pve_backup_all: true +pve_backup_vmids: [] + +# snapshot: no downtime, and the guest filesystem is captured as if it had +# crashed at that instant. Fine for Forgejo — git repositories tolerate it, and +# the database is a separate consistent dump rather than live files. +pve_backup_mode: snapshot +pve_backup_compress: zstd +pve_backup_enabled: true + +# Optional: address Proxmox mails backup reports to. Left empty means the node +# default applies. PVE 8 moved most of this to its notification-target system; +# `mailto` still works, but if you want anything richer, configure a +# notification target in the UI rather than extending this role. +pve_backup_mailto: "" diff --git a/build/config/ansible/roles/pve_backup/tasks/main.yml b/build/config/ansible/roles/pve_backup/tasks/main.yml new file mode 100644 index 0000000..b9890ea --- /dev/null +++ b/build/config/ansible/roles/pve_backup/tasks/main.yml @@ -0,0 +1,161 @@ +--- +# Configures backups on a Proxmox node: a storage on the NAS to write archives +# to, and a scheduled vzdump job that fills it. +# +# This is the off-box copy for every Proxmox guest. Guests deliberately keep +# all their state on their own rootfs rather than bind-mounting anything in, +# because vzdump excludes bind mounts — see src/forgejo/terraform/README.md. +# Where an app's data lives outside the container (the shared Postgres), the +# app dumps it into its own filesystem on a timer that runs before the window +# below, so one archive is one restore point. +# +# Driven through `pvesm`/`pvesh` rather than modules: as of writing, +# `community.proxmox` has no storage or backup-job module. Swap them in if that +# changes — the shape here is create-if-absent, update-if-drifted, which is +# what a module would do anyway. +# +# UNVERIFIED — this has not been run against a real node. `pvesm add` and +# `pvesh create /cluster/backup` accept slightly different options across PVE +# versions; check yours with `pvesm help add`, `pvesh help create +# /cluster/backup`, and a `--check` run before trusting it. + +- name: Check that the storage backend is configured + ansible.builtin.assert: + that: + - pve_backup_storage_type in ['nfs', 'pbs'] + - pve_backup_storage_type != 'nfs' or (pve_backup_nfs_server | length > 0 + and pve_backup_nfs_export | length > 0) + - pve_backup_storage_type != 'pbs' or (pve_backup_pbs_server | length > 0 + and pve_backup_pbs_datastore | length > 0) + - pve_backup_all or (pve_backup_vmids | length > 0) + fail_msg: >- + pve_backup is missing required settings for + pve_backup_storage_type={{ pve_backup_storage_type }}. Set them in + inventory/host_vars/{{ inventory_hostname }}.yml. + quiet: true + +- name: Build the prune-backups option string + ansible.builtin.set_fact: + pve_backup_prune_string: >- + {% for key, value in pve_backup_prune.items() %}{{ key }}={{ value }}{% if not loop.last %},{% endif %}{% endfor %} + +# --- Storage ------------------------------------------------------------- + +- name: Read the configured storages + ansible.builtin.command: + argv: [pvesh, get, /storage, --output-format, json] + register: pve_storages + changed_when: false + check_mode: false + +- name: Add the backup storage (NFS) + ansible.builtin.command: + argv: + - pvesm + - add + - nfs + - "{{ pve_backup_storage }}" + - --server + - "{{ pve_backup_nfs_server }}" + - --export + - "{{ pve_backup_nfs_export }}" + - --options + - "{{ pve_backup_nfs_options }}" + - --content + - backup + - --prune-backups + - "{{ pve_backup_prune_string }}" + when: + - pve_backup_storage_type == 'nfs' + - pve_backup_storage not in (pve_storages.stdout | from_json | map(attribute='storage') | list) + +- name: Add the backup storage (PBS) + # The password goes on a command line, which is visible in the node's process + # list for the moment the command runs. `pvesm` has no stdin form, so the + # alternative is configuring PBS by hand — acceptable for a one-time create + # that only fires when the storage is absent. + ansible.builtin.command: + argv: + - pvesm + - add + - pbs + - "{{ pve_backup_storage }}" + - --server + - "{{ pve_backup_pbs_server }}" + - --datastore + - "{{ pve_backup_pbs_datastore }}" + - --username + - "{{ pve_backup_pbs_username }}" + - --password + - "{{ pve_backup_pbs_password }}" + - --fingerprint + - "{{ pve_backup_pbs_fingerprint }}" + - --content + - backup + - --prune-backups + - "{{ pve_backup_prune_string }}" + when: + - pve_backup_storage_type == 'pbs' + - pve_backup_storage not in (pve_storages.stdout | from_json | map(attribute='storage') | list) + no_log: true + +# --- Backup job ---------------------------------------------------------- + +- name: Read the configured backup jobs + ansible.builtin.command: + argv: [pvesh, get, /cluster/backup, --output-format, json] + register: pve_backup_jobs + changed_when: false + check_mode: false + +# Proxmox assigns job IDs itself, so there is nothing stable to key on except +# a comment we set. Without this the role would add a duplicate job on every +# run. +- name: Find a job this role already created + ansible.builtin.set_fact: + pve_backup_existing: >- + {{ (pve_backup_jobs.stdout | from_json) + | selectattr('comment', 'defined') + | selectattr('comment', 'equalto', pve_backup_comment) + | list | first | default({}) }} + +- name: Build the vzdump job arguments + ansible.builtin.set_fact: + pve_backup_job_args: >- + {{ ['--schedule', pve_backup_schedule, + '--storage', pve_backup_storage, + '--mode', pve_backup_mode, + '--compress', pve_backup_compress, + '--comment', pve_backup_comment, + '--enabled', ('1' if pve_backup_enabled else '0')] + + (['--all', '1'] if pve_backup_all + else ['--all', '0', '--vmid', pve_backup_vmids | join(',')]) + + (['--mailto', pve_backup_mailto] if pve_backup_mailto | length > 0 else []) }} + +- name: Create the vzdump job + ansible.builtin.command: + argv: "{{ ['pvesh', 'create', '/cluster/backup'] + pve_backup_job_args }}" + when: pve_backup_existing | length == 0 + +# Compared field by field rather than blindly re-applying, so the role reports +# a change only when there is one. The VMID list is compared as the +# comma-joined string Proxmox stores, which makes it order-sensitive — reorder +# pve_backup_vmids and you get one no-op update. +- name: Work out whether the existing job still matches + ansible.builtin.set_fact: + pve_backup_drifted: >- + {{ pve_backup_existing.schedule | default('') != pve_backup_schedule + or pve_backup_existing.storage | default('') != pve_backup_storage + or pve_backup_existing.mode | default('') != pve_backup_mode + or (pve_backup_existing.enabled | default(1) | int == 1) != pve_backup_enabled + or (pve_backup_existing.all | default(0) | int == 1) != pve_backup_all + or (not pve_backup_all + and pve_backup_existing.vmid | default('') != pve_backup_vmids | join(',')) }} + when: pve_backup_existing | length > 0 + +- name: Update the vzdump job if its settings drifted + ansible.builtin.command: + argv: "{{ ['pvesh', 'set', '/cluster/backup/' ~ pve_backup_existing.id] + pve_backup_job_args }}" + when: + - pve_backup_existing | length > 0 + - pve_backup_drifted | bool diff --git a/build/config/ansible/roles/unattended_upgrades/defaults/main.yml b/build/config/ansible/roles/unattended_upgrades/defaults/main.yml new file mode 100644 index 0000000..0a37495 --- /dev/null +++ b/build/config/ansible/roles/unattended_upgrades/defaults/main.yml @@ -0,0 +1,19 @@ +--- +# Origins unattended-upgrades installs from — Ubuntu's own stock list +# (security, plus the ESM ones, which are inert without Ubuntu Pro attached +# rather than an error). Override per-host/group for less than this. +unattended_upgrades_origins: + - "${distro_id}:${distro_codename}" + - "${distro_id}:${distro_codename}-security" + - "${distro_id}ESMApps:${distro_codename}-apps-security" + - "${distro_id}ESM:${distro_codename}-infra-security" + +# Never reboot automatically. playbooks/k3s_maintenance.yml (role +# k3s_maintenance) does that instead — draining the node first, one at a +# time — which is the entire reason this role exists rather than just +# `apt install unattended-upgrades` with Ubuntu's own defaults (which reboot +# at 02:00 unprompted, k3s node or not). +unattended_upgrades_automatic_reboot: false + +unattended_upgrades_remove_unused_deps: true +unattended_upgrades_remove_unused_kernel_packages: true diff --git a/build/config/ansible/roles/unattended_upgrades/handlers/main.yml b/build/config/ansible/roles/unattended_upgrades/handlers/main.yml new file mode 100644 index 0000000..07ebbff --- /dev/null +++ b/build/config/ansible/roles/unattended_upgrades/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: restart unattended-upgrades + ansible.builtin.systemd_service: + name: unattended-upgrades + state: restarted diff --git a/build/config/ansible/roles/unattended_upgrades/tasks/main.yml b/build/config/ansible/roles/unattended_upgrades/tasks/main.yml new file mode 100644 index 0000000..e93a553 --- /dev/null +++ b/build/config/ansible/roles/unattended_upgrades/tasks/main.yml @@ -0,0 +1,43 @@ +--- +# Debian/Ubuntu-generic — not k3s-specific, kept independent of roles/k3s_node +# so it can be pointed at any apt-based host later (a Proxmox guest, say) +# without dragging k3s along. Currently only applied to k3s_cluster, from +# playbooks/k3s.yml. +# +# Installs updates hands-off, but leaves rebooting to +# playbooks/k3s_maintenance.yml — see unattended_upgrades_automatic_reboot in +# defaults/main.yml for why. + +- name: Install unattended-upgrades + ansible.builtin.apt: + name: + - unattended-upgrades + - update-notifier-common + state: present + update_cache: true + cache_valid_time: 3600 + +- name: Configure unattended-upgrades + ansible.builtin.template: + src: 50unattended-upgrades.j2 + dest: /etc/apt/apt.conf.d/50unattended-upgrades + owner: root + group: root + mode: "0644" + notify: restart unattended-upgrades + +- name: Enable periodic apt updates and unattended-upgrades + ansible.builtin.copy: + dest: /etc/apt/apt.conf.d/20auto-upgrades + content: | + APT::Periodic::Update-Package-Lists "1"; + APT::Periodic::Unattended-Upgrade "1"; + owner: root + group: root + mode: "0644" + +- name: Ensure unattended-upgrades is enabled and running + ansible.builtin.systemd_service: + name: unattended-upgrades + enabled: true + state: started diff --git a/build/config/ansible/roles/unattended_upgrades/templates/50unattended-upgrades.j2 b/build/config/ansible/roles/unattended_upgrades/templates/50unattended-upgrades.j2 new file mode 100644 index 0000000..01b5944 --- /dev/null +++ b/build/config/ansible/roles/unattended_upgrades/templates/50unattended-upgrades.j2 @@ -0,0 +1,17 @@ +// Managed by Ansible (roles/unattended_upgrades) — edits here are overwritten +// on the next run. + +Unattended-Upgrade::Allowed-Origins { +{% for origin in unattended_upgrades_origins %} + "{{ origin }}"; +{% endfor %} +}; + +Unattended-Upgrade::Remove-Unused-Dependencies "{{ unattended_upgrades_remove_unused_deps | lower }}"; +Unattended-Upgrade::Remove-Unused-Kernel-Packages "{{ unattended_upgrades_remove_unused_kernel_packages | lower }}"; + +// The whole point of this role: never reboot on its own. A reboot-required +// package (kernel, libc, ...) is left applied-but-inactive until +// playbooks/k3s_maintenance.yml (role k3s_maintenance) drains this node and +// reboots it deliberately, one node at a time. +Unattended-Upgrade::Automatic-Reboot "{{ unattended_upgrades_automatic_reboot | lower }}"; diff --git a/build/config/terraform/.terraform.lock.hcl b/build/config/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..4ae0b84 --- /dev/null +++ b/build/config/terraform/.terraform.lock.hcl @@ -0,0 +1,23 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/bpg/proxmox" { + version = "0.111.1" + hashes = [ + "h1:ML2D3UUZTM99yrll/EBXj7wBYMb8xmQgomqFNybEoxY=", + "zh:18fb7c31a08dde6bffa1a4d4a211e604d6d17eec7092fd59331b3db3c6f3742c", + "zh:1cd60761538289d4dd2a1086b3ae62a7b0bdd4b1a2f824e9a44e243413168dba", + "zh:2eb76f6fc8299b6820ff678c8252332cc3366e226b5ae2e61748fd2449c1ed92", + "zh:45e6f7ebd0bf48911d37060359a4f359b5743b3092e985295733990e406d0416", + "zh:4aa8ba912eae37975d2e983394d173e595ca34fc76b5bf220b37d0e99d76e98c", + "zh:58e0789923103a77d502a0a9fc3eb920625e8eb935ec2d4ac0d006aebd1d186c", + "zh:6df8aa85fb8865915537e946c19b02538ad188018a629759c213c6f03730f642", + "zh:6ed47bc00d0913a1d0880618fa1376115e9edab6b4a658c081061a7f0e4ca360", + "zh:c5b10ff4f33df7e4c29e8f1127d49845b561b37b57517e844fb0954d7923d65e", + "zh:d016510e14b738499f0db9d9b3aafe82fc6877fb4ab4e9f831fb68a8d70a1385", + "zh:d941f394069bbf24351b363da1c64383f487067aaee0a84f9b96476d4912e212", + "zh:ddf271dbc2632ae8ffa8de3972f243ee47d260cb2ac90aa784f2746d98e21a0f", + "zh:ed0caa3501c42f611b7e9622c9b1df69fd85dc25a3cd88d3076381829688cd62", + "zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597", + ] +} diff --git a/build/config/terraform/README.md b/build/config/terraform/README.md new file mode 100644 index 0000000..e05550a --- /dev/null +++ b/build/config/terraform/README.md @@ -0,0 +1,66 @@ +# build/config/terraform + +Centralized Terraform config for Proxmox — provider, backend, and the module +calls that say what infrastructure exists. The counterpart to +`build/config/ansible`, which holds inventory and the shared roles. + +## Division of labour + +Terraform's job stops at the guest. It creates the LXC (template, cores, +memory, disk, IP) and nothing more; Ansible then installs the app into it via +the `lxc_app` role. That split is why Ansible is the common tool across both +platforms and Terraform is Proxmox-only: + +| | Unraid | Proxmox | +|---|---|---| +| Provision the host | n/a — it already exists | Terraform (this directory) | +| Install the app | Ansible + `compose_stack` (Docker Compose) | Ansible + `lxc_app` (native + systemd) | +| App config | `src//common/vars.yml` + `src//ansible/unraid/vars.yml` | `src//common/vars.yml` + `src//ansible/proxmox/vars.yml` | + +Per-app LXC specs live in `src//terraform/` as modules, called from +`main.tf` here. + +## State + +`backend "pg"` — specifically against the **CloudNativePG cluster on the k3s +Pis**, reached on its MetalLB address (`k3s_postgres_loadbalancer_ip`, +192.168.50.81). *Not* the shared Postgres this configuration itself +provisions on Proxmox, which would be circular: Terraform would need the +database to exist in order to create the container the database runs in. +State goes somewhere Terraform has no hand in building, which breaks the +cycle outright. Local state would make Terraform workstation-only; CI needs +to see the same state. + +The connection string holds a password, so it is passed at init rather than +committed: + +```sh +terraform init \ + -backend-config="conn_str=postgres://terraform:$PG_PASSWORD@192.168.50.81:5432/terraform_state" +``` + +The bootstrap dependency this creates is on **k3s, not Unraid**: the cluster +and its CNPG instance must be up, with a `terraform_state` database and a +`terraform` role created on it, before `terraform init` works. That is the +one cross-platform dependency the rest of the repo avoids, and it is +deliberate — the alternative is a local-state-then-`init -migrate-state` +dance that has to be got right exactly once. Ordering is in +`docs/postgres-proxmox.md`. + +## Credentials + +Proxmox API token and the Postgres connection string both come from Vault +(`homelab/ci/proxmox`, `homelab/ci/terraform`), exported as environment +variables before running — the same pattern `group_vars/all.yml` uses for +`VAULT_ADDR` and friends. Nothing authenticating to anything is committed. + +```sh +export TF_VAR_proxmox_endpoint='https://turtle-proxmox-01.home.turtlesystems.co.uk:8006/' +export TF_VAR_proxmox_api_token='root@pam!terraform=' +``` + +## Status + +Not yet applied against anything. The provider version is intentionally +unpinned until the first `terraform init` — pin what it resolves and commit +`.terraform.lock.hcl` (which is not gitignored). diff --git a/build/config/terraform/main.tf b/build/config/terraform/main.tf new file mode 100644 index 0000000..c37c86e --- /dev/null +++ b/build/config/terraform/main.tf @@ -0,0 +1,102 @@ +# Per-app LXC modules are called from here, so there is one place that knows +# what infrastructure exists. Each module lives with its app in +# src//terraform/ and outputs the guest's address, which the Ansible +# side then picks up from the Proxmox API (inventory/proxmox.yml) rather than +# by wiring outputs into inventory by hand. + +# The shared Postgres, and the first module here that is infrastructure other +# modules depend on rather than an app in its own right. Declared first for +# readability only — Terraform orders by dependency, not by position, and +# nothing else in this file references it. The ordering that does matter is +# on the Ansible side, where a guest's `db:` provisioning needs this cluster +# already answering; see the runbook in docs/postgres-proxmox.md. +module "postgres" { + source = "../../../src/shared/postgres/terraform" + + node_name = var.proxmox_node + template_file_id = var.lxc_template_file_id + ssh_public_keys = var.ssh_public_keys + + hostname = "postgres" + vm_id = 161 + ip_address = "192.168.50.54/24" + gateway = "192.168.50.254" + + cores = 2 + memory = 2048 + + # Sized for every database this host will ever hold, not for the OS. The + # one-volume rule below means growth happens here rather than by adding a + # second disk: `terraform apply` turns an increase into a `pct resize` of + # the rootfs, which on ZFS is a refquota change — online, no data move, no + # filesystem grow step. It is one-way, though; shrinking is a replacement + # and `prevent_destroy` blocks it, so overshoot rather than creep upward. + disk_size = 256 + + # The ZFS pool, so the guest's volume can be replicated at all, and the + # second node to replicate it to. Both are the whole point of this module + # differing from the one below — see src/shared/postgres/terraform/main.tf. + datastore_id = "AppData" + replication_target_node = var.proxmox_replication_node +} + +module "forgejo" { + source = "../../../src/forgejo/terraform" + + # The second node, not `var.proxmox_node` — a deliberate split of the two + # guests across the cluster rather than an oversight. Written as a literal + # here for the same reason `hostname`/`vm_id`/`ip_address` are: it states + # where this one guest goes. `var.proxmox_replication_node` happens to hold + # the same string today, but it means "where Postgres replicates to", and + # borrowing it would tie Forgejo's placement to a decision about the + # database. + # + # Safe to change only while this container does not exist. Once it does, the + # provider treats `node_name` as a replacement — and `prevent_destroy` in + # the module turns that into a failed plan, which is the intended outcome: + # moving a live Forgejo between nodes is a Proxmox migration, not a + # `terraform apply`. + node_name = "turtle-proxmox-02" + template_file_id = var.lxc_template_file_id + ssh_public_keys = var.ssh_public_keys + + hostname = "forgejo" + vm_id = 160 + ip_address = "192.168.50.52/24" + gateway = "192.168.50.254" + + cores = 2 + memory = 2048 + + # Overrides the module default only in the sense of restating it; both are + # 256 GiB. Growing this is one-way, same as the Postgres guest above: + # `terraform apply` turns an increase into an online `pct resize` of the + # rootfs, but shrinking is a replacement and `prevent_destroy` blocks it. + # Sized generously because *everything* Forgejo owns is on this volume — + # repos, LFS, attachments, indexers, and the local database dumps. + disk_size = 256 + + # The ZFS pool, which exists on both nodes, rather than the module's + # `local-lvm` default — this cluster has no storage by that name. No + # replication job to go with it, unlike the Postgres guest above: Forgejo's + # off-box copy is the vzdump archive `pve_backup` writes to the NAS, which + # is a complete filesystem restore point precisely because nothing here is + # bind-mounted. + datastore_id = "AppData" +} + +# Consumed by the Proxmox nodes' host_vars, which can pin the vzdump job to +# explicit VMIDs. Kept as an output rather than restated there so the two +# can't drift. +output "backup_vmids" { + description = "VMIDs of the guests this configuration creates, for the vzdump job." + value = [module.postgres.vm_id, module.forgejo.vm_id] +} + +# The address every app's `db.provision_host` and `DB_HOST` has to agree with. +# An output rather than something to look up in the Proxmox UI, because it is +# copied into several files by hand and this is the one authoritative copy. +output "postgres_address" { + description = "LAN address of the shared Postgres LXC." + value = module.postgres.ip_address +} diff --git a/build/config/terraform/providers.tf b/build/config/terraform/providers.tf new file mode 100644 index 0000000..2e3c732 --- /dev/null +++ b/build/config/terraform/providers.tf @@ -0,0 +1,42 @@ +terraform { + required_version = ">= 1.6" + + required_providers { + proxmox = { + source = "bpg/proxmox" + # Deliberately unpinned until the first real `terraform init` — pin to + # whatever it resolves, and commit .terraform.lock.hcl (which is not + # gitignored) so everyone and CI get the same provider. + # version = "~> 0.0" + } + } + + # State in Postgres rather than a local file: CI and your workstation need + # to see the same state, and a local file makes Terraform workstation-only. + # + # Specifically the CloudNativePG cluster on k3s, reached at + # k3s_postgres_loadbalancer_ip — *not* the shared Postgres this + # configuration itself provisions on Proxmox. That would be circular: + # Terraform would need the database to exist in order to create the + # container the database runs in. Putting state on a cluster this + # configuration has no hand in building breaks the cycle outright, which is + # why it is worth the cross-platform dependency the rest of the repo + # otherwise avoids — `terraform apply` now needs the Pis up. + # + # Left empty on purpose — the connection string contains a password, so it + # is supplied at init time instead of being committed: + # + # terraform init \ + # -backend-config="conn_str=postgres://terraform:$PG_PASSWORD@192.168.50.81:5432/terraform_state" + # + # Bootstrap ordering: the k3s cluster and its CNPG instance must exist (and + # a `terraform_state` database be created on it) before `terraform init` + # succeeds. See docs/postgres-proxmox.md. + backend "pg" {} +} + +provider "proxmox" { + endpoint = var.proxmox_endpoint + api_token = var.proxmox_api_token + insecure = var.proxmox_insecure +} diff --git a/build/config/terraform/variables.tf b/build/config/terraform/variables.tf new file mode 100644 index 0000000..9b68051 --- /dev/null +++ b/build/config/terraform/variables.tf @@ -0,0 +1,78 @@ +variable "proxmox_endpoint" { + description = "Proxmox VE API endpoint, e.g. https://turtle-proxmox-01.home.turtlesystems.co.uk:8006/" + type = string +} + +variable "proxmox_api_token" { + description = <<-EOT + Proxmox API token in `USER@REALM!TOKENID=SECRET` form. Sourced from Vault + (homelab/ci/proxmox), never committed — pass via the TF_VAR_proxmox_api_token + environment variable. + EOT + type = string + sensitive = true +} + +variable "proxmox_insecure" { + description = "Skip TLS verification — true while Proxmox has a self-signed cert." + type = bool + default = true +} + +variable "proxmox_node" { + description = <<-EOT + Proxmox node guests are created on. Must be the node's **short** name as + the cluster knows it (`pvecm nodes`), never an FQDN, even though the API + endpoint is one. + + pveproxy compares this against its own short hostname to decide whether a + request is local. An FQDN never matches, so it proxies the request onward + — to itself — over a connection it verifies against the cluster CA, which + a self-signed cert fails. The result is an HTTP 596 carrying an OpenSSL + `certificate verify failed`, which reads like a TLS misconfiguration on + this side and is not: `proxmox_insecure` governs the provider's own + connection and has no bearing on Proxmox's internal one. + EOT + type = string + default = "turtle-proxmox-02" +} + +variable "proxmox_replication_node" { + description = <<-EOT + The other node in the cluster, which guests on a ZFS pool replicate to. + Only the shared Postgres module uses this today; it lives here rather than + in that module's own defaults because it names a node of this cluster, the + same as `proxmox_node` does. + + Null disables replication. Set it explicitly even on a single-node + cluster — a null default is indistinguishable from an oversight, and the + failure mode is a database with no second copy that looks entirely + healthy. + EOT + type = string + default = "turtle-proxmox-01" +} + +variable "lxc_template_file_id" { + description = <<-EOT + Container template every LXC module is built from, e.g. + `local:vztmpl/debian-12-standard_12.7-1_amd64.tar.zst`. Must already be + downloaded on the node — `pveam list local` shows what is there, `pveam + available`/`pveam download local ` fetches one. Not defaulted: the + exact filename moves with each point release, so a stale default would fail + at apply time rather than here. + EOT + type = string +} + +variable "ssh_public_keys" { + description = <<-EOT + Public keys installed for root in every container, so Ansible can reach + them. The matching private key is in Vault at `homelab/ci/ssh` — see "SSH + access" in the top-level README.md. Public keys aren't secret, but they are + site-specific, so pass rather than commit: + + export TF_VAR_ssh_public_keys='["'"$(cat ~/.ssh/unraid_ansible.pub)"'"]' + EOT + type = list(string) +} diff --git a/docs/authentik-migration.md b/docs/authentik-migration.md new file mode 100644 index 0000000..69a4311 --- /dev/null +++ b/docs/authentik-migration.md @@ -0,0 +1,420 @@ +# Migrating Authentik from Unraid to k3s + +A one-time, hand-run cutover: move the Authentik database off Unraid onto the +cluster's CloudNativePG instance, bring Authentik up on k3s from +`src/authentik/`, verify it, then change DNS. + +This is deliberately not automated. It runs once, most of its steps are +irreversible in the wrong order, and the verification between them is a +judgement call rather than a task result — the sort of thing a playbook makes +harder rather than easier. The repo half *is* automated: everything from +"deploy Authentik" onwards is `playbooks/k3s.yml`, and re-running it later is +an ordinary converge. + +Read the whole thing before starting. The dump is taken with Authentik +stopped, so there's a service outage from step 5 to step 11 — budget an hour +and do it when nobody needs to log in. + +--- + +## What moves, and what doesn't + +| | Where it lives now | How it gets across | +|---|---|---| +| Users, groups, applications, providers, flows, tokens, certificates | Postgres | `pg_dump` \| `psql` (step 7) | +| Uploaded icons and flow backgrounds | `/media` on disk | `scp` + `kubectl cp` (step 9) | +| Session cookies | signed by `AUTHENTIK_SECRET_KEY` | carry the key over (step 2) | +| Cache / task queue | in-process | nothing — recent Authentik has no Redis, and the chart has no Redis dependency | + +Authentik stores its certificates (including the signing keys OIDC/SAML +providers use) **in the database**, not in a `certs/` directory, so a +successful database restore carries them. That is what lets integrations like +Shelfarr's OIDC (`src/arr/`) keep working without being re-registered. + +The one piece with no automatic path is anything you mounted into the Unraid +container by hand — custom templates, a `certs/` directory you populated +yourself. Step 1 is where you find out whether you have any. + +--- + +## 1. Gather the facts + +Everything below needs these. Run on the Unraid box (web terminal, or SSH). + +```sh +# Which containers make up the current deployment, and what image/tag. +docker ps -a --format '{{.Names}}\t{{.Image}}\t{{.Status}}' | grep -i authentik +``` + +Write down the **image tag** — that is the Authentik version, and step 3 +depends on it. + +```sh +# Every mount, so you can see where /media lives and spot anything unexpected. +docker inspect authentik-server \ + --format '{{range .Mounts}}{{.Source}} -> {{.Destination}}{{println}}{{end}}' + +# The database connection it currently uses, and the secret key. +docker inspect authentik-server \ + --format '{{range .Config.Env}}{{println .}}{{end}}' | grep -E 'POSTGRES|SECRET_KEY' +``` + +That last command tells you the thing this document can't assume: **which +Postgres the Unraid Authentik talks to**. Two possibilities, and they change +step 7: + +- `AUTHENTIK_POSTGRESQL__HOST=shared-postgres` (or `192.168.50.1`) — it uses + the shared instance this repo already manages, `src/shared/postgres/`. The + superuser password is in Vault at `homelab/shared/postgres`. +- anything else (`authentik-postgresql`, `postgres`, a container name from + Authentik's own compose bundle) — the Unraid UI deployment brought its own + Postgres container. You'll need *its* credentials, which are in that + container's own environment: + + ```sh + docker inspect \ + --format '{{range .Config.Env}}{{println .}}{{end}}' | grep POSTGRES + ``` + + and it is probably not published on a host port, which step 7 handles. + +Also note `AUTHENTIK_SECRET_KEY` from the output above — step 2 needs it +verbatim. + +--- + +## 2. Populate Vault + +```sh +vault kv put kv/homelab/authentik \ + AUTHENTIK_SECRET_KEY='' \ + AUTHENTIK_POSTGRESQL__PASSWORD='' +``` + +Two different rules for those two values, and it's worth being clear about +why: + +- **`AUTHENTIK_SECRET_KEY` must be the existing one.** It signs session + cookies. A fresh value doesn't break anything permanently, but it + invalidates every active session at the moment DNS flips — so instead of a + silent cutover, everyone gets logged out and the migration announces + itself. Copy it across. +- **`AUTHENTIK_POSTGRESQL__PASSWORD` can be new.** You are creating a new + role on a different Postgres; nothing in the dump references it. Generate a + fresh one. + +If you're also using cert-manager's DNS-01 solver (step 4), populate its +credentials now too: + +```sh +vault kv put kv/homelab/k3s-cert-manager \ + CLOUDFLARE_API_TOKEN='' +``` + +--- + +## 3. Confirm the repo's values + +Three values are guesses until you check them. All are flagged in-file. + +**`src/authentik/common/vars.yml`** + +- `AUTHENTIK_VERSION` — **set this to the image tag from step 1**, not to + whatever is newest. Authentik runs Django migrations on startup against + whatever schema it finds, and they are one-way: a newer Authentik pointed + at an older dump upgrades the schema silently, and if the cutover then has + to be rolled back, the Unraid instance can no longer read its own database. + Migrate first, upgrade second, in a separate commit. +- `AUTHENTIK_HOST` / `AUTHENTIK_URL` — the hostname you'll be moving. Must + match what's there today, or every OIDC/SAML redirect URI registered in + Authentik breaks. + +**`build/config/ansible/inventory/group_vars/k3s_cluster.yml`** + +- `k3s_cert_manager_solver` — empty by default, and + `roles/k3s_cert_manager` refuses to run until it's set. The commented + Cloudflare example in that file is the common case; for anything else take + the stanza from + [cert-manager's DNS-01 docs](https://cert-manager.io/docs/configuration/acme/dns01/). + +While you're testing, point `k3s_cert_manager_acme_server` at Let's Encrypt +staging. Production allows five failed validations per hostname per hour, and +exhausting it means waiting rather than retrying. + +--- + +## 4. Deploy cert-manager and confirm the issuer works + +```sh +cd build/config/ansible +ansible-playbook playbooks/k3s.yml --tags cert-manager +``` + +Then check it actually registered — a `ClusterIssuer` is created whether or +not its solver is valid, so "the resource exists" proves nothing: + +```powershell +$env:KUBECONFIG = "$PWD\local\k3s\homelab-utils.kubeconfig" +kubectl get clusterissuer letsencrypt -o jsonpath='{.status.conditions[*].message}' +``` + +You want `The ACME account was registered with the ACME server`. Anything +else, fix it here — not after Authentik is deployed and waiting on a +certificate. + +Worth knowing: with DNS-01, the certificate can be issued **before** DNS +points at the cluster. The challenge is a TXT record, not a request to your +web server, so cert-manager never needs to be reachable. That's why TLS is +sorted out in this step and not after the flip. + +--- + +## 5. Freeze the Unraid instance + +Stop writes before dumping. Skipping this gives you a dump that's missing +whatever happened during it — usually invisible, occasionally a user account +that no longer exists. + +```sh +docker stop authentik-server authentik-worker +``` + +Leave its Postgres running — step 7 reads from it. **The outage starts +here.** + +--- + +## 6. Create the role and database on CNPG + +The restore needs the `authentik` role to exist first so the dump's objects +land with the right ownership. + +```powershell +$env:KUBECONFIG = "$PWD\local\k3s\homelab-utils.kubeconfig" +kubectl -n shared-postgres exec -i shared-postgres-1 -- psql -v ON_ERROR_STOP=1 <<'SQL' +CREATE ROLE authentik LOGIN PASSWORD 'the-password-from-step-2'; +CREATE DATABASE authentik OWNER authentik; +SQL +``` + +`kubectl exec` into a CNPG pod lands you as the `postgres` superuser over the +local socket, so bare `psql` needs no credentials. + +You could instead let `roles/k3s_app` create these — its tasks are idempotent +and will run in step 8 regardless — but doing it by hand here keeps the +database creation and the restore adjacent, and avoids deploying Authentik +against an empty database first (which would have it run initial migrations +and build a schema the restore then has to fight). + +--- + +## 7. Dump and restore + +Run the whole thing **inside the CNPG pod**, piping `pg_dump` straight into +`psql`. Three reasons this is better than dumping to a file on Windows: + +- `pg_dump` must be at least the version of the server being dumped, and the + CNPG pod ships Postgres 18 client tools — dumping the Unraid Postgres 13 + from there is the supported direction. Your workstation probably has no + `pg_dump` at all. +- PowerShell re-encodes bytes passing through `>` and `|`, which silently + corrupts a dump file. Nothing on the Windows filesystem means nothing to + corrupt. +- No intermediate file holding the entire user database in cleartext. + +### If Authentik used the shared Postgres (published on `192.168.50.1:5432`) + +```powershell +$env:KUBECONFIG = "$PWD\local\k3s\homelab-utils.kubeconfig" +$srcPw = Read-Host 'Unraid postgres superuser password' + +kubectl -n shared-postgres exec -i shared-postgres-1 -- ` + env "PGPASSWORD=$srcPw" bash -c @' +set -euo pipefail +{ echo "SET ROLE authentik;" + pg_dump -h 192.168.50.1 -p 5432 -U postgres -d authentik \ + --no-owner --no-privileges +} | psql -v ON_ERROR_STOP=1 -d authentik +'@ +``` + +`SET ROLE authentik` before the dump stream, combined with `--no-owner`, is +what gets ownership right: the session creates every object as `authentik`, +so the application role owns its own schema. Without it, everything lands +owned by `postgres` and Authentik can read but not migrate. + +`ON_ERROR_STOP=1` matters — without it `psql` reports success having skipped +every statement that failed. + +### If it used its own Postgres container (not published) + +Publish it temporarily, then use the command above with the container's own +credentials and port: + +```sh +# on Unraid +docker run -d --rm --name pgbridge -p 55432:5432 \ + --network container: alpine/socat \ + TCP-LISTEN:5432,fork TCP:127.0.0.1:5432 +``` + +then `-h 192.168.50.1 -p 55432 -U ` in the `pg_dump` above, and +`docker stop pgbridge` afterwards. Simpler alternative if you'd rather not: +`docker exec pg_dump ... > /mnt/user/appdata/authentik.sql` on +Unraid, `scp` it to your workstation, and `kubectl exec -i ... -- psql -d +authentik < authentik.sql` — but a plain-SQL file through PowerShell +redirection is exactly the encoding hazard noted above, so run that last +command from WSL. + +### Check the restore + +```powershell +kubectl -n shared-postgres exec -i shared-postgres-1 -- ` + psql -d authentik -c '\dt' -c 'SELECT count(*) FROM authentik_core_user;' +``` + +You should see a long table list and a user count matching what the old +instance had. An empty table list means the restore did nothing. + +--- + +## 8. Deploy Authentik on k3s + +```sh +cd build/config/ansible +ansible-playbook playbooks/k3s.yml --tags apps +``` + +A green run means the manifests landed and the database was provisioned — not +that the workload came up. Watch it actually start: + +```powershell +kubectl -n authentik get pods -w +``` + +The first start takes several minutes on a Pi: pulling the image, then the +worker checking migrations. Because the deployed version matches the dumped +version (step 3), it should find nothing to migrate. + +If pods stay `Pending`, check the PVC — `local-path` uses +`WaitForFirstConsumer`, so a `Pending` PVC alongside a `Pending` pod is +normal only until the scheduler picks a node: + +```powershell +kubectl -n authentik describe pvc authentik-media +kubectl -n authentik logs -l app.kubernetes.io/component=worker --tail=50 +``` + +--- + +## 9. Copy `/media` across + +Two hops rather than a pipe, again to keep binary data out of a PowerShell +pipeline: + +```powershell +# from the media path you noted in step 1 +scp -r root@192.168.50.1:/mnt/user/appdata/authentik/media ./local/authentik-media + +$pod = kubectl -n authentik get pod -l app.kubernetes.io/component=server ` + -o jsonpath='{.items[0].metadata.name}' +kubectl -n authentik cp ./local/authentik-media/. "${pod}:/media" +``` + +`local/` is already gitignored, so the staging copy won't be committed. + +Skip this entirely if you never uploaded custom icons or backgrounds — +Authentik falls back to its built-in assets, and the directory is empty on a +stock install. + +--- + +## 10. Verify before touching DNS + +The hostname still resolves to Unraid, so test by overriding resolution +rather than changing it. `--resolve` sends the request to Traefik with the +right SNI and `Host` header: + +```powershell +curl.exe -sv --resolve auth.turtlesystems.uk:443:192.168.50.80 ` + https://auth.turtlesystems.uk/-/health/ready/ +``` + +Check, in order: + +1. The TLS certificate is the Let's Encrypt one, not Traefik's self-signed + default (`kubectl -n authentik get certificate` → `READY True`). If you + used the staging server in step 3, it will be untrusted — expected; switch + to production and re-run `--tags cert-manager` once satisfied. +2. `/-/health/ready/` returns 204. +3. Add a hosts-file entry pointing the name at `192.168.50.80` and log in + through a browser with a **real user account**. This is the check that + matters — it exercises the restored password hashes, the flows, and the + session cookie signed with the carried-over secret key. +4. Open an existing application from the Authentik dashboard and confirm the + OIDC/SAML round-trip still works (Shelfarr is the one this repo knows + about — see `src/arr/`). + +Remove the hosts-file entry afterwards. + +--- + +## 11. Flip DNS + +Point `auth.turtlesystems.uk` at `192.168.50.80` (Traefik's MetalLB address). + +Lower the record's TTL a few hours beforehand if you can — it shortens the +window in which a rollback is still invisible to clients. + +--- + +## Rollback + +Viable until users start making changes in the new instance; after that, +rolling back loses whatever they did. + +1. Point DNS back at Unraid. +2. `docker start authentik-server authentik-worker`. + +The old database was only ever read, never written, so it's exactly as it was +at step 5. Nothing to restore. + +Do **not** run both instances at once against their separate databases — they +diverge immediately, and there is no way to merge them afterwards. That is +the same "never declare an app on two platforms" rule from CLAUDE.md, and it +is why `src/authentik/ansible/unraid/` is a README rather than a compose file. + +--- + +## After it has settled + +Once you're confident, in this order: + +1. **Delete the Unraid containers.** A UI action, not `state: absent` — this + repo never deployed them, so it has nothing to tear down. Keep the appdata + directory a while longer. +2. **Drop the old database**, if it was on the shared Postgres: + ```sh + docker exec -it shared-postgres psql -U postgres -c 'DROP DATABASE authentik;' + ``` + Take a copy first if you want one; nothing else does. +3. **Upgrade Authentik**, now as an ordinary change: bump + `AUTHENTIK_VERSION` in `src/authentik/common/vars.yml`, read the release + notes for anything between your version and the target, and + `ansible-playbook playbooks/k3s.yml --tags apps -e only_apps=authentik`. + Do this as its own commit, so a bad upgrade is one `git revert` away from + the version you know worked. + +## Backups, afterwards + +Worth stating plainly: the cluster's CNPG has **no backups configured** +(CLAUDE.md → "Key decisions" — HA replication only). Before this migration, +Authentik's database sat on Unraid, inside whatever covers `/mnt/user/appdata`. +After it, the only redundancy is a streaming replica on another Pi, which +protects against a dead SD card and not against a bad migration, a dropped +table, or a mistake in this document. + +The gap this leaves is real and this cutover widens it. CNPG's answer is a +`ScheduledBackup` to object storage or an NFS volume on the NAS; that's not +built here yet, and it's the obvious next piece of work after Authentik is +settled. diff --git a/docs/forgejo-proxmox.md b/docs/forgejo-proxmox.md new file mode 100644 index 0000000..2a65bab --- /dev/null +++ b/docs/forgejo-proxmox.md @@ -0,0 +1,287 @@ +# Bringing up Forgejo on Proxmox + +A run-once bootstrap, same shape as `postgres-proxmox.md` and deliberately +downstream of it — Forgejo's database lives on the shared Postgres that +runbook creates, so none of this works until that one is finished. + +The end state: an LXC on the `AppData` pool of **turtle-proxmox-02** (VMID +160, 192.168.50.52), running Forgejo natively under systemd, with its database +on the shared Postgres LXC at 192.168.50.54 and a nightly `pg_dump` landing on +its own disk in time for the node's 02:00 vzdump. + +## Why turtle-proxmox-02 + +Nothing forces it. Postgres is on `-01`, so putting Forgejo on `-02` splits +the two guests across the cluster: a node going down takes one of them with +it rather than both. Note what that does *not* buy — Forgejo with its database +unreachable is not a working forge, so this is about not losing both +filesystems at once, not about staying up. + +It is set as a literal `node_name` on the `forgejo` module in +`build/config/terraform/main.tf`, not from `var.proxmox_node`. **Change it +only before the container exists**: the provider treats `node_name` as a +replacement, and `prevent_destroy` in `src/forgejo/terraform/main.tf` turns +that into a failed plan. Moving a live Forgejo is a Proxmox migration +(`pct migrate`) followed by editing the literal to match, not a +`terraform apply`. + +## 1. Secrets + +`homelab/forgejo` needs five keys. Four of them may already exist from the +Unraid stack; **`LFS_JWT_SECRET` almost certainly does not** — the Compose +deployment let Forgejo generate it, and the native install renders `app.ini` +in full with `INSTALL_LOCK = true`, so there is no first boot for Forgejo to +invent one on. + +```sh +vault kv get kv/homelab/forgejo +``` + +If `LFS_JWT_SECRET` is missing, generate one and patch it in without +disturbing the others: + +```sh +vault kv patch kv/homelab/forgejo \ + LFS_JWT_SECRET="$(docker run --rm codeberg.org/forgejo/forgejo:10 forgejo generate secret)" +``` + +Full key table in `vault-secrets.md`. `install.yml` asserts on all five before +it touches the container, so a missing one fails on the first task with a +message naming it rather than three tasks later on an undefined variable. + +## 2. The container template + +The template must be present on **turtle-proxmox-02**, not just on `-01` where +Postgres was built. If `StorageOne` is not shared across the cluster this is a +separate download: + +```sh +ssh root@turtle-proxmox-02 'pveam list StorageOne' +# if the ubuntu-26.04 template is absent: +ssh root@turtle-proxmox-02 'pveam update && pveam download StorageOne ubuntu-26.04-standard_26.04-1_amd64.tar.zst' +``` + +Whatever is there has to match `lxc_template_file_id` exactly — the filename +moves with each point release, which is why that variable has no default. + +## 3. Terraform + +State is on the CNPG cluster, so the Pis have to be up (see +`postgres-proxmox.md` § "Why this order"). + +```sh +cd build/config/terraform +terraform init -backend-config=... # as per postgres-proxmox.md +terraform plan -out=deploy.plan -target=module.forgejo +terraform apply deploy.plan +``` + +Read the plan before applying. It should be **one resource to add** and +nothing to change or destroy; anything touching `module.postgres` — in +particular a *replacement* — means stop and work out why, because that guest +is live. + +Note `deploy.plan` embeds the state and every input variable, `sensitive` ones +included, so it holds the Proxmox API token in the clear. `.gitignore` covers +`*.plan`; delete it once applied rather than leaving it in the tree. + +Confirm the guest exists and Ansible can see it: + +```sh +ssh root@turtle-proxmox-02 'pct list | grep 160' +cd ../ansible && ansible-inventory --list --yaml proxmox_guests | grep -A2 forgejo +``` + +If it is absent from the inventory but present in `pct list`, the `terraform` +tag is missing — that filter is what tells this repo's guests from hand-made +ones, and a guest without it is silently never deployed to. + +## 4. Install Forgejo into it + +```sh +cd build/config/ansible +ansible-playbook playbooks/proxmox.yml -e only_stacks=forgejo +``` + +This provisions the `forgejo` database and role on 192.168.50.54, then +installs the binary, `app.ini`, the systemd unit and the dump timer. The +database step runs from **your controller**, not from inside the container +(`delegate_to: localhost`), so 192.168.50.54:5432 has to be reachable from +wherever you run this. + +Check: + +```sh +ssh root@192.168.50.52 'systemctl is-active forgejo && systemctl list-timers forgejo-dbdump.timer' +curl -sI http://192.168.50.52:3000/ | head -1 +``` + +## 4b. The first admin user + +A fresh install has **no way in**: `INSTALL_LOCK = true` skips the setup +wizard, which is where the first admin would normally be created, and +`DISABLE_REGISTRATION = true` means you cannot self-register either (both in +`src/forgejo/ansible/proxmox/templates/app.ini.j2`). Nothing in `install.yml` +creates a user, so the account has to be made from the CLI inside the +container: + +```sh +ssh root@192.168.50.52 + +sudo -u git forgejo admin user create --admin --username --email @turtlesystems.co.uk --password '' --config /etc/forgejo/app.ini --work-path /var/lib/forgejo +``` + +Three things that are easy to get wrong: + +- **`sudo -u git`, not root.** `RUN_USER` is `git`; running the CLI as root + leaves root-owned files under `/var/lib/forgejo` that Forgejo then cannot + write. `app.ini` is `0640 root:git`, so the `git` user can read it. +- **`--config` is not optional.** Forgejo looks for `custom/conf/app.ini` + under the work path by default; this install puts it at + `/etc/forgejo/app.ini` (`FORGEJO_CONFIG_DIR` in `vars.yml`). Without the + flag the CLI reads a config that isn't there and never reaches the + database. +- Recent versions default `--must-change-password` to true, so expect a + forced change on first web login. Pass `--must-change-password=false` to + skip it. Confirm the flags for the pinned `FORGEJO_RELEASE` with + `forgejo admin user create --help` rather than assuming. + +Then log in at `http://192.168.50.52:3000/user/login` — by IP, because +`ROOT_URL` is `https://git.turtlesystems.uk` and nothing resolves there until +the cutover in step 7. The login form works over the IP; some links and +redirects Forgejo renders will point at the not-yet-live hostname. + +**Skip this step if you are restoring a database in step 6.** Users live in +the database, so a dump brings its own admin back and an account created here +is overwritten by the restore. Create one only if this is genuinely an empty +forge — or after the restore, if the dump turns out to have no usable admin. + +## 5. Node backups + +VMID 160 is on `-02`, so it is `-02`'s vzdump job that covers it. +`pve_backup_all: true` in `group_vars/proxmox_nodes.yml` means no edit is +needed — but the job has to actually exist on that node: + +```sh +ansible-playbook playbooks/pve_host.yml --limit turtle-proxmox-02 +ssh root@turtle-proxmox-02 'cat /etc/pve/jobs.cfg' +``` + +The 01:30 dump and the 02:00 vzdump are a pair. Move one, move the other — +see `src/forgejo/ansible/proxmox/README.md`. + +## 6. Repository data + +**Ansible does not move any of this.** A fresh install is a working, empty +forge; the repositories are a separate restore into +`/var/lib/forgejo/data/forgejo-repositories`. + +The original instructions here were an `rsync` off nas2, which no longer +exists as a machine to read from — whatever repository data survives has to +come from wherever it went when that host was retired. After restoring: + +```sh +ssh root@192.168.50.52 'chown -R git:git /var/lib/forgejo/data && systemctl restart forgejo' +``` + +Restore the database the same way, from whatever dump you have, before +starting Forgejo against it — an empty database with populated repositories +gives back every file and no issues, pull requests, users or permissions. + +## 7. Cutover + +Last, once the above is verified: + +- Point `git.turtlesystems.uk` at this container. +- **SSH clone URLs move from port 2222 to 22.** The Unraid stack published + 2222 to dodge the host's own sshd; this container has its own IP and uses + its own sshd, with Forgejo managing the `git` user's `authorized_keys`. + Every existing remote needs editing — there is no redirect for this. +- The Forgejo Actions runner registration is tied to the instance. Re-register + it against the new host, or CI stops running (`.forgejo/workflows/`). + +## 8. Sign-in through Authentik (optional) + +Forgejo supports OIDC, and `app.ini` is already set up for it — but only the +*policy* half. The provider itself cannot be configured from `app.ini`: +Forgejo keeps authentication sources in its **database**. That makes this the +one part of Forgejo's configuration Ansible does not own, and the reason it is +a runbook step rather than a task in `install.yml`. It is run once and then +carried forward by the nightly `pg_dump`, the same as every other row in that +database. + +**Do this after step 7, not before.** The redirect URI has to match `ROOT_URL` +(`https://git.turtlesystems.uk`), so DNS and TLS must already be live. +Configuring it against `http://192.168.50.52:3000` means doing it twice. + +### On the Authentik side + +Create an OAuth2/OIDC **Provider** plus an **Application** for it, as for any +other app on `auth.turtlesystems.uk`. The redirect URI is: + +``` +https://git.turtlesystems.uk/user/oauth2/authentik/callback +``` + +The last path segment is the *name of the auth source in Forgejo*, not a fixed +string — it has to match the `--name` below. Store the generated client secret +in Vault (`vault-secrets.md` → `OIDC_CLIENT_SECRET`). + +### On the Forgejo side + +```sh +ssh root@192.168.50.52 + +sudo -u git forgejo admin auth add-oauth --name authentik --provider openidConnect --key --secret --auto-discover-url https://auth.turtlesystems.uk/application/o/forgejo/.well-known/openid-configuration --config /etc/forgejo/app.ini --work-path /var/lib/forgejo +``` + +Same `sudo -u git` and `--config` requirements as step 4b, for the same +reasons. Verify the flag names against the pinned `FORGEJO_RELEASE` with +`forgejo admin auth add-oauth --help` before running — and list what exists +afterwards with `forgejo admin auth list`, which is also how you find the `id` +for `update-oauth` if the secret is ever rotated. + +No restart is needed: the source is a database row, not a file Forgejo reads +at boot. + +### What the app.ini side already does + +Set in `src/forgejo/ansible/proxmox/templates/app.ini.j2`, so a redeploy keeps +them — the comments there carry the detail: + +| Setting | Effect | +|---|---| +| `DISABLE_REGISTRATION = false` | It blocks OIDC auto-registration too, not just the local signup form. Left `true`, Authentik logins authenticate and are then refused an account. | +| `ALLOW_ONLY_EXTERNAL_REGISTRATION = true` | Restores "no self-service signup" without blocking Authentik. | +| `ENABLE_AUTO_REGISTRATION = true` | Creates the Forgejo account on first successful SSO login. | +| `ACCOUNT_LINKING = auto` | Attaches the OIDC identity to an existing local account with the same email. | + +`ACCOUNT_LINKING` is the one that matters on the first login: if the +break-glass admin from step 4b uses the same email address as your Authentik +account, signing in through Authentik lands you *in that admin account*. With +linking off you would arrive as a second, unprivileged user and have to +promote it from the CLI. + +### Keep the local admin + +Do not delete the step 4b account or convert it to SSO-only. The login path is +now Forgejo → Authentik → CloudNativePG → a healthy k3s cluster, and any of +those failing takes SSO down with it — including the cases you would most want +to log into the forge to investigate. This is the same reasoning +`roles/k3s_traefik` uses for keeping the port-forward-only dashboard route +alive alongside the Authentik-published one. + +### Git over HTTPS + +Users who arrive through Authentik have no Forgejo password, so HTTPS clones +need a personal access token. SSH keys are unaffected — and since step 7 put +git-over-SSH on port 22 with Forgejo managing the `git` user's +`authorized_keys`, SSH is the smoother default to point people at. + +## Rolling back + +There is no `state: absent` for `lxc_app` — removal would need a per-app +`uninstall.yml` and Forgejo has none. To back out: stop the service +(`systemctl stop forgejo`), point DNS back at whatever was serving before, and +leave the container in place. Destroying it needs the `prevent_destroy` block +removed by hand first, which is deliberate. diff --git a/docs/postgres-proxmox.md b/docs/postgres-proxmox.md new file mode 100644 index 0000000..6394bce --- /dev/null +++ b/docs/postgres-proxmox.md @@ -0,0 +1,237 @@ +# Bringing up the shared Postgres on Proxmox + +A run-once bootstrap, in the order the dependencies actually demand. Each step +exists because the one after it cannot start without it; if you already have a +piece, skip it and check the assertion at the end of the section. + +The end state: a Debian LXC on the `AppData` ZFS pool of `turtle-proxmox-01`, +running Postgres 17, replicated every five minutes to `turtle-proxmox-02`, and +answering on 192.168.50.54:5432 for every app that declares a `db:`. + +## Why this order + +Two chains have to be satisfied before `terraform apply` will run at all: + +- **State.** Terraform stores state in Postgres. Storing it in the database + this configuration provisions would be circular, so it goes on the + CloudNativePG cluster on the k3s Pis instead — a cluster Terraform has no + hand in building. That cluster therefore has to be up first. +- **Inventory.** `playbooks/proxmox.yml` targets `proxmox_guests`, a group + that only exists because of the API-backed dynamic inventory. Ansible cannot + install into a container Terraform has just made until it can see it. + +## 1. The k3s cluster and its CNPG instance + +Needed only for Terraform state. If the cluster is already up, confirm the +LoadBalancer answers and move on. + +```sh +cd build/config/ansible +ansible-playbook playbooks/k3s.yml --tags services +``` + +Then, against `k3s_postgres_loadbalancer_ip` (192.168.50.81), create the state +database and its role. This is the only hand-run SQL in the whole flow — +everything else provisions itself: + +```sql +CREATE ROLE terraform LOGIN PASSWORD ''; +CREATE DATABASE terraform_state OWNER terraform; +``` + +Store the resulting connection string at `homelab/ci/terraform` in Vault (see +`vault-secrets.md`), then check it: + +```sh +psql "postgres://terraform:$PG_PASSWORD@192.168.50.81:5432/terraform_state" -c '\conninfo' +``` + +## 2. Secrets + +`homelab/shared/postgres` is the same path the Unraid and k3s instances +already use — one superuser identity for the "shared postgres" concept +wherever it runs. If it is populated, nothing to do: + +```sh +vault kv get kv/homelab/shared/postgres +``` + +`homelab/ci/proxmox` needs a Proxmox API token, and `homelab/ci/ssh` the key +pair whose public half Terraform installs into the container. + +## 3. The container template + +On the node, once: + +```sh +pveam available | grep debian-12 +pveam download local debian-12-standard_12.7-1_amd64.tar.zst +``` + +## 4. Terraform + +```sh +cd build/config/terraform +terraform init -backend-config="conn_str=$PG_CONN_STR" +``` + +Pin the provider to whatever this resolves and commit the resulting +`.terraform.lock.hcl`. Then: + +```sh +export TF_VAR_proxmox_endpoint='https://turtle-proxmox-01.home.turtlesystems.co.uk:8006/' +export TF_VAR_proxmox_api_token='...' # homelab/ci/proxmox +export TF_VAR_lxc_template_file_id='local:vztmpl/debian-12-standard_12.7-1_amd64.tar.zst' +export TF_VAR_ssh_public_keys='["'"$(cat ~/.ssh/unraid_ansible.pub)"'"]' + +terraform apply -target=module.postgres +``` + +`-target` on purpose: the root module also defines the Forgejo LXC, and +bringing that up is a separate decision with its own migration story +(`inventory/host_vars/forgejo.yml`). Drop the flag once you want both. + +If the apply 403s on `changing feature flags (except nesting) is only allowed +for root@pam`, the container already exists without nesting enabled and the +`ansible@pam` token cannot add it. Set it as root on the node, then re-plan — +it will read clean, and Postgres's systemd unit needs it: + +```sh +ssh root@turtle-proxmox-01.home.turtlesystems.co.uk 'pct set 161 --features nesting=1' +ssh root@turtle-proxmox-01.home.turtlesystems.co.uk 'pct reboot 161' +``` + +Check the replication job exists — this is the step most easily missed, +because a container with no job looks identical in the storage view: + +```sh +ssh root@turtle-proxmox-01 'pvesr status' +``` + +## 5. Install Postgres into it + +```sh +cd build/config/ansible +ansible-inventory -i inventory/proxmox.yml --graph # the guest should appear +ansible-playbook playbooks/proxmox.yml -e only_stacks=postgres +``` + +Then from the controller, confirming both that it listens on the LAN and that +the Vault password took: + +```sh +psql "postgres://postgres:$PGPASSWORD@192.168.50.54:5432/postgres" -c 'SELECT version();' +``` + +## 6. Node backups + +Replication covers a dead node, not a dropped table. `roles/pve_backup` writes +vzdump archives to the NAS: + +```sh +ansible-playbook playbooks/pve_host.yml +``` + +## Growing the disk + +Everything lives on the rootfs — that is what makes a replication snapshot +atomic (`src/shared/postgres/terraform/README.md`), so there is no second +volume to add when space runs short. Raise `disk_size` on the `postgres` +module in `build/config/terraform/main.tf` and apply. + +Check the pool has the room first. Terraform will not: ZFS lets an apply +overcommit the pool happily, and the failure surfaces later as a write inside +the guest hitting ENOSPC. + +```sh +ssh root@turtle-proxmox-01 'zpool list AppData; zfs list -o name,used,avail,refquota -r AppData' +ssh root@turtle-proxmox-02 'zpool list AppData' # the target needs the room too +``` + +Then: + +```sh +cd build/config/terraform +terraform plan # expect an in-place update to disk.size, NOT a replacement +terraform apply +``` + +A **replacement** in that plan means something other than the size changed — +`prevent_destroy` will refuse it, which is the point. Never work around it +here; replacing this container destroys every database on it. + +The resize itself is a `pct resize` of the rootfs, which on ZFS-backed storage +is a refquota change rather than a partition operation. It applies to a +running guest, ships no data, and needs nothing done inside the container +afterwards — no `resize2fs`, no Postgres restart. Confirm from inside: + +```sh +ssh root@192.168.50.54 'df -h /' +``` + +Two things the larger number does *not* change: replication traffic, which is +a function of what actually gets written rather than of the quota, and vzdump +archive size, which covers used data only. Both track the databases, not the +headroom. + +Shrinking is not available — it is a replacement, and `prevent_destroy` blocks +it — so overshoot rather than raising this every few months. + +## Failing over + +Replication makes the far copy a volume, not a running guest, so failover is +deliberate. Planned, with both nodes up, is an ordinary migration — fast, +because replication means only the delta has to ship: + +```sh +ssh root@turtle-proxmox-01 'pct migrate 161 turtle-proxmox-02 --restart' +``` + +Unplanned, with the source node gone, means telling Proxmox to run the guest +from the replicated volume, and it loses every transaction committed since the +last successful send — up to `replication_schedule`, five minutes. Check what +you are about to accept first: + +```sh +ssh root@turtle-proxmox-02 'pvesr status' # look at "Last Sync" +``` + +Automating this is what a HA resource would add, and it is deliberately not +configured: automatic failover on a two-node cluster with no third vote is a +good way to get both nodes deciding they are the survivor. + +## Afterwards: repointing clients + +The address is written down in several files that cannot discover it. The +authoritative copy is `ip_address` on the `postgres` module in +`build/config/terraform/main.tf`, published as the `postgres_address` output. + +| Where | What | +|---|---| +| `inventory/host_vars/.yml` | `db.provision_host` on each app that declares one | +| `src//ansible/proxmox/vars.yml` | that app's own `DB_HOST` | +| `src//ansible/unraid/vars.yml` | ditto, for anything still on Unraid — no `unraid_shared` network alias reaches this host | + +Apps on the k3s cluster keep using the in-cluster CloudNativePG instance and +are unaffected. + +## Migrating an existing database onto it + +Per database, not `pg_dumpall` — the roles are recreated by the `db:` block on +each app's own entry, so only the data needs moving: + +```sh +pg_dump -h -U postgres -Fc forgejo > forgejo.dump +# create the role and database by deploying the app once with its `db:` block, +# then: +pg_restore -h 192.168.50.54 -U postgres -d forgejo --no-owner --role=forgejo forgejo.dump +``` + +`--no-owner --role=` rather than a straight restore: the dump carries +ownership from the old cluster, and the role provisioning in `lxc_app` / +`compose_stack` has already created the owner here with a Vault-managed +password. Restoring ownership from the dump would fight it. + +Stop the application before dumping. A dump of a live database is consistent +as of its start, so anything written during it is silently absent from the +restore. diff --git a/docs/vault-secrets.md b/docs/vault-secrets.md new file mode 100644 index 0000000..25ad4e0 --- /dev/null +++ b/docs/vault-secrets.md @@ -0,0 +1,551 @@ +# Vault secret layout + +KV v2 mount: `kv` (matches `vault_kv_mount` in +`build/config/ansible/inventory/group_vars/all.yml`). +One path per stack, fetched whole via `community.hashi_vault.vault_kv2_get`. + +Paths are prefixed `homelab/`, not `unraid/`: an app may run as a Compose +stack on Unraid or natively in a Proxmox LXC, and its secrets are the same +either way. If you populated the old `unraid/*` paths before this rename, +move them and delete the originals: + +```sh +for p in shared/postgres forgejo ci/ssh; do + vault kv get -format=json "kv/unraid/$p" \ + | jq -r '.data.data | to_entries | map("\(.key)=\(.value)") | .[]' \ + | xargs vault kv put "kv/homelab/$p" + vault kv metadata delete "kv/unraid/$p" +done +``` + +## Prerequisites + +- [Vault CLI](https://developer.hashicorp.com/vault/docs/install) installed + on whatever machine you're running these commands from (your workstation + during bootstrap — not the Unraid box, and not the same thing as the + Ansible controller needing `hvac`, though in practice it's the same WSL + shell for both). +- `VAULT_ADDR` set to your Vault instance: + + ```sh + export VAULT_ADDR='https://vault.example.internal:8200' + ``` + + If Vault is using a self-signed cert, either trust it properly or, for a + homelab-only instance, set `export VAULT_SKIP_VERIFY=true` — don't do this + against anything reachable outside your LAN. + +## Login + +Log in interactively once per shell session (or whenever the token expires); +the resulting token is cached to `~/.vault-token` and picked up automatically +by later `vault` commands and by `VAULT_TOKEN` if you export it. + +Token auth (simplest, fine for a single-operator homelab): + +```sh +vault login +# prompts for the token +``` + +If you're using a different auth backend, log in with that method instead — +adjust to whatever your Vault instance actually has enabled: + +```sh +vault login -method=userpass username= +vault login -method=oidc # opens a browser +vault login -method=ldap username= +``` + +Confirm the login worked and check the token's TTL: + +```sh +vault token lookup +``` + +## One-time setup: enable the KV v2 mount + +Only needed once, the first time this Vault instance is used for this repo +(skip if `kv/` already exists — check with `vault secrets list`): + +```sh +vault secrets enable -path=kv -version=2 kv +``` + +## `homelab/shared/postgres` + +| Key | Purpose | +|---|---| +| `POSTGRES_SUPERUSER` | Superuser name (matches `vars.yml` default `postgres`, but Vault is authoritative since it's paired with the password below) | +| `POSTGRES_SUPERUSER_PASSWORD` | Superuser password | + +```sh +vault kv put kv/homelab/shared/postgres \ + POSTGRES_SUPERUSER=postgres \ + POSTGRES_SUPERUSER_PASSWORD='' +``` + +Also seeds the k3s cluster's shared Postgres (`roles/k3s_postgres`, +CloudNativePG) — a separate physical instance from the Unraid/Proxmox ones, +but the same superuser identity, same path. See README.md → "Postgres +(CloudNativePG)". + +## `homelab/forgejo` + +| Key | Purpose | +|---|---| +| `DB_PASSWORD` | Password for the `forgejo` Postgres role (Ansible creates the role with this password) | +| `SECRET_KEY` | Forgejo `security.SECRET_KEY` | +| `INTERNAL_TOKEN` | Forgejo `security.INTERNAL_TOKEN` | +| `JWT_SECRET` | Forgejo `oauth2.JWT_SECRET` | +| `LFS_JWT_SECRET` | Forgejo `server.LFS_JWT_SECRET` — **Proxmox only**, see below | +| `OIDC_CLIENT_SECRET` | Authentik OAuth2 provider secret — **optional**, see below | + +Generate the Forgejo secrets once and store them, rather than letting Forgejo +auto-generate on first boot — that keeps a from-scratch redeploy (fresh data +volume) reproducible instead of silently rotating tokens: + +```sh +docker run --rm codeberg.org/forgejo/forgejo:10 forgejo generate secret +``` + +Run that four times (once per secret) and write them all together with the +DB password: + +```sh +vault kv put kv/homelab/forgejo \ + DB_PASSWORD='' \ + SECRET_KEY='' \ + INTERNAL_TOKEN='' \ + JWT_SECRET='' \ + LFS_JWT_SECRET='' +``` + +`LFS_JWT_SECRET` is only consumed by the Proxmox native install, which renders +`app.ini` in full and sets `INSTALL_LOCK` — nothing is left for Forgejo to +generate on first boot, and a value it invented for itself would be overwritten +on the next deploy anyway. The Unraid Compose stack doesn't reference it; the +key being present just means one extra unused line in the rendered `.env`. If +you only run the Unraid stack today, add it before the Proxmox cutover — +`src/forgejo/ansible/proxmox/install.yml` asserts on it rather than failing +three tasks later with an undefined-variable error: + +```sh +vault kv patch kv/homelab/forgejo LFS_JWT_SECRET='' +``` + +### `OIDC_CLIENT_SECRET` + +Only needed if Forgejo signs users in through Authentik (step 8 of +`docs/forgejo-proxmox.md`). Unlike every other key on this path it is **not +read by Ansible** — nothing templates it, and `install.yml` does not assert on +it, so its absence is not an error. Forgejo keeps authentication sources in +its database, not in `app.ini`, so the value is consumed once by a +`forgejo admin auth add-oauth` run and lives in the database from then on. + +It is kept here anyway, rather than only in Authentik, for the same reason as +the rest: a rebuild that restores from a `pg_dump` gets the auth source back +with the dump, but a rebuild from *nothing* has to re-run that command, and +this is where it looks for the value. Generated by Authentik when you create +the provider, not by you: + +```sh +vault kv patch kv/homelab/forgejo OIDC_CLIENT_SECRET='' +``` + +The matching client ID is not a secret and is not stored here — it is an +argument to that one command, recorded in the runbook step alongside it. + +## `homelab/arr` + +| Key | Purpose | +|---|---| +| `SECRET_KEY_BASE` | Signs sessions and cookies | +| `ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY` | Encrypts every credential Shelfarr stores — indexer API keys, download client passwords, OIDC secrets | +| `ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY` | As above, for values Shelfarr needs to query on | +| `ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT` | As above | +| `SHELFARR_SETTING_OIDC_CLIENT_SECRET` | Client secret of the Authentik provider for Shelfarr | +| `PROWLARR_API_KEY` | Prowlarr's API key, for anything integrating with it | + +One path for the whole stack, not one per container — `compose_stack` fetches +`vault_path` once and renders every key into a single `.env` that both +services read, so Prowlarr's key belongs here rather than under a path of its +own. + +Shelfarr's entrypoint generates the first four on first run if they're unset, into +`/rails/storage/.secret_key_base` and `/rails/storage/.encryption_keys` — i.e. +into the appdata volume. Set them explicitly for the same reason Forgejo's +three secrets are fixed above: a from-scratch redeploy on a fresh volume would +otherwise generate new ones, and everything encrypted with the old values +becomes unreadable. + +```sh +vault kv put kv/homelab/arr \ + SECRET_KEY_BASE="$(openssl rand -hex 64)" \ + ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY="$(openssl rand -base64 32)" \ + ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY="$(openssl rand -base64 32)" \ + ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT="$(openssl rand -base64 32)" +``` + +Set them **before the first start**, and set the three encryption keys as a +group — the entrypoint only falls back to its generated file when the primary +key is absent, so a partial set silently mixes provided and generated values. +Changing them after Shelfarr has stored anything means re-entering every +credential through Admin → Settings. + +> **Do not add `RAILS_MASTER_KEY` here.** Despite the name it is not a +> generatable secret: it decrypts the `config/credentials.yml.enc` compiled +> into the upstream image, so only upstream's own key works. Supplying a value +> of your own aborts startup with +> `ActiveSupport::MessageEncryptor::InvalidMessage` during `db:prepare`. + +The OIDC client secret is the one key here that isn't generated — copy it off +the Authentik provider: + +```sh +vault kv patch kv/homelab/arr \ + SHELFARR_SETTING_OIDC_CLIENT_SECRET='' +``` + +`SHELFARR_SETTING_` is how upstream maps environment variables onto the +Admin → Settings store; the non-secret half of the OIDC config lives in +`src/arr/common/vars.yml`. Both halves have to be named in +`docker-compose.yml`'s `environment:` — `compose_stack` renders every Vault +key and every `env_defaults` entry into `.env`, but Compose reads `.env` only +to interpolate `${...}` in the compose file. A var that nothing references +never reaches the container, and Shelfarr just carries on with OIDC off. + +Prowlarr's API key is generated the same way as Shelfarr's secrets, and for +the same reason — left unset, Prowlarr writes one of its own into +`/config/config.xml` on first start, where the only way to find it is the UI: + +```sh +vault kv patch kv/homelab/arr PROWLARR_API_KEY="$(openssl rand -hex 16)" +``` + +Prowlarr binds config-file settings to environment as `PROWLARR__` plus the +section and key, so this reaches it as `PROWLARR__AUTH__APIKEY`. Releases +before the auth section was split read the same value as `PROWLARR__APIKEY`; +an unrecognised variable is ignored rather than fatal, so the symptom of the +wrong form is a key in the UI that doesn't match Vault, not a container that +won't start. + +## `homelab/k3s-homelab-utils` + +| Key | Purpose | +|---|---| +| `K3S_TOKEN` | Cluster join token — shared by the server and every agent | + +Not app config — this is what makes `playbooks/k3s.yml` reproducible. +`k3s server` normally invents a token on first install and writes it to +`/var/lib/rancher/k3s/server/node-token`; every agent then has to be told that +value. Pinning it in Vault instead means a full rebuild (wipe both SD cards, +reinstall) reproduces the same cluster identity, because both the control +plane and every worker fetch the *same* value independently rather than one +generating it and handing it to the other. + +Generate it once, before the first run of `playbooks/k3s.yml`: + +```sh +vault kv put kv/homelab/k3s-homelab-utils \ + K3S_TOKEN="$(openssl rand -hex 32)" +``` + +Changing this after the cluster is up means re-running the playbook against +every node — the server re-issues its cert bundle around the new token and +every agent needs to reconnect with it, which is disruptive enough that +there's no automatic rotation path here, only a manual one. + +## `homelab/authentik` + +| Key | Purpose | +|---|---| +| `AUTHENTIK_SECRET_KEY` | Signs session cookies | +| `AUTHENTIK_POSTGRESQL__PASSWORD` | Password for the `authentik` role on the cluster's shared Postgres | + +The first app on the k3s cluster, deployed by `roles/k3s_app` from the +`k3s_apps:` list in `inventory/group_vars/k3s_cluster.yml`. Both keys are +named for the environment variables Authentik itself reads, because +`roles/k3s_app` passes Vault keys through to the app's Kubernetes Secret +verbatim — so one value at `AUTHENTIK_POSTGRESQL__PASSWORD` serves both the +database provisioning step and the running app, with nothing restated. + +```sh +vault kv put kv/homelab/authentik \ + AUTHENTIK_SECRET_KEY='' \ + AUTHENTIK_POSTGRESQL__PASSWORD="$(openssl rand -base64 32)" +``` + +`AUTHENTIK_SECRET_KEY` is the exception to "generate a fresh secret" — it +must be copied from the existing Unraid deployment, not invented. It signs +session cookies, so a new value logs every user out at the moment DNS moves. +See `docs/authentik-migration.md`, which is the only reason this path has a +"copy the old value" step at all; a from-scratch install would generate both. + +## `homelab/k3s-cert-manager` + +| Key | Purpose | +|---|---| +| `CLOUDFLARE_API_TOKEN` | DNS provider API token for cert-manager's ACME DNS-01 solver | + +Consumed by `roles/k3s_cert_manager`, which renders every key at this path +into a Secret in the `cert-manager` namespace — so the key name above is a +convention, not a requirement: store whatever your DNS provider needs and +reference that name from `k3s_cert_manager_solver` in +`inventory/group_vars/k3s_cluster.yml`. + +```sh +vault kv put kv/homelab/k3s-cert-manager \ + CLOUDFLARE_API_TOKEN='' +``` + +DNS-01 rather than HTTP-01 because HTTP-01 needs Let's Encrypt to reach the +cluster from the internet on port 80, and this LAN deliberately isn't +reachable. Scope the token to the single zone it manages — cert-manager only +ever needs to write `_acme-challenge` TXT records. + +## `homelab/ci/ssh` + +Not consumed by `compose_stack` — this is the SSH private key Ansible itself +connects to the Unraid hosts with. It lives in Vault rather than as a +Forgejo Actions secret so there's one place secrets come from, not two; see +README.md "SSH access" for generating the key pair and installing the public +half on each host. + +| Key | Purpose | +|---|---| +| `PRIVATE_KEY` | OpenSSH private key `ansible_user` (root) authenticates with | + +```sh +# `@path` reads the value from a file, which handles the embedded newlines +# in a private key correctly — don't try to inline it as a normal KEY=value. +# Use $HOME, not `~`: this is a plain command argument, not a shell +# assignment, so bash won't tilde-expand a `~` sitting after the `@` — it'd +# get passed through literally and vault would look for a file named `~`. +vault kv put kv/homelab/ci/ssh PRIVATE_KEY=@"$HOME/.ssh/unraid_ansible" +``` + +## `homelab/ci/ssh-k3s` + +The SSH private key Ansible connects to the 4 k3s Pis with. A separate key +pair from `homelab/ci/ssh` above, deliberately — the two host groups don't +share a trust boundary, so a leaked key for one shouldn't also be a working +key for the other. Same shape as the Unraid key otherwise: + +| Key | Purpose | +|---|---| +| `PRIVATE_KEY` | OpenSSH private key the `ansible` user (per `inventory/hosts.yml` → `k3s_cluster`) authenticates with | + +```sh +ssh-keygen -t ed25519 -f ~/.ssh/k3s_ansible -C "ansible@homelab-utils" -N "" +``` + +Create the `ansible` user on each of the 4 Pis with NOPASSWD sudo and install +the public half as its `authorized_keys` — see README.md "SSH access". Then: + +```sh +vault kv put kv/homelab/ci/ssh-k3s PRIVATE_KEY=@"$HOME/.ssh/k3s_ansible" +``` + +Unlike the Unraid key above, nothing needs loading into `ssh-agent` to use +this: `playbooks/k3s_ssh_key.yml` reads this path and writes the key to +`local/k3s/homelab-utils.key` on the controller, and both `playbooks/k3s.yml` +and `playbooks/k3s_maintenance.yml` import it as their first play. A manual +run needs only `VAULT_ADDR`/`VAULT_TOKEN` in the environment (the "Login" +section above — no AppRole involvement yet, both k3s playbooks being +manual-only for now). See README.md "SSH access" for the fallback flags if +Vault is unreachable. + +## `homelab/ci/proxmox` + +Proxmox VE API token, used by two things: the `community.proxmox` dynamic +inventory plugin (`build/config/ansible/inventory/proxmox.yml`), and the +Terraform provider that creates LXCs. A token rather than a password — same +reasoning as the SSH key, a password means an interactive prompt or a +plaintext secret on disk. + +Create the user and its token on either node (ACLs are cluster-wide), then +grant it a role. This is more involved than it looks, and the failure modes +give errors that don't obviously point at permissions: + +```sh +pveum user add ansible@pam +pveum user token add ansible@pam terraform --privsep 0 + +pveum role add TerraformProv -privs "Datastore.Allocate \ + Datastore.AllocateSpace Datastore.AllocateTemplate Datastore.Audit \ + Pool.Allocate SDN.Use Sys.Audit Sys.Console Sys.Modify \ + VM.Allocate VM.Audit VM.Clone VM.Console VM.Migrate VM.PowerMgmt \ + VM.Replicate \ + VM.Config.CPU VM.Config.Disk VM.Config.HWType VM.Config.Memory \ + VM.Config.Network VM.Config.Options" + +pveum acl modify / --users 'ansible@pam' --roles TerraformProv +``` + +Three things that otherwise cost an afternoon: + +- **`--privsep 0` is load-bearing.** A token created with privilege separation + on (the UI default) carries its *own* ACL and ignores what the user was + granted, so `pveum acl modify --users` looks like it did nothing. Either + turn it off as above, or grant the token as well with + `--tokens 'ansible@pam!terraform'`. With a purpose-made user rather than + `root@pam` there is nothing gained by maintaining both layers — least + privilege is already enforced on the user. +- **Privilege names are version-sensitive.** `VM.Monitor` appears in most + guides for this and is rejected outright by current PVE (it is a QEMU + monitor privilege, meaningless for LXC). `SDN.Use` is required from PVE 8.2 + on, where attaching a container to `vmbr0` is gated on + `/sdn/zones/localnetwork/vmbr0`. Check against your own node rather than + against any list, this one included: `pvesh get /access/roles/Administrator` + prints the complete valid set. +- **`VM.Replicate` is there for the shared Postgres**, whose module creates a + `pvesr` job (`replication_target_node` — see + `src/shared/postgres/terraform/`). A module without replication won't need + it. + +Verify the grant landed before reaching for Terraform — this answers the +token's effective permissions, not the user's: + +```sh +pveum user permissions ansible@pam --token terraform +``` + +Every `terraform apply` failure here is an HTTP 403 naming the exact missing +privilege, so the recovery loop is `pveum role modify TerraformProv --privs +"" --append` and re-run. Note `--append`: without it, `--privs` replaces +the whole list rather than adding to it. + +Then: + +| Key | Purpose | +|---|---| +| `PROXMOX_URL` | API endpoint, e.g. `https://turtle-proxmox-01.home.turtlesystems.co.uk:8006/` | +| `PROXMOX_USER` | Token owner, e.g. `ansible@pam` | +| `PROXMOX_TOKEN_ID` | Token ID (the part after `!`) | +| `PROXMOX_TOKEN_SECRET` | Token secret — shown once at creation | + +```sh +vault kv put kv/homelab/ci/proxmox \ + PROXMOX_URL='https://turtle-proxmox-01.home.turtlesystems.co.uk:8006/' \ + PROXMOX_USER='ansible@pam' \ + PROXMOX_TOKEN_ID='terraform' \ + PROXMOX_TOKEN_SECRET='' +``` + +Ansible's inventory plugin reads these from the environment, so export them +before running anything that touches `proxmox_guests`. Terraform wants the +token in one combined string instead: + +```sh +export TF_VAR_proxmox_api_token="${PROXMOX_USER}!${PROXMOX_TOKEN_ID}=${PROXMOX_TOKEN_SECRET}" +``` + +## `homelab/ci/terraform` + +Connection string for the Postgres database Terraform keeps its state in +(`backend "pg"` — see `build/config/terraform/README.md`). Separate from the +superuser credentials above: Terraform gets its own role, scoped to its own +database. + +| Key | Purpose | +|---|---| +| `PG_CONN_STR` | `postgres://terraform:@192.168.50.81:5432/terraform_state` | + +The host is the **CloudNativePG cluster on k3s** +(`k3s_postgres_loadbalancer_ip`), not either shared Postgres this repo +deploys — Terraform cannot keep its state in a database it provisions itself. +Create the role and database once, against that cluster, then store the +string: + +```sh +vault kv put kv/homelab/ci/terraform \ + PG_CONN_STR='postgres://terraform:@192.168.50.81:5432/terraform_state' +``` + +## Verifying what was written + +```sh +vault kv get kv/homelab/shared/postgres +vault kv get kv/homelab/forgejo +``` + +`vault kv put` (used above) replaces the whole secret at that path — to add +or change a single key without touching the others, use `vault kv patch` +instead: + +```sh +vault kv patch kv/homelab/forgejo DB_PASSWORD='' +``` + +## Adding a new stack's secrets + +```sh +vault kv put kv/homelab/ KEY1=value1 KEY2=value2 +``` + +Then reference the same path as `vault_path` in the stack's entry under +`build/config/ansible/inventory/host_vars/.yml`. + +## AppRole for CI (Forgejo Actions) + +The bootstrap commands above use your own human token. The Forgejo Actions +runner (`.forgejo/workflows/deploy.yml`) authenticates as an AppRole instead, +scoped to read-only on the `kv/homelab/*` paths. One-time setup: + +```sh +# 1. Policy: read-only on every stack's secrets +vault policy write unraid-deploy - <<'EOF' +path "kv/data/homelab/*" { + capabilities = ["read"] +} +EOF + +# 2. Enable AppRole auth (skip if already enabled — check `vault auth list`) +vault auth enable approle + +# 3. Create the role, bound to that policy +vault write auth/approle/role/unraid-deploy \ + token_policies="unraid-deploy" \ + token_ttl=15m \ + token_max_ttl=1h + +# 4. Fetch the role ID (stable, not secret on its own) +vault read auth/approle/role/unraid-deploy/role-id + +# 5. Generate a secret ID (treat as a secret — shown once) +vault write -f auth/approle/role/unraid-deploy/secret-id +``` + +Store the `role_id` and `secret_id` from steps 4–5 as `VAULT_ROLE_ID` and +`VAULT_SECRET_ID` in the Forgejo repo's Actions secrets (alongside +`VAULT_ADDR`) — that's what `deploy.yml` and +`build/config/ansible/inventory/group_vars/all.yml` +(`vault_auth_method: approle`) expect. + +These never live in a file in this repo, in either mode: + +- **CI:** Forgejo repo → Settings → Actions → Secrets. `deploy.yml` reads + them from there via `${{ secrets.VAULT_ROLE_ID }}` / + `${{ secrets.VAULT_SECRET_ID }}` and exports them as job env vars. +- **Manual runs** (bootstrap, or any ad hoc `ansible-playbook` invocation): + export them as shell env vars first — `group_vars/all.yml` picks them up + via `lookup('env', ...)`: + + ```sh + export VAULT_AUTH_METHOD=approle + export VAULT_ROLE_ID='' + export VAULT_SECRET_ID='' + ``` + + (Bootstrap can also just use your own human token instead — leave + `VAULT_AUTH_METHOD` unset, it defaults to `token`, and `vault login` from + the "Login" section above is enough.) + +Secret IDs can be regenerated (step 5) and old ones revoked without touching +the role itself if one ever leaks: + +```sh +vault write auth/approle/role/unraid-deploy/secret-id-accessor/destroy \ + secret_id_accessor='' +``` diff --git a/src/arr/ansible/proxmox/README.md b/src/arr/ansible/proxmox/README.md new file mode 100644 index 0000000..4bfc58c --- /dev/null +++ b/src/arr/ansible/proxmox/README.md @@ -0,0 +1,22 @@ +# ansible/proxmox/ — arr + +Not built. Shelfarr runs as a Compose stack on nas1 (`../unraid/`), and there +is no reason to move it — this folder exists because every app keeps the same +shape whether or not both platforms are in use. + +A native install would be a poor fit here in a way most apps aren't. Shelfarr +is a Rails app shipping two containers' worth of runtime (Ruby, a bundled +Solid Queue worker, and a .NET companion for the Libation sidecar), with no +released binary or package — upstream distributes container images only. Where +`forgejo` drops a single Go binary and a systemd unit into an LXC, +reproducing this natively means building the Rails app from source on the +guest. + +If it ever needs to move, the realistic option is Docker inside a privileged +LXC rather than the `lxc_app` role, which would be a new deployment primitive +for this repo — decide that deliberately rather than by extending the role. + +What wouldn't change: `../../common/vars.yml` already holds the portable +config (version, published port), and the Vault path stays `homelab/arr`. +Only paths and PUID/PGID are Unraid-specific, and those live in +`../unraid/vars.yml`. diff --git a/src/arr/ansible/unraid/.env.example b/src/arr/ansible/unraid/.env.example new file mode 100644 index 0000000..501d278 --- /dev/null +++ b/src/arr/ansible/unraid/.env.example @@ -0,0 +1,101 @@ +# Reference only — real values are rendered by Ansible from vars.yml (non- +# secret) and Vault path homelab/arr (secret). Do not fill this in and +# deploy it directly. + +# Non-secret (see ../../common/vars.yml) +SHELFARR_VERSION=2026.08.05.1 +SHELFARR_HTTP_PORT=5056 + +# Non-secret (see vars.yml) — check the media paths against your own shares +SHELFARR_DATA_PATH=/mnt/user/appdata/arr/data +SHELFARR_AUDIOBOOKS_PATH=/mnt/user/Media/AudioBooks +SHELFARR_EBOOKS_PATH=/mnt/user/Media/Books +SHELFARR_DOWNLOADS_PATH=/mnt/user/downloads +PUID=99 +PGID=100 +CHOWN_ON_START=auto + +# Docker network to join, and it must already exist on the target host — +# Compose declares it `external`, so it will not create it. Defined in +# Unraid's Docker settings, not by any stack in this repo. +SHELFARR_NETWORK=caddy-net + +# Non-secret, supplied by the compose_stack role rather than either vars.yml — +# the path it copies ansible/unraid/icon.png to on the host, for the +# net.unraid.docker.icon label. Empty when the stack ships no icon.png, which +# just leaves Unraid's question-mark placeholder. Override it in vars.yml to +# point at a hosted PNG instead. +STACK_ICON=/mnt/user/appdata/icons/arr.png + +# Secret — Vault key: SECRET_KEY_BASE +# Signs sessions and cookies. Generate with: +# openssl rand -hex 64 +SECRET_KEY_BASE= + +# Secret — Vault keys: ACTIVE_RECORD_ENCRYPTION_* +# Encrypt every credential Shelfarr stores (indexer API keys, download client +# passwords). Generate each with: +# openssl rand -base64 32 +# Set these before the first start, and set all three together. Changing them +# later makes everything already encrypted with the old values unreadable. +ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY= +ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY= +ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT= + +# Non-secret (see ../../common/vars.yml) — OIDC against Authentik. +# Upstream maps environment onto the settings store by SHELFARR_SETTING_ plus +# the uppercased setting key. There is no redirect-URI variable: the callback +# is fixed at /auth/oidc/callback and is registered on the +# Authentik provider. +SHELFARR_SETTING_OIDC_ENABLED=true +SHELFARR_SETTING_OIDC_PROVIDER_NAME=Authentik +SHELFARR_SETTING_OIDC_ISSUER=https://auth.turtlesystems.uk/application/o/shelfarr/ +SHELFARR_SETTING_OIDC_CLIENT_ID= +SHELFARR_SETTING_OIDC_SCOPES=openid profile email +SHELFARR_SETTING_OIDC_AUTO_CREATE_USERS=true +SHELFARR_SETTING_OIDC_DEFAULT_ROLE=user +SHELFARR_SETTING_OIDC_LINK_EXISTING_USERS=true +SHELFARR_SETTING_OIDC_AUTO_REDIRECT=false + +# Secret — Vault key: SHELFARR_SETTING_OIDC_CLIENT_SECRET +# The client secret of the Authentik provider for Shelfarr. +SHELFARR_SETTING_OIDC_CLIENT_SECRET= + +# Do NOT set RAILS_MASTER_KEY. It decrypts the credentials file built into the +# image, so only upstream's key works — a self-generated one aborts startup +# with ActiveSupport::MessageEncryptor::InvalidMessage. + +# Only if the Libation companion is enabled — see docker-compose.yml +# LIBATION_CONFIG_PATH=/mnt/user/appdata/arr/libation/config +# LIBATION_BOOKS_PATH=/mnt/user/appdata/arr/libation/books +# LIBATION_CONTROL_PATH=/mnt/user/appdata/arr/libation/control + +# Only if running behind a reverse proxy at a sub-path, e.g. /arr +# RAILS_RELATIVE_URL_ROOT=/ + +# --- Prowlarr --------------------------------------------------------------- + +# Non-secret (see ../../common/vars.yml). LinuxServer tags are the upstream +# version plus their build suffix (1.37.0.5076-ls117); pin to a full one read +# off the registry rather than leaving this at latest. +PROWLARR_VERSION=latest +PROWLARR_HTTP_PORT=9696 + +# Non-secret, supplied by the compose_stack role — the path it copies +# icon-prowlarr.png to, for the prowlarr container's net.unraid.docker.icon +# label. Per-service, so it's separate from STACK_ICON above, which is the +# stack's own icon.png and labels shelfarr. +STACK_ICON_PROWLARR=/mnt/user/appdata/icons/arr-prowlarr.png + +# Non-secret (see vars.yml) +PROWLARR_CONFIG_PATH=/mnt/user/appdata/arr/prowlarr +TZ=Europe/London + +# Secret — Vault key: PROWLARR_API_KEY +# Bound to Prowlarr's config.xml via PROWLARR__AUTH__APIKEY, so the key is +# known before first boot and anything integrating with Prowlarr can be +# configured from the same Vault path. Generate with: +# openssl rand -hex 16 +# Leave it out and Prowlarr generates its own on first start — then the value +# only exists in the UI and in /config/config.xml. +PROWLARR_API_KEY= diff --git a/src/arr/ansible/unraid/docker-compose.yml b/src/arr/ansible/unraid/docker-compose.yml new file mode 100644 index 0000000..eab6d4f --- /dev/null +++ b/src/arr/ansible/unraid/docker-compose.yml @@ -0,0 +1,252 @@ +services: + shelfarr: + image: ghcr.io/pedro-revez-silva/shelfarr:${SHELFARR_VERSION} + container_name: arr + restart: unless-stopped + # Unraid's Docker page normally reads icon, WebUI link and console shell + # from the dockerMan template that created the container. A Compose stack + # has no template, so 6.10+ falls back to these labels; without them the + # container renders as a question mark with no WebUI or Console entry. + # + # STACK_ICON comes from the compose_stack role (a path under + # /mnt/user/appdata/icons, or a URL if vars.yml overrides it). `[IP]` is + # substituted by the webgui with the host's address; the port has to be + # the published one, hence SHELFARR_HTTP_PORT rather than the container's + # 80. If you drop the `ports:` mapping below and reach Shelfarr only + # through Caddy, replace the whole value with the proxied URL — the + # webgui has no way to know about the reverse proxy. + labels: + net.unraid.docker.icon: ${STACK_ICON} + net.unraid.docker.webui: "http://[IP]:${SHELFARR_HTTP_PORT}/" + net.unraid.docker.shell: bash + networks: + - proxy + environment: + # Runs the Solid Queue background worker inside the Puma process rather + # than as a second container. Upstream's own compose does this; Shelfarr + # is a single-user-scale app and a separate worker buys nothing here. + SOLID_QUEUE_IN_PUMA: "1" + PUID: ${PUID} + PGID: ${PGID} + CHOWN_ON_START: ${CHOWN_ON_START} + # Deliberately NOT RAILS_MASTER_KEY. That variable decrypts the + # config/credentials.yml.enc baked into the image at build time, so only + # upstream's own key works — setting it to a generated value aborts boot + # with ActiveSupport::MessageEncryptor::InvalidMessage. + # + # These four are the ones the entrypoint actually treats as overridable, + # and pinning them is what stops a fresh appdata volume from orphaning + # stored credentials (same reasoning as forgejo's SECRET_KEY). Left + # unset, the entrypoint generates them into /rails/storage/.secret_key_base + # and /rails/storage/.encryption_keys — fine until that directory is lost. + SECRET_KEY_BASE: ${SECRET_KEY_BASE} + # Encrypt every credential Shelfarr stores (indexer API keys, download + # client passwords). All three must be set together; the entrypoint only + # falls back to its generated file when the primary key is absent. + ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY: ${ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY} + ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY: ${ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY} + ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT: ${ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT} + # --- OIDC (Authentik) ------------------------------------------------ + # + # Upstream maps environment onto the Admin → Settings store by + # `SHELFARR_SETTING_` + the uppercased setting key, so these ten are the + # whole of the SSO config; there is no redirect-URI variable, the + # callback path is fixed at /auth/oidc/callback and is registered on the + # Authentik side. + # + # Every one has to be named here. The .env `compose_stack` renders holds + # them already, but Compose reads .env only to interpolate `${...}` in + # this file — it does not inject it into the container. An unreferenced + # var reaches the host's Compose process and stops there, which is why + # OIDC stayed off with the values apparently set. + SHELFARR_SETTING_OIDC_ENABLED: ${SHELFARR_SETTING_OIDC_ENABLED} + SHELFARR_SETTING_OIDC_PROVIDER_NAME: ${SHELFARR_SETTING_OIDC_PROVIDER_NAME} + SHELFARR_SETTING_OIDC_ISSUER: ${SHELFARR_SETTING_OIDC_ISSUER} + SHELFARR_SETTING_OIDC_CLIENT_ID: ${SHELFARR_SETTING_OIDC_CLIENT_ID} + # Secret — Vault key of the same name, not either vars.yml. + SHELFARR_SETTING_OIDC_CLIENT_SECRET: ${SHELFARR_SETTING_OIDC_CLIENT_SECRET} + SHELFARR_SETTING_OIDC_SCOPES: ${SHELFARR_SETTING_OIDC_SCOPES} + SHELFARR_SETTING_OIDC_AUTO_CREATE_USERS: ${SHELFARR_SETTING_OIDC_AUTO_CREATE_USERS} + SHELFARR_SETTING_OIDC_DEFAULT_ROLE: ${SHELFARR_SETTING_OIDC_DEFAULT_ROLE} + SHELFARR_SETTING_OIDC_LINK_EXISTING_USERS: ${SHELFARR_SETTING_OIDC_LINK_EXISTING_USERS} + SHELFARR_SETTING_OIDC_AUTO_REDIRECT: ${SHELFARR_SETTING_OIDC_AUTO_REDIRECT} + volumes: + - ${SHELFARR_DATA_PATH}:/rails/storage + - ${SHELFARR_AUDIOBOOKS_PATH}:/audiobooks + - ${SHELFARR_EBOOKS_PATH}:/ebooks + - ${SHELFARR_DOWNLOADS_PATH}:/downloads + ports: + - "${SHELFARR_HTTP_PORT}:80" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:80/up"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + + # Indexer manager for the rest of the stack. Shelfarr doesn't talk to + # trackers itself — it asks Prowlarr, which owns the indexer definitions and + # their credentials. Both are on `proxy`, so Shelfarr reaches it as + # `http://prowlarr:9696` (Admin → Settings → Indexers) rather than via nas1's + # LAN IP and the published port. + prowlarr: + image: lscr.io/linuxserver/prowlarr:${PROWLARR_VERSION} + container_name: prowlarr + restart: unless-stopped + # Same reasoning as the labels on `shelfarr` above — a Compose stack has no + # dockerMan template, so without these the container is a question mark + # with no WebUI entry. STACK_ICON_PROWLARR rather than STACK_ICON: labels + # are per-service, and ${STACK_ICON} is the stack's own icon.png, which is + # Shelfarr's. This one comes from the committed icon-prowlarr.png in this + # directory — see README.md → "Icons on the Unraid Docker page". + labels: + net.unraid.docker.icon: ${STACK_ICON_PROWLARR} + net.unraid.docker.webui: "http://[IP]:${PROWLARR_HTTP_PORT}/" + # Alpine-based, so `sh` — same as forgejo, not `bash` like the + # Debian-based shelfarr above. + net.unraid.docker.shell: sh + networks: + - proxy + environment: + PUID: ${PUID} + PGID: ${PGID} + # LinuxServer's init sets the container clock from this. It matters more + # here than for Shelfarr: Prowlarr timestamps search history and applies + # per-indexer rate limits, and both read wrong at UTC. + TZ: ${TZ} + # Prowlarr binds its config file to environment via `PROWLARR__` plus the + # config section and key, double-underscore separated. Pinning the API + # key means Shelfarr's indexer config can be written from the same Vault + # path instead of being copied by hand out of the UI after first boot — + # the same argument as SECRET_KEY_BASE above, applied to a value that is + # otherwise generated into /config/config.xml on first start. + # + # Older releases read this as PROWLARR__APIKEY (no section). If the key + # in the UI doesn't match Vault after a deploy, that's which form this + # image wants; an unrecognised variable is ignored silently rather than + # failing the container, so the symptom is a generated key, not a crash. + PROWLARR__AUTH__APIKEY: ${PROWLARR_API_KEY} + volumes: + # config.xml, the indexer definitions, and Prowlarr's own SQLite + # database. This directory *is* the application state — same standing as + # SHELFARR_DATA_PATH, back it up the same way. + - ${PROWLARR_CONFIG_PATH}:/config + ports: + - "${PROWLARR_HTTP_PORT}:9696" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9696/ping"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + +networks: + # `external: true` means "attach to a network that already exists; don't + # create it, don't delete it on `down`". Without it Compose would create a + # *new* network named `arr_` — the deploy would still succeed and + # Caddy would silently fail to resolve `arr`. `name:` is what stops the + # project-name prefix being applied. + # + # The local alias is a fixed `proxy` so the compose file stays static; which + # real network it maps to is a per-host decision in vars.yml + # (SHELFARR_NETWORK). Same shape as forgejo's `shared` → `unraid_shared`. + # + # The alias MUST be a literal, not `${SHELFARR_NETWORK}:`. Compose + # interpolates values, never mapping keys, so a variable used as the key + # stays the literal string `${SHELFARR_NETWORK}` while the service's + # `networks:` entry (a list value) expands to `caddy-net` — and the two no + # longer match: + # service "arr" refers to undefined network caddy-net + # `name:` below is a value, so it interpolates normally. That asymmetry is + # the whole reason for the alias indirection. + # + # caddy-net is defined in Unraid's Docker settings on nas1 rather than by a + # stack in this repo, so nothing here has to deploy before Shelfarr does. + # But it must exist on the target host — an absent external network fails + # the deploy outright, which is why `unraid_shared` (created by the shared + # Postgres stack, and that runs on nas2) is deliberately not used here. + # Shelfarr is SQLite-only and needs no database anyway. + proxy: + name: ${SHELFARR_NETWORK} + external: true + +# Now that Caddy and Shelfarr share a network, Caddy should proxy to +# `arr:80` — the container name on a user-defined network — rather than +# to nas1's LAN IP and the published port. The `ports:` mapping above is kept +# only for direct access that bypasses the proxy; drop it if you don't want +# 5056 reachable on the LAN. +# +# Reaching the download client and indexers stays a runtime concern, not a +# compose one: they're configured in Admin → Settings. Anything not on +# caddy-net is addressed via nas1's LAN IP and its published port +# (e.g. http://192.168.50.1:8080), which works whichever network it's on. + +# --- Import layout --------------------------------------------------------- +# +# The three media binds above follow upstream's documented layout, which works +# regardless of how your shares are arranged. The cost: /downloads and +# /audiobooks are separate mounts inside the container, and rename() across +# mounts returns EXDEV even when both sit on the same Unraid filesystem — so +# every import is a full copy rather than an instant move. +# +# If downloads and the library live under one share, replace the three binds +# with a single one and set the paths in Admin → Settings to match: +# +# - ${SHELFARR_MEDIA_PATH}:/media # e.g. /mnt/user/media +# +# Admin → Settings: /media/downloads, /media/audiobooks, /media/ebooks +# +# One mount means one filesystem, so imports become atomic moves and hardlinks +# work. Worth doing if your download client is already writing under the same +# share. + +# --- Libation companion (Audible backup, beta) ----------------------------- +# +# Left out for now; purely additive. To enable it: uncomment the service and +# the `volumes:` block below, add these three to the main service's +# environment, and add the two mounts to its `volumes:`, then redeploy. The +# appdata bind survives the container recreate, so nothing is lost. +# +# environment: +# SHELFARR_LIBATION_URL: http://shelfarr-libation:8080 +# SHELFARR_LIBATION_TOKEN_FILE: /run/shelfarr-libation/token +# SHELFARR_LIBATION_IMPORT_ROOT: /imports/libation +# volumes: +# - ${LIBATION_BOOKS_PATH}:/imports/libation:ro +# - ${LIBATION_CONTROL_PATH}:/run/shelfarr-libation:ro +# +# Note the two services must then share a user-defined network so the main +# container can resolve `shelfarr-libation` by name — the default bridge has no +# DNS between containers. +# +# shelfarr-libation: +# image: ghcr.io/pedro-revez-silva/shelfarr-libation:${SHELFARR_VERSION} +# container_name: shelfarr-libation +# restart: unless-stopped +# expose: +# - "8080" +# environment: +# PUID: ${PUID} +# PGID: ${PGID} +# CHOWN_ON_START: ${CHOWN_ON_START} +# LIBATION_FILES_DIR: /config +# LIBATION_BOOKS_DIR: /data +# LIBATION_IN_PROGRESS_DIR: /config/in-progress +# COMPANION_STATE_DIR: /config/shelfarr-companion +# COMPANION_TOKEN_FILE: /control/token +# ASPNETCORE_URLS: http://0.0.0.0:8080 +# volumes: +# - ${LIBATION_CONFIG_PATH}:/config +# - ${LIBATION_BOOKS_PATH}:/data +# - ${LIBATION_CONTROL_PATH}:/control +# healthcheck: +# test: ["CMD", "/companion/Shelfarr.Libation.Companion", "--healthcheck"] +# interval: 30s +# timeout: 10s +# retries: 3 +# start_period: 40s +# +# Upstream uses named volumes for libation_config/books/control. Bind mounts +# under /mnt/user/appdata/arr/ instead, to match every other stack here +# and so the data is visible on the array rather than buried in Docker's +# storage. Add the paths to ansible/unraid/vars.yml when enabling. diff --git a/src/arr/ansible/unraid/icon-prowlarr.png b/src/arr/ansible/unraid/icon-prowlarr.png new file mode 100644 index 0000000000000000000000000000000000000000..c1b5f2d5875f8efbf57ac289e5bd9bebf85ccbb9 GIT binary patch literal 25627 zcmV*AKySZ^P)004R> z004l5008;`004mK004C`008P>0026e000+ooVrmw00006VoOIv0RI600RN!9r;`8x z010qNS#tmY1F!%91F!)n#1E|i000McNliru=L-)P5ITK<-pc?0V!=s7K~#9!?Y(!L z9mjR&`#sg&H_n|LF~}JNzyMMtMKQ>ta-`x)4x(hqvNl+@Wy^lH->zl9=Y6|Qvc0=r z%WGS~wro+Bs4P)|6f;GF1V98L=LrBN&y5}Gyg&NR00Rtw0dNNclK6c-!wb&z?XIdm zr|RS*T*Y~zAA`+#SdawY0wO3NU1{hqHlflYA`Plq53~aFfo_F1L7G9Efo32FB!Cp) zR6haAKvezofpK66WJu8=ffGQF!V!T%5g9{uR8AS%Sz?U=gqYSXlk+1loXlrmpLyuCbo> z?=fPA=m!o1M}Q*=hXmc{rZ_lqf^se=C%-7*7-b-YYmy*;JJ&0H<=`p<07pIo)&bWC zL9hZ?bq$hHv|B_r02_d{KsWI&-%PxlUkUJmA!5ep1-1dtspuwA=@pSu6yY0DKq(X; zk$^>Cn(JZDJzmxfaOh{_PtOBXu#RSo%0gfXumR;&zy^dIvAAO9ddxF~B^IX&zz*OE z;7OF-s2(w{6s=VpH@>*z@8){AbBC7|1N7cYEJT=i+SWo8s_S_ySLBr<+y>kPtmdSA z{!&31*a>U|o&|ltW75UeXM3A1!6z&b%8ItdEFS)i3RBToO1jl(D5R*9K9$BdZK5L(J#$&8@HaYuh?gnx@*IG0m$%~$qGa7C_U z--qIXX(SBaDKtKAuLJoFVkP$_9(amB6Tk@tcakhDRt8ARTN^6SLm$S-cF zp6Bmi8at?a+wYOS?dLI>rm24}jZ%8*U&(*-4~d3*uhf%Z0-F{76r=ZxHN^mn1(tk% zu7@=hSJVpj{{nGjdy^2AWvQT_0PY1E=Xz{2hU&=_7CkAx;~KNoJ8mCT0RR zfrw!;AU_IG8zPA;-t}+8Y3e@bQ|(9}!+-rg{9TV#SLcd&jb|M$act55>!OcGU^|z@ zOLryc26Mi02R;N1fk(-S5Mxsy?*o39*r9dhKu>Px;(FjFF{wJ-wk5b7D{xyE<2295 zWSfv=9VU@Mk{KkC24^C1Os9OD{D**zh$3+2ubI(s3P1QCl%M_*(ZE4$Igf@F2!kux zVzd+ZucG=DissiGrB(5_Z$ut)GrVo+lSw? zov>#+(QpsZXg{_v8ee#2oe(3y{}hM+LCEpU3Z;B-ZZp#6Y-Nt#8^4=C3<-3J#v;%A zf%M!h#{}YSRJ;^!#|jb)ufveh%{bnK zK2d#s6c7&g67=pM?Ad{TbSpvc4zyC_ba8DCQ3k#&!XJd89B^HWD$M)*+ze(;R;1^{ z@$2Yr$1cYZ;c1K zt-dND&V_P(*;k>rO8j9Gam&0~kpBkUeEFwdg(`^PG|k8BUPE%_og`PjDwgHFYJ`Q2 zt0~MjqEs_ZQ|HWveQXaBRbqml7BWfX8dlKKXv_#Z9>hPom1wwUa_)18jiA3>2M^_| zTTu7sFYf>@>+E|z6n~loqCRf~`D5Zll{tWdadF$1l3H~a$t2Jnsgr-d%!rar zIL%!%8~u41;B-tdLz$pw2bJBAQQ7qdov|(6TIOQMpE5if5k?N)!fR%9eRy;yu7t6? zf3YwIqKIhx1m*2NqP*oh1U);@(4Pw?I0X88A^fWYk#Knw<5{0&&xfmKX@k*-_kjF& zVDaovZ2}5naNC!Yx#4Z3*1zGrj;6VQMj>Hf@s?aaqw8&9l*-Yar0a7vczT@p9u4GZoR=6-OL%-A zr7ho~xbc4M_=(G{EFT2@qz7LQE`wq`Yi7ruUyQlS9Yb(=6Ud(f%V%#Yr$7}+q)4s4 zi~2YJ7U>(`5sN$VA1})gE*K)HwOAEQ>ioqk2KMdd@y~sh?ce?y*}5!s&CNKO?Bx6} za^V0lb*&^8Z6LAmTC|)e8tsq0%$J7=8bLN#*k4rkCImip1N`ac*?HKkFhJjjV|9Ms zk2)E)>14bv#ca>~#7polRzP2D7y-HOSz6OHs@ z3&WRdFwO>q0#PP!M(!)@upX>7x(Z^mh8!KrV?WE&92LmUUKnoo_F0N3T@jZO>SsIFnq zvyH-IU!=VK$7oQ#T&KY^0v`=Sc|Or=@mpZQS8Dr!n$AK$4Tm>7BpNI#EC&7;@a9=x zCRLo41!Q0Ov!pk?BaWlL66O5!L4^GAV`yVHUSm6j!2w2k4s+<4$2suiBMconfM1M5 zpyd<-R3Pob^=-n^25=8f|tUnYrxGlQFJpLDY|647t=6*A?b9AOhW_N z_D)PH6I;ekH4BTu7Dp&Odq0IIzeF^2^l~x5_XPf40cb{Nf z+BM>jfuCnq!y6RDA+h8p>VD>9B$wU{;&A1QJLg8X!1_Y;(h)li=4 zUsuM1>)H%U8)c&K^=Xj!w5T$JF@_U`kVg&|DF)ymWYTFGJ3DDxxrVmxE;^Slr)~8* zjO$Ig4;oR~_fzr@{Vo2%XD^4PC<6Xb;I|^niSA{J{C@3jUNhqC{Rreo1QAjwp8|e~ z*=!(DRdh4gy_MXXK1pK!OD9d((TH8&|2hYs--?Z*nD9>{{)y1Grj)Ru!?5_Q#D8+V zPz%-~B9yGqTM99*;J6UjfU%(=dUn3R!A+alziAT#JGar4f>cv$92O$64&k;eB)RAY z)GrdAI569uWJ6r8J~Zf)g;By!ZGb=BRHL=7#Z3-;DBgITBtjq`0Dhm@%-vPRm%e|^YPfi^J0?q&jYV6NBg2h9D89KW5c6U#GUH&NxHD4LAbF~=*Y^*Zx zy@N^+#jbhR6}*IyNIHlolnQx*<40-l-Gm<6hPVlwhL$)eP!*GHBDv%y#7o6qWWP9@ zOyC07A#ymCU|STRzq%4Wv*Xfle8~(j5!sni!3J^%ab)MrLK-gyRe8Vu^}kByrgtOB z%u9;+C+3)GZKb}Ym4n;2QOp-`s@uC~U|vpG*J@bPV#s(GL39G6Wo7S>5{3{4N)%3* z`{NVAbq&qA1WOxIL_>X)cWopZK918AE7nl;tj}ME)6h=XzXw|!olPc40c*oZo|lvy zvOfIgE8x#JU)oJCnE^hz0X%C^m8A%O0^BlND*)hjt)k&AzfJ0zyAj9Z#V+rk3yt%- zY3k}^|Mu-r$+Nh@VQs5nWs{*TBRF#LLSJtlj*g#PJ8NT8e;6t@ilZdDb4j`yvKTjk zh5`PO-2{hr;nX+dwRVFsh;i{cSKziSB{;SVn;)9ZC20Y=HI&DUBO~f4e}0Lg&?PZ| zej19Sac?b@-y=?FnYAq-vG6+T-~M|fm)%m0f_&)`KY$oR>%v8>UcZKxUkv$+yWZleAzzfEG{buT@U=RC-?cF?xwI+FGElujHYKc1%$z<3}O!uY2c38hdd zMe$GB#yyRF3xPru*V+PLq@wH?fZ!aK;;yW)yxt-O*C{kF55CY5MZ^49Q>EhdC#x8F5*|#a?vNjMKOSW8r%TJi^96=qw7)m z@62L!XJYBi)W7X_@Vc*g*%AK)oOG6kW!KQOay{w#27=Kc!oVjC0{qb8+nDf+5fnox zh9{XL9}0ya{u~S7x#JN>^A=x~P!&b1dGu967PzL>(3+0pi!_R(a$+Z1Y!2_?&V~-$d20wy?2nUfW(O%?$@=S~pW2G_OEdiG8!ql7xF`nrKpj{|FhZlM z`~mRB*_vr$$&E4b&s+1dCH|8T5>2f%EWd`1n_o?R=RCYb5+{|3ck0l`h9PP#Xk4w? z0#*e}dkUBLyYztEA*$@9~_l4i&q8Q-+eift=NEpe_1HaBJ z+55eP*U|8n-^S}+dzJVnpw&2?WQOd5WwhOR2kqD0OyiR`G%jo8068`KW#GngSSSLc9QCWrXZ_GwYqiSsJx##yu zEV}+G@t+SOu`)W516(N+=EsOidBSpuDDbgi#L(a{ySHrP*uet~pBNxtEaCehl~NJE zQX#TRbJ}G?XM&6uTbUXY(3Di%Qz%8;1SXk9!wR;PC%ODunm_QHxGh~%Lr_HU4{c`j z{!bJ3@189a=~jgIs_HJ}!uu0k(1?aU1{ELFl0oE85dQsz-s8Mb#c7^T!#n;n$yIk; zCH{+ooHPk>%BPKp{0dPJ5QPy|6~9TY?%G(!<8yB=Zmn}0+!cw{!1KoR~%(ccg#c>--0(5`95B2fP? zfa5{FD7c>7gYYS8-F2~gbk@KA6Qr)W`vMZaui{MT)acIY&TJe9$4lU)QY2C-(z$x( zt-gk}cizSNSKdnZ!fv!uq&#$D(!A<(34@Cd30Jx`4t2P$)-OyFDeCU} z2$@&B2XPX-)b0FN8oW%F+~QRO`(<8k!+jC^7nt7)|KlfMbJ=T#E+e5r+i`3M5^T?mHk_pp0MLwjqI&I zOZwV3S0i?7j=K?kM_Szr@{iAm0ebI^GrTG;%YctjYpaiH9Mb&iUqaILSLOb>hP8N& ztt@=qJ4vKt-=+}z7s{tsj;V@RVc>IY*9$~JW%_%nBI$avcYTz^;u~k<9VdWaE;+$6 zB~Hnm3C_y^3b;ey466~54+Cp!I_zq2Tiwrm45xMRi!+hsC5qhA)ikeKAB#G~v{0$O zo(Ub>DvVWNQ@>^7w!`H6j-I_#i(1^a#nip}mvEZr&ju5$L-~*yRpSoKOoliw104AT z1g#J?%8kH#sBwZzd^CO_PG-5f^Itebs`WtQ>@OTAu61HzR zUx$c6tB$o&2!#NKO3LAJ<=JDF2M=3b7=pop5ypCZ&aLFCl3ac(+1q|D4y~>&9N;~+ zL2d}!;P9`X{|oN9{@8mjFalypsLOkZyRX+2$u+Mgz2TmhL?+klBUU&k5rx>o7&<p^cCIk|N{-C)-_}uGi~;n4I@H7n8D-}lWC&v_InBYcM~4l zTT9cZ0PhmyK6TjS44!*aJtqSM6>yDu7J0wcvZjwJl1P)e`Mq)VhO5MX5+*Kh9*&Jt z9_Yv9hcHKXp-1-+9XpII7tkPxiG4C@T~7I!aDtuuok7WZLe_I>&B8LPjQc)EMoS#& zJ5Jxei2adZ90%`P-y@gH(!FRtOYis@#s`k$VJC0tTZ<7$8Z*T)cEM1KH-8BU#NWVRpz z)-rl!bTxtZ2z*YB?CkxJp0&>CSsB3hq2enI z_400DLG8w-m|Pp#SA7su*YffvelZ|QSS&KQ`vrD9^kWX~+|A&TBa|wBtQS=S)f|DO z18GmlB!#-9(2{jo*pMKXNZ`2!;}`@%C#qo2Ul10GVIFG1wK@dawhPUIr3=>R8d1X75m$Ln9Ri@*!cM_z{ehm9~uVs!oridCUH>h+?RO zR&i0fo15c0rS<|uT2;IS*OR{CZ4@5+LM>^+Q+S7u{1b5K=m(YgUp(`ovl4b9VF&{I zMrs-DsfyRRip&jfi~DT79A$qIY`MhIO&i(%;E&k%>~oA2N=zg#np3n(Gilw7OP<&W zqEM^8Cq?aqA1b1EAmDkYKzrJ-AZJ)uZ)i@+q*#4EveF6h$ry6RaD7XH#W|NfBNYye z`V`8Rz=!^#Qi))Fi_7u`hfM`wEtxyk)eaq+Ce3GAXB@ z6_a&?qUwV}(qidI8d{Quib5a?1`571P*l3Jh8x>m=I0#R8&a&i^KR-_UJSV?PPPdu z#kkm(jcUaN2_Qu6gCZ1v^ZGM`Q{UU8it+Ek-c`uWJpdb#I6{?B%rk8$)?` zkQaXNeV+ODw;AX^juFTtU5v!uND(GYi`8Z#lSwL8mB5~i!TS`2%}U~fKR)Wl)&Pag}pw$(Wt}1qiJVDuXE1R8-doDJYgmfFG=dUH{mqR zdwCH*FtTeKJHPc6`ks55E+Uq7xujho=?boZV`3%O5kZNaOOeLAe54Rsr5r&qP)2=a zxU7s+l;|9ECRIW&1j^Qdh(lwRl}(0KO%4so3)@{)m9zuvn+)wq*g9xA5?VrqEdvpe z!iJ6#(S!d=DBv!;_QE^YsKsfTPx{(75%%qj9eK4u3&QIRJf=u+I{Tl&04gA=ZNOV= zH3C#|+n1AC`#R3>X1-J*0#QV;^BKbL{SzIB_OYnLka9en(`aMrYREMIs$Qo|VT2>& zmZN#gn6HG@5}juw?NMKO?s&wJymD=e!{P=*!d%#!5TGk#$a%t+0eE59QnoRJ0ox2<1}<8*1`s zRlKg1WbgPzOnuu+-}EVh7RRYP`gJM~-G>%u5F2xmFailjXv-KD)f<{p1|cSLdzNLM zh~OM3DFY=Xp8kcBr<}F3M3viA(E+(15 zKd=!yK2WQ*^MI0d^qYX4mI01_5X1rzYY_QQKzl8oj0l;V-bH%d>$!Z}zf6V{O#aMf zEqjqzu@{4m4^#fhzu`arP4LTzaS&q=$H5o}<2Z;Dk4X{2Qwmo_HZEf_nZ?M|YH#O) zXN0!2&|T-yoHmq0C`K_O$mv6sxVUa$mBF$y?#K0kLKP}eoc$G_BvP#xboObtqz!eR zFi=*;{rE&^NE(_lRAJ}5Jynu zmDH+>V>Q7&x$3UV(}%Z8h#FKF@9m?K&l5!<#yBLiS#q6Sm{dBpC0^=DG!d+3qc|FK zdM7aG)qHRy$xCZgE)n*nkLFFqJRVAA+B;vKkW5TAdgps8ZMvVD(vbGPZB#e?U zf@{E)*e5XE=&i<$GFCJiy6c4fqn2%h5ksZ;YR`!o#a6{7fMTeeC`C8}knx1(w9u9j za-QHCA>%?WA!MAhmtSApU=VS6vNvR~to*b;BT-53^5Wk-v=kUg- zII?RueLJ=>J}^M&`#7FQuDO}cHS1WgdKHUqx`kvz%UmG((ZIA_S#nFFFh7|R^AYZAzrCrZMu3j+0Lf{p(dZHxKB(Hf5;-$_% z>5G0y97NW?{ds=+Z})L<>o)SEV<&^csu(yi%z+p7u(iI9MQb1Awtx42I@Vo3^Jf5B zE@1l(5bfGXbaXd5I)E0&s>kO^F>6BM94>IIEB4ts%116U918ZqZ$2kF~!msXa3+oMv%cDIZhsQ0?pNP1*)5Xh= zQ2EinU{YDU>uzTzqOiCrQmbE0W#=PtPqNzJCaY!v=&pke@k%%V|K(1IDhA_l3-BMP zN$ii}wk@UZH6O<$Yg;|#-w&VL@EO%GqTc-RNgYO57_VtoG zv5Us`Hj-`K=Qv_xF-P+wM0++7Jp6BjPko2z$WC;05CZ?CC^SL*0;N#ddA!0S2MQb* z@+tW`Y5keFCa}s-G2+m$&v@CA@(gu}>ErB01kW*~5)SogkGhnL)%ZE1zNHYvx`PpL z%+zKXh0s!rES@8TRv0WQYAs?6+2&TVEp52TBw?k34Z~PpAwCiELF}WbPe?TsI@VQA z7l$ly$wkLNN5+ux;*g2{f~7ubXiXcmTqNqC% ztl;$K&J#_AgpN8xw>w5~WDm)X1ti+$O?z+k%S1bWO7Pe>2%i2Pdg4f&Z~|v-Thuom z^7P>%oBPV-eLHz+Hep!YgBclOUCXb?Nh;)qTdE-Q6Q)^Owdet{d_^K-0y(>*M`?JnAu zts*ED$)7k@eS7f|7DJ_>8h#BBM6tszbBg)Ysd}IHj#_ews)^JPL}{C)kuec{)WJ>lml|qG0k)7>crBS%ya$s$Kc`;|RF!A` z^&i>(_>)W=kE%-J{6*aP??1__f91DXe#hM`*zgJ#-gFzQ?tT+Z3l`%Z-$%C$$1<-7 z!RP=^rjE?gwTSDYr2^6Bhwy*=C89$w#A$KoIl+$SL!LZTWJ_O}f^R4Jq&@3$>!J)d z&d<=2b?{7VS*cGsENJv-&$<)?OQB*XYxj>z`A7hHJ@9U7;r&%euYV({b+4P9@`hp!6@B0d)gF}c2YAw0; zPG0*P|AqB$doPJhHnzEoiBtVjX*yP|rE}2&{C%5=^23u=&7$!U>aTkhGIE^Yv9A+8 z{XMiW!en~kSrA0Z?g5|24;DBw62vxliA{|wTa(g&Zbkaedz&^#wT8MX~t6zI&xJq8BKafIr;y4drqHpCu3iqpLe z)7*K^@t9&sz@eB=B9V!&0jk^ zfp4zEfhQhi+k+1gh7lrhKI^sbcptC$*^f-EaBvz@U5l}WF^W4kR>vj~2BgKJ&;F2T z-{$kS{S<-mishLjMK&HSF;=#dL6EtW;pPPyZd#C`F@63DGD%P9%DJ>mCRpj}Hu>A)n?|;_mDfEm6{AkZO zoBPU?!;@l0SG~*K%j>v)UOL|CW+3^g#+IAK%?Vz=D#vwQ)$dk^Q4B0wdP{t7=Qw)? zDw8_HSr@?oYnH4s)I~*C0mEsA6`<};t`hmDR-`UpOTD-2+I1L@OlB6A!^^NT$ zJLZG&0Y7eUH1AUHJh1|9^Fo~F1+y3S97R;h6{1sKP%>Lb^P(j)`g)>c0Zy(tu796R z(~wDF&Y|R1r-dIW+xpA=a8I7YBf%tnxJFprp5%2aax892Ook#~2vngyX}E21hC7#L zX~{T~&sE@9KIEZ&1%7(C#OP#w!c(3if+Vv@t|cxn>9~jp*;InoT!xHe;>rs%Kv*sj zg~7>U69N^hyf7RwTsiqn|19y?X;)a^<}g^c>==v)ZR{vpT<@@JIR5|GJs#jc^FzY@ z+s?5t7N>O~Zu7!gM$&W#Axr!a-0DkTN{wRw#r5#!uS49~YZ4FDS3hO_<&1{XXCI`& za~v1}a( zxTVu0mk@jlrION;nyL#3Fu%?)zs_ODP{h1!9IUsd+3@tSh@QMdF-1w z^Oqy(9Mf$b;-+xBuYuikFZYnu-SFtuewY(&fsYj9-)NvWg2RwajkbQ$iDwP1^7!*Te z%Hb6Y(yVGr;+eCHJj11ky#qcwPgEE$M>t04%6Y7APcXkBK_+qLcf0F7Ub{TUvpq$2 z9IsFb6g0xCtH)h)J^t(8jCNHny@F*eg!svNxyUU?7p*df9vzDD@`Pw)#PcyIa` zX5RE=p2A9n-4A}B(f-~^`Cox@RYC7BTK11dbY%?n37HJ7UYIkJ*=F&^04tjeJ$cLK zfrz$@p)Mh;YI5kwM|5W6a1IqA+_!~b^CKi)`R22{F(uKx7V#2jr6vr}0xYr$ZgDh6 zwtjp8HL3Wlm{hK6`=7<>SOJAV8m z(WpWqiRoH_S-2Y08Cxf$t_kZVN#!eCH#E$mqlf5!VJnB8eu7Nl7`HA+v!HQ$k!M0= z+B@j8wXa0~czE)fNZCJB;mD}Z;-&;^J5tPVaB+=HUZX%=(s0|NEKO;b&3zRz%WvfR zcfOzHuYMib<{2E=B9KU97OW=m?oVNs-9Yfb-&6V0ACh?cui&h>X^PmVQOMAP_hC1F zo947KQa)LdWysfwp|gK9VqLRKS0-+!x_vOB94afDoGEFY-F1ffIpN@F%rtRqk}z5k zhRRA;wrW-K%T%8J0q(N(n2tr$h|YrBxg3+q5mgG*Ko&{>3)06toZq>%(ggAWYLfge zB<8OpbK^a;E&@3s9N4^x{O~9k!Dxt#9cTW^RiwKXPWv5ubSM6O{|&ou6BAA^$+h5K z`)U$z{3!07@58(C4LHkgz;rG_n!7NKZDiZKXkN60dFyXr@h!Kr!H*wT}6NS-(cTh^g2 zQSB}Xj8?5vu`gXn8bf#0!<+SlUBi}fUunynQYcGef5F}nOEJ{R>(gG$0J%iq*}8 z3B23igPFey$u`E7l+OXjOHkL|MRwh-NMk#;XAe}9UM6h#`v-jus@lhr5YbN z*@T%4ftZM} zsAxn)e(3zh^=2t>FR@&7$;fPY2Z@E(GTX^7B0^(FJDKCVS?CPYnu!-Z8a+;7>l64r z`v{I7rqZ(yf5(&92fv8!-vo&i&bm8Ey!GGXz4`-4OZRz+ep)y#X8uY{V>{7-=b>Dn z99o|1Df83AC5FqNFI3ZtfMQBMIf(WdYNqr~pEAw-P zlpCJ{Nk@2oAfgaLd-Z)dCT9MFJ!E`PGF$`>dn#cEz zvF%ueN>yQuJ*L8{w%9AYvNcId#-Y8=WnM!fE?-iNt13(|i*J?w@sPoMK=ZQIw7mb5 zF_YF3j*H1Q676~hDUFd#cqob=p3K>ZtWXJMGNvi#39Fl3P?runf~_*An%+7r6#Jrb@k zP>k@arqx(CP{u=t&bRzL-FLnDqK9_QVP_lAfg{*M+YvD&T$jiyZX-&s`e;F_55 zPxmCNAIJWCD>qEtpv0i1F)BMBM$6;Wz$2Vg!*>jE;oD1>R}@UH6-j29HS5oG4;4eh zjX%S@fAIgu?P)_b&W@}Lw{|&k2AIUav)PB#swR0X5RW6 zC>-l$WqXpF7i4J8IOjbG9T8TwrSKfXqx%btmLrVovS7o_v@AKzRekXvB&}!@RoK z5t#FYn>t+bLBReIOQa!7>kU~?7%nTldCS7w^vOSn!44fG*#0E$!j<49CY_QdnZeXG zolNbxRI-6?7noUq$oZgPvdu^`!<-bpQj1&PLj8ujNGB8c$96%bgnP@|;+_9uego`5AOh2hCSHT&egTp;dg^XurECm{Y%FntU}Z8M(W-L&V@y?*f6Ab&u(&KKJD%9SP#nW7yaxC7_r$%PE)@QDwDb^Yp8n%mwmvgXxg62eHOiVbyUCP)h%|m0 zvhM+j#j52uy@JJ0HepJm7xj)7Hzm*|S++Bejy2VoeGSmk1K{r8f$m}NV6+wOD)J%h-0^@X7OBhh@kmlf+xO-_U@yi23Folx^wA; z-z)4vi{D2wuZ6^Q=-77t?Hj+$uC3c>YBt<>W5|a;w3m5{{%^$Tg7jSgq*^;DH?(3) zqZc~`mNX}5U4IklTx|B#@*@x&KpOrduB7QLtmCi#>UR;b%$sMq=N_Loz43mMCXKZG z1(=ydw3J4M7&&?vr{EJ=b4F5qV>Usd5;0ne-{p9qqQa z5AFgH1_aOkh;ZlAD9W)S^l0v)H?2z4h9GUcx<_PFYXEB(rg0vD%=;5Z6t-G^iqOG8TX4sPJvL z`&q(gegIK~AHnWX!8#X_+@?jO{9^`t9%ti2Ut{kJyZF|ZzR2NydvP3tiIc5(`e~O< zn_R?y45F%g-f^Mn61}x<0@Jy)+L3M>w)`ynp8pz~pL&Ar&ppd`@B0#gU%{BT)$iz- z@QrUKDU1!F(C`rAt&HPs8ZV)uq$<+bhD;=`L`SjVFpvJ=alZMz|A_0l^c_BY#@7xW zbQm2o4PzOrlEEv!CL=?P%AsI3CE4F5Q875=obuM1g{cgE;Qx zJpe?g3?8TS!+*u=-8b1#Ztp;aK$Cc>^u^v7F~aEJAcv0(;kxFGT!X1qQL0C>agf|4 zNR(I@**D5B@!Aq2%ZXTJQC4KrpY8jKu0#G#E(F=0=Xb5x&jomdnq1tlUI zJF-S9O4^NW{*yx|&J%7*o&8=*8VsYp%f@42wdYt|+Px_$)02%c1VhL8>0ka4+r|x% zh2;%-ZfXin#!6ip5{4QvKwKQE_D-gAAuN{|c<_GMznM5j5sJR$SU!v^!#NkRQ%C}w zmCT=S>FA=GSU+{Kt%a5rWy#VAB!{>s;|L|&c<~cbMRF}Lp-VaQ@sdp}TOLtYXU-a6 zxO8bmeS-ye0hn_?zc@U^#(()dg`pF1F15%MLY@xUq(^<~G)qn$S4QmY;S6GG|bI08h7!jJ<$ z{x-SokK#F#zhEMV}$0q1o=ux5XFQ) z<|}ock~(kmu~TOUDM#4Q;V|YahsR@c=(5J?ll}8SY<^8TLS4dP<+25=>huYZ*Nnyz zmv_sq5!6}^JoyN^@%z-7kW=@Z(Q-sNw0ORcUo2kK{Uv=jWZ%i%ulW~(0KV~!Bz=8` zD1wFt<>p&Le&OefYL zsqHyF`q6`=)3NeBJ}x*8baz|c`K}7L-yR`#?|}5wk|&9}Ix6F1JbT|4IkM?#{Cu9! zub|aX5s1s1&%~&bPPjCrJqGe|>muLAVGzl*T#(Ic!jpVUm&pjm1}s)=tW(xIl#(ghIzHHfzb7(?Epd2M@62zAvyo zI7TYxO|gbe)V)`QCQ#dO%{aqNC28tnGQu*9)YR zF7+vQ@)QVdTv9!izVHq+0xWAZ6oR;3(1iEEYV0X4hOvz2rqZ;{pGVq@%U-HZWF6OJ zqxD^&ChhoCBh-eocK{f}=&^nt{l|ZxYxn@|o#|;KEN!)*jxi30o_T`%e&s_nEm(wc zJ<8*wjQ989mkM-Ub1k=j?AOi=I6DO){(qlj(s22lle<+h46R0DuF@S0IUN zL4T)e>vuphh^b>Hi|%*{*1!F|Pt60>8q7j(VE2UiH15DF<2AYDo0S`!{71 zjFtj|s_C>4Dy2xtx^lt$vMX>+i<#0DXQ&b@*Z{_LXzT8vA(yTeJ2p&K?5GWD#{fii zomEg=L6nAZw?RXI;2zxFAxLlw5Zrx`;5OKx13`mpLJ02e8r&t=Ai*^dAi&Jd?pAHp zR_#9BmwT$Kx=)|mx6c26P5)ZoS*Y$iQL&&n!lMKu?(gBMGV>?@^HqZxol5CHzYFhX z=;^o%kIEOS0ya$pmUu!yzlSI6uRKw|zNwHoG^iw}QXXSJ7(Yngl)lXLUpk8;4e7hQvmT^-FpL$Ybuk-o6p_-!} zNN~pv>6h0KCczh+zlc6Y@ib9m;w5JrB$N0+TGbr%#*L;LTwpy}Cv!(`@6`;?=i1CK5l3NwYd-h+Uuk~$y^#G~mw;}J-;?Sxm?BA>k+f;a2 z&)_1upvF&M{py}S(vH0U#(NqT<9Iq4r4@!L;rVKS2#EI>f8Z}&R(i4c@TQo4kIw|h zx;%e1Z!Kk(eJHq?EvkNA0-EM>IDtp|J@QatB01oLfN6%-!M zj(j`Xu8-F6rlR>#OMio!e7>GsSjdu~nqg)t;bIOKZm2$&gx~Q9_&dt1jo4>yKd$FM zfLA-xe?SGb|pSg7@BbOUu_slb__f+M)@CU)+xlM zQC{p>#n!xEM!pcXIbvM7X*@?H(WW?!%}LkN7@bjecCUNpY!OoQGEIA!d2C^u-Fzy= zi9>hUGw&0MY&iXe%vJqaCAgo->nY8vDh=18cz2 zsC558u6bPAyE^!F?1!h{L2ZA}FRPD@-AL;XT;KyR);GVWAz_~n{5GWS^ob)Eg?6ig z(Wh_l6~9x@xUI_>bJw@A1MWp@2Eav?q#RJD5EHVKu11l02`YJ9)}Ly{uVwSzF&~*U zgm+B(loSw?LI4>t&O=BIYX&~=oXZS)y};G>F9fdp$tdJC$_#MCY=7Gu`NS4v0K9S9RpSh22p&x*W;oQWbkLxMSO)DOIQk zl$jNF(HiwutcrI0xqVga-S2GX4%~p?BNrUb*89`n;rlL-8yZrT=(y<{%A(VFp<_4$ zMF3zNg$j$d+b&B01|iMvXyKDYGOgs;^FK5fleBsiTfF(zWG9QL#@j&S00B(G+u6V= zUqP<;jCF(BAkJ7}SbFvLKOEzPw0FDZJi^>Z1D0m@c#j`_18bZ1m_7AavAc_1q-FlZ z!4pOvh1ll=`0L#yoqUU^;8+c9SXPmK{OE=-t>5|k>{AQ% zrel?;+_1#+O$CLYc62;xb7dK9nn9DIhaz#mQSIuiSzie8xu%+0gc%MUzx^kK`YCm7#u5$;rVQ<`)X*Q2^nSD9Re zBpycaxwgYCJ_MiU+x`-l&4{^(-uRblK`$fNHyaV+`tozW^}LN^nh(^ zX>-4QJFMz<#A*R)&5R}=SA2#hjj0~Xb`VEs7YRMNZHQr3)YHing7xVct%Wyut`&?@ zn6&+fF%tssRI~zvd3CGmMzt24U)Q0QdRm{nB<*P{zY=XLjDZ|)1x3Bt@_`>?C4g!;=Ens~PzH*OipSn~?bv~9L z2k#CJ|JNgI?hIrk{FgTWVAc@#I;C2L4|LyJIa+!zPUn2AA|Me(6%taJuqBPB)5z9XtEb8X7U!_>NnoRAnf`g`c;>`~gzIPD}{Gja3lvS>?DyMXNvgJQJK$eS08 zX-^rhsU&dR-czH%#-l-o54ZZ^GeKV5{G7m}`MynLP61LZp3iJ~KExH_Y&Y@;Li6?c$cQ)ei&==&_r+m82fcFn z(SPo3?yz)UwZq^^XrX5N@;|bi4yxj(=?C#$*LK`EbYHM1P9)H8jFRDIbdMMOnv2HM zf9`?KeMu}ZTa@K!UV=A@3{-H=WV9h%I)Oe+5@dy}X@w0Eka!b_a;%ruP|6UP`oSt~ zj|3}{p&a4nF9;1+JhYzAPvE(*FGu;Z%j=GxxrRYD(ju7{c1GvB_V&xzaJs!Av0aDn zghp%h)1HI-EXw$Abu1=Qxru-#UVpiN>iU%5?T9u~2Ixu60O+Dc9FdFR$bxuSfi0tA z^#WBx!O+ilx1ALQz()0LH?nu-3;yFwa(pvmani1O6kimr?c?iBp8G@9F#S?w+VZDi z3S}IFQSX!5?kLpXsf8XY<46?Jr%lbzE(*FegK@v?-SR-+o(8jbqu=nnlrF(a_CYwd(An9?edK0||ecjt$lr%$#nzv@Sh z$io02#45k;YUq2h4lq%7Mp`Y_ylA7f(D6Z(TTMayJ@9mpn)agfI#cAcc^ytW5oRvO z<1{l7yj;FA0@(O~#vlX1=RRkW!4FCGMCx;s!bnvCHbtDH)IxPT zhjCX^LqR?H2pS1p;I#mfEQ_g%lMgiw)j$i6RhMy*HV0HJMseHR)gn*ZT0P(yx))U% z>8RcB7Qkqspw9yYPrOY>cti?UU+2YuE$zHOf-9 zfytxvFkKUDqPltd{)uShI^SDIWnGQ2(A-D`r9A$a)5rLcspg0slMMBErYDNU>8w%b zho)Ajr)jXL=CN>>&pYMNWXXr-J^#iWzYL7I1sRleWFah@5Yuf?np?OnjtiF+Dbz?U zy&xcbw^4>6YbwRXu_Df6*SbDzTPO4;&B3^?C{5y`E7et4N95qk|&v-NHU7WgIILC9TX`C_yD(KtGG(;Ef`jLR?&$nYbr*zR4&IXS0{sWO`T5M2cVe`MPR^JH9-CDJm- zL}`X*OC+>v(yMzJ1?yuj;M8mu*3heYwm4KO_KcB=Ca(SoLQU99$)`T&IC&eq0t(Kd zgSsf1V@5I0SC_MflB@%&i~F8EZ2<|ql7c@=A(AR#+kNTVm~vwRGp>wjs`Z~XZ`S!L zl%vWcoZWq=t}5uFD1z&ESU)Q!_>IIL~kKez|A9rmSlZ&B<43BVMC70fRk~qvUyd*Dr4g` ze^#nTwCKI&JT;}Aaarv7uO@-$MSQq`c)m0APb-dVSeOKRw5!Gw*k)GRc{if=ufJ`S z%acBWJ|v#QrEg;KjB zm1!iR9x#z_dg=tA81c38j?_5zUbnS_{wgK@I$VW381iq-# z)6#Q5hz*o0tSEKT6TRmQ=G_8AJ$t zg2o@v%r!l!2fG5?z*xi9UmOb8q{};Q(Gmgc>p8{^KB3Kl#pYeA`do6wy(cf#YwZ3= zx4=<(9~??en0G(Lp0+l{CzP2LSM(*_0I1bKghI2cn>Se&V3}jtq-3mMDZSXFo4di- zg{PmEPEoF)&NJ^dm1cnmpi=|Q4eoQ3<={FF*hQ6L`kVbxToEhzynp+C@yrO)6Q6~Y zUgr-Qqx}}s8HVjIt27PnMVOV&3k&z#X;D>Nguk(swH=$TUPqW{o^Y6h>g2G)&ai~G zULeN)+W0=Oy+xTkAPSacigd)~;@LGrREPw-2OL<<`iYE(5AAX5n-AHqSY;FIvakSdNB@DIU8%oS}zL@Fh zOuclQZdr$H%Se>a9+8VdZ3$JYR|iigJGnPsP{kvWOT<$el&liyAgCVIOYcBiH0iA` z;b-Uf%O{yed=p@Xgv+NiHMgLrwR;70oaj~HK?ir-P_*r;$@BG_61Dexp-6AY<_?EB z<^;LkOk0N6!0|_Aa1KC08|un3NapHX!6*#Z>~+VcOh#Qet=?{1s?K*mRNuCL z&rIu%f2}~~<@;f|+CuKTeoxUu!NtX2N^|z81A0*XIwj2ytOyT9)lDAq4jGXbB?sn{ zDCAKx^lrla$iCaq(gOv!@yJEMPws_b_*<{WY-5O~w`r~Q1yP2|l?q|R zvZTCaVPCIe&BIo}F3-$3Qn_SX4N3?414r|%bsT~ur+$6>RF{#H+0_|Ofh>5#PI#G% zKWr57gx|4Lq(4m(`biTb`#w4wdr=nvyHqSkp6Sn z`yCYiB(GjBNw8Tcb3c5Tq-EYfPAn}Nu!*%nxVY$FTdegsCV)1G*p+`3weKmsEodnB;n*RoWi-24#Wk~mVQ9f0otRx)b1+b z07?ev${A-YIfW=fkXjPvW>kEwBYiYF4+j(Z^*dn5iuAahXwvFtA6qjy6?na}Z~9qh zizKOVy~(@@SBv;qQZ)l zU!uRg)g{%Iz7d4gQ`C%Lcc&K#Z*3$?l4OZiNogX%iA>IciQPrShXOLlm8wHdT66Du zBfVsr?d{@loiqN*$n&;^36r02a7#p49wC*ZLP|odgcWJgd$?0WXA}UI{f7>)1Kgjw z9xzkzXIT*FwKVEk#Y^C@8W4D?fL1DHwuEUuNi;B5N_sC1=1Q8(jIim%Dy+xG(ZN}_ zXM57x0Zky(#^g+BW}E2G%)GefZ|OPmikc2pqHMa2xDhzrQ!Gla?P z$N;}{Wlkfv7UxdJuj~p>9y|V;j;POq=6btu(RnznE09eBV*bu zYKyJQ%?VGt@nNAyj79gGMO4;d0172~XpD*e=WKEi2J^t&8(HA#I>0*)|DtwxfF2ZM z_fw^y*k=dWBq{tWv%_k~kC^NwxwpBg3}9#oz1)zkLt(^@5I0lvgK*c^VWSuHMtlPh6g8bd5}@Kc-NVO#0s~BV ze4mI5fENidudiRs;-HTfCm)Ia{01UM~$r5C1Z zejhz~2JfLhijyxLSS0;~BP~z;^D;opi0bsVYy*O9I)Ny^+~lCUKL~l1_VJpcenkM< zb9t{x-JxvdKl zmfDx8dnuyXrmWd|&S zAt2BzpGmxiey-)%A*X0dFMakr(u;j$X$p z+tQOZbX&M&0NckMi`KHWsE{A9olITj1p^rhnZ{3uO%xCg7qZANUMVZWvL_sA#J&Gw zu>JzAbbMAk_1Z8~nZ)9;s=$vM`G?r1yMz}BO8M<}&(&aSz)7s|joGtM$a(kPwfu#@TNjFur=! zU@T(VSg0?DGnH8nrDGqy7xm_A-n^TG7_}o@=P-dxM{av5D;h5;$Qa=Yau{A$!5TrX zD22Y9Y#tiRX5vZ%X{{4Lvjsh^s#f@MnX`~m5eb>fdcsC=7#UQofEM{wcc3G9^Fujb zjQJVMAL*gGLDfym1v@(=+Hd_J=LgwQyWhZutEbe*lAo8T@Z=8(4Bg;iAhBq>W>-vr zs4tCZJ3*cX$r#!C^;V4HFn!50-@vs^RH>BW<6fgd!lAVe28`^0u!<2gpN+uf;@^Lv zTU#S+Xrpg++0IB8XEC1D?!E!JM}^RrXe_!&Squ=^uz_)3=&~PTivUE>N8}PoIJ2&^ zG#}d*Nfa~#)R)2>qt>xpqcE6D29Th}v~&KnI!qMolFIN!o2+VZa8 zA5^<8Cx0(xk$CrrchjPtN`PJXiocQu@LUB=Mn@mz&89WEQV+F(vj)bSc5F?7{9M@@ zB)jE&nkXA+yG(%51g*sJy0SlfQF&d43W7MZv$10;jI^f|acYCByF@xL#A$&DpA+bn zlEmTxMDJQ(s3&<~)bA}x*5CLP*~?QI;5rk%8*f~W{rLz^=Px~?_46DHPU(-|m~CU` z{R%U!Vvwof9Yf?bKR8R7zJ`BbURlBzDrn|L21#=PHNa~ z#KwKbdI)1Kg*aj7r`*+|?RGM_hLYgU)e?b(RNu_S2Wu$FAP0jvRS_6F*C;N?kijeq zFF=XzD+h~My)gqwUx~$l%EFPWB8gJ&ZW1es)7wFYLOFs56g)yErG4GcXS9r0Q&x}U zx!+ySc3y^O=&GFJhg^teUuyuhi`0bo>TL~>-K%kv&oDm6k-g;9e?20>{QLG1L8*4( zs1d!k?l*NJi}4 z-0K8Zqc_)sWIek$cs&Flbd*f{=%8>5^ks`ec}5Tx*9se&Puf#9#ofh6yKaIwzt5>pR4MxKf3J(WHwk(`S|3y*x ziloNhs!cbY5cv|xu-EwTjv*W+NT-LRq_EZQllr>jJfu|B6c$hu>j(9pBmS8Rk+q=k zxyLFY3gFOWTPc2)*x59zdEUz+BtW{y-8f7>6$sU=V)H4cKD5QHrl2xMfO9wkLjg=k z?@`Dk_r{uEQyg?)W)qrK=lcV};e&lI49`E}d|ZySHIpI>b90ZTO!k=jk%$VXV#Dw^ ztacwl=b{Fz4FVuT4cd#U=dR8aDWIyvB1(-;t%c4G0;h^?B&^{hrikB>}xgfO}NY3b9cPfmM{8) zU~~_y4XWoLv-~%8pOHY2Sh9JY*;8cTH15x53RGWjspW|%eovxRJV{Ha96*dkTc=rc z$#S7RaiQ1D{{v#h0xy{j$u+3GYjK>H1P1t(%$Wy%gj2s|(cMQH}0azH{7px792z_T}%)%$?^U0gnqtb|2IN&7}`ljz}Ir9FLi! zb5vKd5|=>*5JvhFPG0ndlNIq(5)&?$9vVdv?ts2oF{Tqq8qDme1TgrFtcbA=bE=7z zx^Mf5^i5U_A?AYR6YaqFZm%2+8}%4iu4G_~|I-bY9Bif8g<=Qa=#+uvqaT zm4@Ri51c%Yt{TAC>~$7E+Jq)ruGSy1e2}^5xQeERSe27k*|S)tM{=*!(<4%hU9rJ7 zC$X|IvX~KaMHHeac;Si2rPk-#M%VYT$@Z|i%3qwt5zFRBACNg=qh_@m84x&a)27$w z-JOO}#Qhe`K1#+uwpKoN;x=A(hzAlsFTVg6A0HQ=kUqbFIG?b%5I-j`uQ)HS&{C-O z{|a#PuywKz`2PbG!6Rpg0Ji@b!N5^Q!aaCGC=?*DrknW|sySqyomiormCwRUe zvoK8EJLmjQpSwY?lx4BdNzvip;IQQ7p1*;ELxleRpdtWAoXZ3HfIsk#Z)Bz5iU!EH zfq$NxYRQ`^DZ#M-`>1g60hVw`zb^qkq`(KbH4PpP5%>%L`&b&n-)9k_X^8*s|Gtps zX$&$PoFtt5^Ji*q@OusNRjYC_n*l+siF0BbU4dWv@_--G+B~)StoIb`PGKJ7Fwm{u z>9g`7DTRFoiZdziOqu9BHR%Do?d1bI>Y!)7pYUGbmfw5wiT}c*zemLphZ?x-&kSNh z+x?bl8?W|MGCXqg(0%tgHC8~C+nhJMJKmPwD-KQUg%Mm=q2fexD81P~^5}2JyQbM3 zBA#NaW_7Zxjh7P}+Hfgmk{BkF$V zZh39EI8X9kVoxVEWvvcuDstLwi&Z$$jl*7TdMBB0tB~8nh(JsI@5b~}68SX9r67Uf z-($2TR0z{PZ7=gs13&!U$OYlgQL8V^`#VGBXF{Ob_+L?AY;}5}MDTdHe{WOKA4G@h zB22y{wQ^`D+dvUrrP7O8}!8$aUMlgx=YY(SC6KMB8u4--0ZU0c>q-jshDjjms zSHgA2cHey2$@t~=9AlDKap0UkMQ#8wS>;>|JKihChqj@{nK#T&db3)bsG{m1K(%*j zJ5Bk*W^pErT=Q#I9i~nZSNj@d0YB|$(1QsDexRX7e7h=%^!&kwfLf4kkM#JB1|EnI zVQ*MbKio&zFLH+FH~pk(paZkg!zvz0DDduSgBhvgFaISGE+hrD3vr%6E=Wijp)ZmK z=M;RNQw|UjTCr1XIvvLAof5G)HhRP0;MTy_Xgw!4w^M9oz~tKA=6YNx@ClY-seb%g(IH7dI-INBDCE5&o8>~NJ^&Bww= zU_g2u)Tfa~P*+CIG*ooT(E0RS@k*s<+ur`lMp__owEJ*ucg@v7*M9)Lckx?O?uY2s z5zB_PYGa3pOsd5nqdB7s%wz9%mUcZO!S*maZJNl|;0x(^dCl4f*!X~&^aP${h-o7I zeb|G;P#?6Rr{vd#8Y43QFk|Tk^|La&!Dm=R>l*7V7~xn_VnEjO-}AtQ2B82htw81f zW6vM5UeiHEJJUH!xrMGrbt7hx)*!Z{L;szD=rcen2tJhS{D<>1NFIqT&6Hwo5JR2W zo{Gu&40Ij5;)7h6){PXtIXB;e;SZxCz5Na&F*W6=&+X3G;bil#?Q6PE57*54!PC0P ztV2Q0+Z;K1U=rbN@Abn#u`8txtnGcu(rMk5lhP6o?fVhZ=_3UUivZLD)RLo)j?KI^ z7b`XF8p`%HuYonttqpR_zPxF2lCm=6*Y(54u8C|Ck%J>54!QzWEAPu$+3r`Tg*{8B zzb)eJ@>E-Me9{9anrhpkFVeKDk3HRG*~tvBby*WB^`6E_hkf2S+&T=d^RIqf*D?}M z?w%$BSMl^;B_Kv%;G#7+E;pwDi0bW|O!e@;nE}1Rv(GNm5t2Y=np#=Nwojr?B({ve z&hgO~^4)lU zYY$uqKf*wgGpkiy1-Kkwh`9Y^Dcwug;5ui_)eg|1M(>FNw|Cs#VDW4GKT_~60&d{s zJZj;cJNzRjZ+G~vA5xTCvC9Q-{}GabkPl_1^F8hr_nA7Zh_K8$%}|2BhXawr#}5P; zH!UV?KgK?1t_emje&OBjd5$joo$RAayA6+6eA3cs%1W;W1I8<*5o2Fn^T}WAWGyR7 z=k1}}1T0OFsX@+#W+X}Ei4apq2keTKQ_&~{or&F3>qw^{9rS4$ox0kJm9=WE-V7`} zRC>uyowTH-y){_N%AO0hSaxM^2M?FnMRMwR?}(nXFo}P*xvW^#^st1Q4O2j&(^{Fe zNRFdl(ELwFB}7iSFje<7lm)vtDi6SDpoL^2iIXpp8TvO@err|eZxI=D_BBhnhFmO0 z?(ARV4Mw6ffBO3xXfxWYx`U+ziX1>xsLY$>tsZJ8L_E}U$-`^YW<)S(`h7ggTi5SQ zv+M8U{<_;57q#Jl_u5eUS3F}EqIWh$v02CiUAgxH5EHYnZ29a=ODq67Y&!ci^Hg*<1O3R>7 z@OO~5>1$o>P$xtBmB>zw5@SF`FX3K6$&I2o8BV%Gm{^!4N`^pqw4?#KDiKOD1!rfl zsb<5cLBu)@CysTg)nm%Q5VCt0xDvyX2A9F8*2nf9wz+80su71YqJ|vK`B}%)0Z(cX zVrG+Fgi8BTg1)!Jlnd$ILEkSQaNLqUO3}QP`u?N9Vatp-79~_Bc)lDeo*_)X3Ya~P9sdFrDM5jm(){1S_pd{g}a_jw=JOS z!UmsRpXUj?X4#8$rWFc*0JsGg0&TESMvVx-x}w)p9I;&?UK@K8Ze(LE+=saQ|MWL~a@DRDX9-&YI{P`+{{&w?)m&3RdFxw@SJEL>)an z7N%AvW4`hqtau70g5R$`Mm5N^#h&PoQnkpNdEjnd)0bStqF&_In;c3(N%|96>jG6t z{^bjZ7l9`&mg1nGdjJkK6uo85sqMV+%B^84#GG|Olw-K?v?viN1qU)D>$WNVf?W&+ zK{zBunv=?o?84iJa^}(;R&qL&GX6O1l~HFh)oQkw(_F2N(7}eh=f7i2NdWQ~`z8KS zO@dydp7|@G^rF4}JjWWXhGfihDA4& z>0yZZvlZ6P!dFc;)dfv);`=|P-??s*%t}u5EhIu&9AG7+A|5<eLr zskc>}#YqnpC#}5(V4G6aUGq!-J zfFAi}gyg>=OOhbYIf0iahx8nZXRrxTQYm4_PgqnR(>6S|@u(xJ8#s1tqx4dx-1_7S zdHhf80z8R+sc*fK!TT|_i6oVbpuGIUVbXEClJ*yenNKySLB48E@5>YAf0l0-N-_zA zy|KF+YKw*nOLANJ{FqwaU;>-uw$LxI_b={kooYsUIrzG+^Wx&#-ngC$-6E!sE~N;N z3^JcqV~hoKh>0B^b#uKMD3CM|CcF0@?_DeL*uI>0z5fZWJM@tE<+i04Q94k-E;KL@ zNun4Z4hgGLZZwNrcLgT{ZeSsv^t}tLy&JquamC>+{r!5ZZHAJ`o}E9qI!U>MNnjn~ zgYebmp@9yROoYah4jnXR7L$DwP%V6TE{O~BNoLueaSx+)zCcsp6vGvg_uE^7l+l*B z&kN;mIa>_E;b2< z3Ls4Z0^>p!zxj1jQEbRos|4(b?OiPJF!Yi-Zw_meSPkepAq_x24V!Ny18o(C_tvI=nt4%@4zqrcCh{z80QiIf$@%5!TG~D^b`ov zor*yWiVw`p(iqwhXow@sAQqak8d3YgtSKEQRpVFJ;1INSyUp@bU;Ajz^QaQ$|Uvsp7ALJ7)^ z+XZtlZPGF>R9SG}s2aQP{u3avr7&9+vrcs{JwC|1sO(+|^4Gw9g2Jsc;=0pE92@L?O`N;75?EhZbl4D;2a)%tT>3NwFcMl45DLI!o(g>J8pTd+M;lI_s zd!;-WwR(zEcu{jj;ya-~Sa_)0%RBHzmihkGRl!2ah#-z6>!rbJeU1x-A|@FQ*qJN3 zo-h*pL^fAYL?pHBXq9lpm^R<_q{Wcc^vY9hn@?u=NH@jIQ$By{%HCo%FC#~_IP&1_ z_K1yg?4s$EgX!i+4fvcFYBKp}%nolW2&&Y&6GOWVVXpnJieE82)Fr^{X#tJ||6jqP z2ldj8V4vZrGzI?u(g3a0xZ-}sPdpSb01Zb>y`=b^_=kczIVYqa%%e!=5%-qDu1UcW z4`dERNvV#9`TGxrz8N4P_KR!;T6=LM0kM$E{o}Rd!6#GMSuRQCvVMBztL9|!*IlC% zCG5Ty9+l3cKw<&Ph1P{A*5~t0JLTnG}=yjr~DPs@{K;ZFkG_Mn|&Thp4qzigEd!|(#r!59sOG?o$_f8 z*2P>1Xvn2hD4C@D)YZebre)2V#q3cJVgrNLe=v#*FD;2&-T`?4tq-V3xS*-WVQ>`D z>PXL*y3k5w4cxkX*VrRre@L(?CrP>GxaVA<{Xjwtelz3ni}E8t4^%QhMTd?h>=a&I z{R#K|p96Q$ocTNbn@|8!5266BKONFOWqzQYx#$s=RY7iZoi5KMuiGkb=PDaq<&a`( zKX=?4UtPa{OQxr@(YR|fnRRwTvxM;eAisOBe)u!yK@!!}zhBgp1R|!kZ1LZo`H31 zGTd)LM&n{n7fnloZSu?+>;#(f==x$qr4mUB*7BTaw)NEO3IIxGbs;ujC`CVf9gq%y z=)ua}ar=jcMGv*>!-8Kz&wm7({kK5Rp1rSrKy@ewpc4Q`U-?F~sw(n87@FmoMU&V+)V&ehG!<;s#cSn+ zHT?!VUr=ozX1qbSatKts3ue|=9waEtEk0`PU}~MnbyZIa4KUtz{ccWB6?ODVm`2hq zV(z85Cc<8Qv)d_Ok?>}zx=lK);AK(qzDSDjDs*JBV%DcOWqICx0bIsvTNc@zRTpfH zOneu=dB}uao?lf)i{pe6DDcn%g#|&glN;>zO(hQ#OV%DbI(tA?C}qB<@%{U5tdy2v zmVEUG)C4lIPvFh@Ts6)`Aaub%7D_8N-{`@4$R@#-*5DxVWhRn)TI%ym%!d?`re9Jt zXWpDB#Xp>W#Ex@{EBndm-v9s}Zz@G{)?=Y@Y(NN@)gK)H!wTf*(D>o&3ClxOkgivw zvVv}{%}xk^bw)>IVtR0v`V{Y{k0v;Uf<{M-*I}!d^rb1hxVQ6{+iIH8NX*|xpBD9t z;icW;=Qf>x)-?j?K#Cyia2Fn#H+-q>TXQQpP{nB~q17JevG?o&N0AV#jEire*OUwiFuxm>Hnf^c!kYL>$ zn|+`E5N&0E#=EutjQ`sq!Zo0@INo`ws_6R{kCuJ_d2nyFc?Q#mOv6?Jys30g$+L(2 zA}56OwGZyLZ8~my^XJf|2JMe2q@o^P6ND-@<&1<(QFWG%7DYetJs-atAdQyZDSYV= zKBnr`7UofB$Kx&M)x+|YEHtv8&kR;_)0v_Sl7*0OU4H3VrB=%_^yrY$x*M949%WMV zT_=Wk;~H${dyk!A#Ln{94z{Y9G6o!(08w79+c<7SdSoA_L~Nn4 zip!d-fL?C0W}IG*(2xk7`ZK_sjMCwHm{-~%NROH??nit5Mh^%Zp-s%4yhlOd0hZwL z+$;f&*2D)M2fRlvT?9xtPAP~Ah&PVv`HXunSDJx5wpr`ERuel8z5b$i zbg9^|2JvB(BqaWANB`PDoWISsrf+`5KjO(G`N2I+Mc@Y%3R9%qf;b0`J=CmE1OR>6 zdO&6Q(2eKfLa0+&E*V6fi~RWWw3Es?{E6_D9t`@y`Uk-iVlMQ4swd{d;Mo|aAiMhnebXD>~yY31j5zpq{9yIc}>`|HdBA=kGmDIsEaB@Yk0-i$~ z5;n!vec-UX^Z(uzl0h>xX>&_}<#GX^E~ur_HE#Pa831~?X(KdKZoR^oiXSM_OUKuJ zRtO*jGg)$pt|VPh7WPNZF%tn@dBp>31d4j5q#6Gv(0_}!!6MQ-pk#F}Ts3(yFWL=| z8F=O%1ADgL!7>R+)*nd~RRXAcVPn%cG~UZ^x47?~Q10+w#lnkP57ZDavDDQb*cnMA z#s9zjF1MEWw&e}79hnpLE}M#w)K}%#az2*i z<$3uKXt?k=fCDb}DvBN<=jGC?lqKnL&%E0oE`R7MstizN`fLkqndrF=Supwj_`!tW z@zMZutjhPNYWNRM6OjO?Tg@l2E;v|e`=vWK!rnI|&(B9j!O7k-Y(DIWs}hYC_c3?X zz1(PHy6QgOQu-UL#!V@B?oYYQuIQS?zbLC^eJeC3N%JmqE1Wv!Ba}UY4k)bM$j^$D zfKJs07hJxAW!CMm;S!^}4Y%*mr3O>?KxYb_uWYeJb%qG-BD8CoGs}*=%vL9o?Q8;y zYWa^-Q&C2JnZ5@iE?9<(qyc3s8~9Li#&7ge!$`67+*zT$;34^L$wz3KyMejrHTR(-O5a-k+N=VJEe>Gh^(ys$2 z2VtbS@}v3&`#*(_P>Oo1nR>}Y=!w@8d4CXUy= zUgv?3Q?q$mz=k>;f83>C$$M&2Xkx6*Q@l+*#z}ytgcm)l((Cz;nU&-MbWn4b+W+#$ zf4$XG5*jmlr_e>?%%2prBwBTi3Yk!s;I7~?KU$wAAxqA;oTlG&W`u_ANbhFB9-7nV z248_lh0>%-?&<%42azT~J(C=*>YVERmRvQc>)GA=-nK~t7z#db=wuL-st#8*^m3BbOOeO90i_;SoD`0xYIIc91>fsY-#hK zpd1?iyTF)ALdCC%q#O+AY@MveI{xG@YD{rL7BIJ%Qls~(C?b65k@%FuGBcEQl?1Q@ z&gDET=AqSoJ0vUt>o`fn1bG|H$7SaZez5qK=5;HeE-e`=X@~5jSyBlIfWIt&=H;^> zzCX@4NF1SoWN0r;n=*No?D6%O)~ZRW6N9G?0%^c(#M{A1VnIgphRSiG-a*l!PM>+h zeLXyF{n1|gUD0v|1kPt^f`~AND0K=z$_DA6-))ePjoni4S=j$Vm#nhnYfxA{Vpiir zx2hL^svXfLHEde{Iuoraz{AAfBNHOzZc!>bMN~vOhFKZw-1~H=)>q=?^ zQ(~n3_fm{D(z*|wW~Nc}S=tUacH3zvqs$!N*tuz}^Fcv*nLNAM{&{bwtdkeZ$z9*< znUP-wc;NNlTp_ML|9KQseO{#5di+~}qWxMcUbc}Yj`po+bjUPnG)@N+NThaI zeW%^w^r}+8HR_B6cESoKluj`cN;iny#W$j(Uox*FGr;RqcD7GkAM4QHffK(SO$Vb` z50;*lzEY>-De4(BnbPg)8uZnLt-=<~?rzSvW}{Lo+NvYDezs>exbw_E{`gqHc63g{ zR=#S8YZ1Img>&lmMZRGTs-PtiaS|kg(UB~9h$%4~ptQ^ko;hXz0+e*Uu1(^~^Jqm` z9mM5tL|1>tsHQUX2os6Z3L8^1WU#g57I&?Z(>3x1NPG#+WQnj^Q(|f3F18Q-5m6tQ zLHo4Rbn5E!nTSX@H@0`yiQB#*|Et?>XL8M%67(C!@X}tJg zYS`d3e}?+TI3dDxuPaUnpHi7MPtTjAu^txAV58Sj##+N0pA+aob`t`8ZDS-i(M$BZ z`yp&P<8uR0%-y{A+DL^6Yr<1KrHyZOZg%BeLX;Lx*Ja+ugw}h;89tU>YuQi zfu{Cg9aUf1%j;Zb?A|GuNP(8<*JEFFNXx)_IWxu2++De)zs|d@?(IZ7avzhW?fG;0 zL3VZUTOj;`RoHkLkFovmKXsB0;Z$oXj(~-`J1oB>F61R?J}SbZtJc`5qj%KiVj|Y% z=Njvp?$1dddA@Xobdzy^<-oA4h&t8l=g655sV!qYl@KU|<`J{DvVknXI*yVG` z02!|s#jYJ5_P|JSm=LWAlXOd=?1Cwm0k(>wJ!My)a^5Dn20>7p`~@Y-E(&Uvp5l8o zpLcP|x3#xn(iu&TPm=3nWrQf`J2*+CdXB0ZeSmTHzQc1~nes?+^B%xI#7Xy|?F-UZ!_P##A|cGlh2>S-zqYYz9(nyY{Ti0yKhZk9IF%+yi-y zW}zfQbR#A7=ZkK0lgQo~hP~R#nEm-{-(MT%H(?kJ^9F2xnOow<$RGAnC24x5`)yU2 zkn3`NTT`+i9>SM{Vb~X!1yKU|DUIO;D*fI9t&3Xsh%t z%^;XniX@S&>(QRK)zNBilBF=Isd_2Y6dA?8h9i!lRq*L&c>F0H!Y%%<)Qamu z=@I3)54_34FRa7}CR-z(t0H0ro}kLtVKuDjIgah4t9!aK{8)qY%$}gtPxm52sP5oP zaf3&HJY-7CbN&teE#&DKd(&J8*5hgE2;-(4QASmYg9P{*ny|6NODrcxs%`zQr?KEJ z#(d8lqE-OqMn$ zek~x#n>nEA4EJL3mnWvq;z)C6!!!NiI)hN74ZhiSEfx@`QR~gU zS;DezQZrSOnGE3|@p>z*{@T^Go5}gAV(=&($J@BPkWuD0U3Bw-eGp?+Y}zNx3k-At z$oZ43NI@pH+`Y=9qvL+>*>R58etP73%Xx5wX@qagp8b>^w6~TdofdzGR|8*0SHV!Ft(15)M?ckMWBk3rTxhZ^S*@5!K_|VAPT7H}r;u7zp0b&D zz>^-toH*mKB`5W)I!pV~Hy$Pqn`1=|x9&T& z(pCNyl8Ea>;6rn=T-C9jWQdJo7HfRm!Irv6++JT>+d*Qrt(42~otb>HPxYW!EtmF} zC_>vyx0F}W;#3F|A3ACGCPz6uv;`0F5rB`rxbcN{ zXb!I@I`P%OU`+NNhVnmOM;JC2$I~Z=-g45Yk4F>VnXCvtenBj_a6@OL#q=Dz&LGk3 zz%eN3St@R8vJh>5VTc{gA^D4pZ>c#d{_@f|9nmcOO&HWWHD=MSB2h}{pEF;C2&zEpukx?6 zHj4}C9e7jh)&_CTcHm?)nfISmBSXhq~cU3f7b)V9Pp*Y%h`>G;NBO4lVDXZ2*c z6~*pbl%kl?Ef3}OAmY1axupJsS+OwZx}a_11zK0fdvJiG@B($`zLGW@7kg&crW*!w zB9+gre$Un3`JLDuLx^X!iapkc3{yxf@#|0AHLnRa?wW1-7wCSm0Dgc$ejOJhv@ugsrDn!+`#q4 zC;@ zGXw3T<;L$Uj&W^Oi}%*&2mAYnKJFKZzU+&)9^#&wZ6|oT{o{os)f@M)55bOfQ2NVV zXLJi0BoBIO#9G-nbC#Zz8r{`f;x)4>q1kB#p<}~Go6|F~ymjFjyMnQ)zPB5@sdHXO zD3Re!0~``BJBSa(uk7f>zpnDJ+@{#s zN-kFAJMJ>nfQ=0I;&PJa7Hd3eS>pzVl=7-hbnda_qp$R7P4*dbuwY2tq;p$Nb&|vd zOM*0$ID}Gq6M!bXPzd`Mc=B+bA4OiGn}UT^~I+9 zc?sgfCKbk@((nC>Y>@;sd$022nFu0t*^*&EOPDuqV6nutHkCt6Kz}x~OWI7UHLtJT z(>3h4?i>$%9F#~$+wHSQa2-!6f6%OPE6&M09t@q$u9Z`0bLoiOfTi=K48_Y@uh+SW zhJ}Q@hUE5_xu5w6g#8T92vPaY>3h5`C~*bD@rjxD5)bdVP)8m_V{}#v@_mVX!uHd( zc4|&z!!S4~{ly;o<>;;EohSn`ZFeeu>&Nk4%3Ly(COgAU3QwUg6vX1w;aQ~~+QRu& z)czQ~RFQ7W;oGD?!}~IqXuHUFv*>Ix$U|}lzMG6KV+dqI3*>JF-ALj|4aq$g{Ms7M zH)M?PFF!Z8X`pILrNXIwyAML4v~kbZ`?@-oI89$9Ix(?Q+^!M#YQ5>mpz|r&;HjHA z`nRJO;UX&*1}oaab==Hy= zO~8GH$8}<++kN*V%I|%PzSTNy+w1+E)-M%PR$la;`ne}qvXmaN_qX9#TXTdpJ!-eN zA@VW{&cBFv&^m%~7uzu{FZFsTT7F^LeHD{hG3~2iS`*Cyq-4)dYa;QKRn9+tj+%+q zQF-<0`j=R_+T)fbu|8pawT`FT*w}~I^jVP= z$g;LpnDk0ay-#xc@qu@wWh)0)KSQRgjPi5pM1{9aT053>+&ZE{`Yb`AI=;8|WfUgM ztGtT$uh!ig6i0l{43RlTJJuFY=i{EcFs4Pork`Qa36yAgNbHp4vMCC;@UP_3eNS8o zLRe6!@iY_X$wD-mk<| z;tyV$DMwWIR!vHYYP42l_(<3J$iVJecelr-OE1ss>b8zY!|!kGd@oP#VH=TD{kC^V zqYJj1{~WHmb=`4v$pLbkljp^wqKK>Gl?0cg{&qKEQl-V4EzbMZ{>}T#kna2QbE>O+zsw zKm6VOY4QEm;%z|h;QiKhz}75DW4vE6t^tfJ`zkaN>>psoB{20^TOfyvNtKh7Um!u6 z5^THX$pU%OxW_=V5yZ`@?Lq8fn1Kw>x)g28h!1mxVs9DKG}G8qm>=J6I~^2Ai1?>1 za_wg*iFBtShc-3E(dc};TM@)R=bFuuCabT!p$_{s zzqq)5(In5Yatr|*^E2xqxd0IIaCKpsriI&`eurl7!&hXk*kn;;Bjf!@-cWkA?_%Fb z_--eW^X`&U;(B`V1(mm>NU!Z?Zv3&Rz$j9w8X;|Ff3Fu1uYz@9`acid*k*#i5-`i# zM$ZkhT>V06vaS^4*7@;VPsb3;jFR*GXFE|sp{e2VBu|BaR>NLz7@My}13O`*E)aLI zQkqGY{MhNk>z*acW=f2BYz^hcWJ@OJWD^NFZa$V+K|d!#i)rH9S{CPeMoZ4IzX(?sa@JD%ms3xck}^)g9s_q z=LthqhX`-&-A!jIv73#U?}f_!dL2$vV$r*m(}@hg&5H!|jksk>KAV-8AaXf(a(^|? z~o9%Z8p8lJs$r~h_J$NubkaxuK`YhhE%Mq!(T z&(xf%fO6cjEsl*{`$Ah>fAjH1hnOH4BfalS_Xq@tDvy>E1uaDm)ol1>h4Q_}qnc?@ z`Zm&M-#At0q(@EPKA{?^^2?oLl`0{04`!c!omg2dyG%eA?+DtAtEM(sd82J4;H0KI zU*FMbYUlX*NTo>pZk5xARP*HM@c!;TxX0I$PwT#yMQbpN9S;?#=m?|6^4Q@+ACVEG zc}|yjg4elHOh75%87oEeNW=-Kti)BwQ9kEp#%}x8c&oHf(|DJ z=F-t?A3k32^;qk0@!eNC*oeE_@JLow9pXI}?*7FvjrLKRj#O4iUoGU(iny`M;_Y{j zIKt`n#BhoGvSVrY>!Ls(8O$v%{%9SfO4=bEa*GR7x@$JG+n_#;yeY-K*KCA%UI=Fj zMw}|u{#xxH$4@3*q;k)*m`5=>Z00~8A_M9!R|O;AAI)3e2{hE#=D+J-4bP^!-^#zg zOr`Z=@N{~e6>BOWFo5KxTJpTF)<=2n`hZ%H61LMaP;GRqEv#z1NUqh;nKDrAcWizY z@6{374c?qpeiqXdS$LtOX=g5a@orcWsrFo4l}dmT7ICk52IEwI_Z@2;?t-_T&ux+X zFe&GJecw(H1!+#zA!>*y2JJ(_rP zXjNV)yq<3?86)3y^=Xi!>dBuXj|=Tr(+viaOI1V3#K9bT?vJTqOvL)P%T%EYULSw# z#H|Xx1kNlC855B?<>aZ$A&~>6F$(m;y%z?zl-5G~I0R|LZbn3%88Uo6P}jO*anG|) zsqeh+0$kk$J{4oqioe#LnM!q=PVg4=6!ZgZxeKE`TZ&%x<`U)b!5jgLTQJg8HI|K) z2zsb{=@U&n;}=Y$D!y(fSAvV)7rj?Ev#;?@(N8ed+n$g`#tNl2#+uZlEds3n}J9YhaLaW$^$=qFD(t#YeCn$C; z=RnQsPc=k{fXl~RwC(e5Rtj>c1%+wSa?dfByW`0xi6k%BnRiN`+3=KC6<2!>=3spE zOrr!)jXDh9kC*e`w=%Nf11Bu$3Mde=ub6xx{;`R=ChC1V94paqx5q@)V~Mv)@X5^; z(Z@oo-wo4z!}CcM5=N?;!e}^ZJtq7H4OxP@s|BoH@$g5CF%vY5bAm%)0B>->%ZY#( z?*xfAcQK1wGx;ztB!trk1{1u8CFVr=o>zS1)ymEZmC`$J4bxGQ6~{I8Tbfq(&nzLRNcdmjDNq%s z>yY+vcLtifUZBxp^`D5!pbmy49-T4{WS`JB<1>)>td2eY{0Sa%d=^-Up*~Cdiy7W= z=`uzZ!!ido2^HS)Q7I?h#35eGmkx8ZWxIh$OTLD9U0=QipX5-^P4}a(#7A-}Gb)*j z0TY-Ygu3i-bv&j`YCRDLjrL^YpLDh)rRkBi7fM9>U)Ag1%O$t+mqiW)H)++XTKAI1 z-zj4EQniz*bnZ~<&oLEuM!Ft~vyHS9OI%%Dp4cr65Ko)%vF_Jka%|F;rs{L+lPONe z8g>7g`x$*}ooerH2R#=8jJoqRw0$mb~9wRdsBkK_hfVN zeuJv```Vd_!?mqIQ*C$8d)@VnyyuExG7)P>%qlD38n4UgZ-e3%?c&yLGrc3}mkyfV zjo=B55^pK-d7w_UU2}L!JX+@X&H>S^46^@i=xIf~dy{5!KP<{Y90_-uq*(~>l0;>Q z3pX@CGUTt~(O!^b5)p&=w+P175GM!A1#QkqX1)1w=sO01&6LLxQ>u*Ai5pR#7awd| z-O(F+nR{OlPiAb(d_K$>Lqk!MSO3$Fk%S4qDeDw=1)>;4Cy;V>x?*YYdL(%A9l9$J zSyAGubUhoSlyDa;2?rTZ)|R9(B?PdzWU1*F{!C9di&zlVLX%@k`h?x&>EF{DLrU7- zbJHRtZB90miEm#$J?`H+?iW=)!kRlm<(z{i8+#2-++yBiGdjd&(rF9K&NBOm+F0<3 zXEgzhIB)E*f3T~6)N%&Ln{J;nHdM1j;0f4Fg~D4tRsW*{l>_0j?bXg6hnaYgv}@xn zCn5^y=2_A+I)Uo6fu8W4^Hx3?%B@y=iQBhFF~|$yFA8eA`6yL->3i{JZ7qtCQhWq| z_yGkvTKo4%1+o+UbCM@9lv_OE`Hv6C>V1qGU&Wc&uVpe0B)dOC$u83_Ro2m(=Bfy9 z^XvNEMM~vjLe>3Ch>fDkeCqG-0-ztKREEpp<=vkCOhhU>jgKlSFNjh;--Fkh5ic3h9eMrlpSY6B+rBz*62ZYiJElIodpKOv~Om*T|VZCVJh0(dF?CWcpp`MdlN;D*hZ zA${|$elc?b_l>J!yjOeEm zvGqi-D3x*+zr4w(_2SF(p2(YA=H%)<#x!Sj7%v41+kpArhc=8aHayqIa+-S*?;_JR zw_qYOv#ru&XWrobXEYR2#AK0!p&3GAZGmCwZjAH8m{;0!~PXb&XSMm-gxIax?FeW@H@2 z-XD5HoI5(7R;IT))H{#h?bk;BoI72RzbtEf&SJ%xtW%bsY{vS1oZse=dC1J2K=&zU zy5fmdz=NtRu&kUP=&>-LzkOf-;IJe!yv#BxL-x&IjK!5)>Nf9${~1QZYIPW+(h;5> z1gtIC(X9In^gllA&M;5_Op+bi%jv(48W1CFZhRA~IzQlHRNJ^jv%o$*x=4Tf2QGhp z|N00t{WMWqt3Uj%1^XK$Wmj~RAmVDu^U`sMlPs%>RdfSpGo@ZdFt$hZ_TY@w$bM5- z$5vbMRRG-iC9%4tHf?spRRAimYHY>r4XOrwHZuFfp8X>chqybXnge|z4oCJc&Q)K{ z>!{gXz=|TKZ+Niv>V&VP1xW(FGXo7yKG`Y}r;aKEYpzjFJ2lbD{$b>?{e@1)7nPQ@ z8P`P5<#DuE+7>yL{VC*ZqnuYZYOSraT02c0y{!HFtp%i+5?K#4gdgeo0LuzlRgSlN zaMl)wn^oUSc62Kg%oDkR;-i3Dr!g|R`qROeU<(^lUuyE8S-G0UQy<)OZ8Nx)| zf$EV=5aF^Xi~2|3sbs#Rn*4FU9=?L@xXSWQ(z!`-qtsJ|1+S##AF*P)gB|9~ye?V+ zJ+FNp^iBXhDi{cc2Vcu1zrXma&j2cfk9wxtgW?*N@!Y2Jcf&IX|09*mzE*f}kYp+n zpY3(#Q|&*D-u(T&7d7YgGk=%zZGdhNm34^i>L|<|Sn=Lq?8~e2PCs(gRZXG&`O<3y z>pAhaaw7k9*Z_S=AKta5WBoNDkS0R9W7=SJB=Ed%;_HRa9`cLdvW+OM z^@;4AOARK4CAKw_7?X#CILGIs1bPj-mn*boH-t8mZ=G#3jBezY<}#_1GCT}~YWmSX zrVHll+!o3%X$aXc63HgDkLuCy;SX(DUgtDNjDQa`BLu2Q0_Kg@E{$$`FNZGM)}Y2$ z1fL19-@fb^<;;3{bJU{k*}W{|=|>T+bY-(q2#o|QrWGoZ9F3inw}Z4OCn@bX{lPYIN*!=x15$>mD zZ)@t2`KS{xFud61KAvF;hBP=$iA6Uk$Av$gh(+Jyd-|(x%1WND^Ok{w!YgOkR9pY3 z7cyurmNexdLIx%qc^TzaiKgx>GvF)dC`hng<2EIgjk#-KLb!C1w# zSUgRL@^D@IuC7_t#Cbt+N$Dg*=`f_KSO}`}#f7H+D!mrPo3yK6ywm&g=Q>dZdh$O4 zW>r^><@;Wlsd}Sp4R)|~Om_=oQgwvzP&sXJmFUV9cP6B8IQ=_H~d{HY2s&f z`R9?F_v7kF$`*3vno$Pf82qo27=3J_cbE(~m+rh9)m+yv_D?hYNYm1L$)Ryy5p2dp>D0t?tzt@9J(Q5|LR^79 z8Y_Wt?WY`MC@6YoYiTL4H{K$VlsmHLj?OOWdf8kG1{O*89Gc7OABvXzA6xGjomsPm zYlq$G*tTsu>7ZlVwr$(CjgD=nW81dX@p&KNG$ zK7fA8^;GGdjo#qyjM!Sa^6UoVDf>h0prxy8sGX*x9f6)m*gNk$4z=LZ7P&M;q3zPI zmdVr0gj)?HdZtt!+yp{xToBu&m58WL@`0H?O^#I+2!}GNmCRqLSwPhdNGR{g5(0Fk zLgQ&9bVtQDO`NWf6U7h-@-OX!IV**Amj@(|Z6R)+DFGe@)?g_szdWc3<{ie1*qVzs zR^Nhk*_I@%=3H<~K@5Qy;2%5olVd6G_Y84Lz8Qcn3#O}C3pBm@zFkZ#zO~+ zwGDXI*1})z*()#26X$ghToC|0bUI-5lo*u27Vv8o0q>0|*q;Y~mL<-^duKP2N&CDTy*I0|!eYjaemYvTDFbj|%b@ zp5+ElXFM)Tk=UDQjx3!TrSi-!X(4MtZo@LUY^<&++EWBrOvZ3YU9 zQL-;I%ja}+h5DACeW12mC*HJ)0*nG2qq*P@A1-=MB#*+=BBroIclvhgb#iHFw%H+C zVv_gdIkzmlHG&%me1hu~Nm@7)?JrG3#3_%ss7~zLzWV};AdMSi>!WkSQt{85!66fQ zZbxEjw)ejU4K1cim-m|RyzB~Y2PI<^s90Phf9Xc-QuYu6c$oN;ie+wG)D!Qpk1C1B zhA8x@kKp85dvrJt!Uk+Mr)`G8Jhghtz9X=&pUWp7N?P*q>IyBDNpU$>T2$!YiLdLu`TB)BzXq}XyP$^AGD#bt*SgmiBH7G6|&!mk-sa$+!EYu z>K9&1c}0NQnEDPLirw9t2>ie~j?{$`>b4innW&c*UGAV; z4}SvOm``RP>K@e%ziPEEI=Nnrc;uZe-|Ws0eF?s~4lwR+mw06KsGZLzIC>lpM`Zfv z-?P`HoBJlQ-i{yOD$P;6t)=K#o4Lw{^YoNa|dxLFT;UUzi17%zqPF zh4%bFy3WaxY3VgzZAHBH2o>EUW|2-euT4!64w^u(N`BU2tV8Q-OY{nD1v~1N9gO<~ zIj;)EIX?#xt8&Sj{VjDEN1~mwJZ^nYJ>Jnn*2&%KCNBw1YFs@_uPrnMo))O_azXx= z=KUMq>#e>rKtz+qQs^FbNydQNeS~^5-W)$4G8Tf??|u%mle|<+_56F<@uIjNOfUYv zw@x)cj2M6@a#GxMyHjy3 zn{SRJ`Xur&E@hac_Brg{^4kz>LEus2ds#J{O$An|yRx^1dpoqM9DAC3%fRxg8+Quu45`lR->pHK{#%=WGKT@vfLh%~^bxQl4` zb+>kp8lAjCOj~zQpka}{%4Q}hX6#yI9N%oBmrZ$yV3?P;{?MrK-2u2(bT(^O#+tD| zVgpF1c555|5ft~wt5;dl9e!0sYSIVC=y}`=p@9O#8yhZ$8(7cgd+MTt)shplMx_s% z!z%J(_d;N1{s^{$@o~Mk;fw4~rpG%&`{gJYrfuC4x0JfH_?Z=rco*nd!~NJcltiNo z{k>Liw*rNd6+;<%L{_#9qJH9}g;|3OOXc>yNzn5a+d-}e;{D)qLWS0CrORJ68TD&x zj9J7587Ie_G9^%wj#S&RB8?w=f2f4Q2yOMj{;WI?sFxjv0uj{w*^#HB-P4b!Hoo$H z&(L|>c-x?>Gk5lQn$#qzrKY($aPrq$o*hbpsjD&UzQL!2ul*PAfUNe@sE&|8f$@0E);q>t@h{A9%XM=KhI!34 ztGYSjwfAqfPRuxOVg;5{AVXd;OYT^AFG5T8_m1mB9n+|G60*u3S$hhn^%E3ah=USkTvvAZd# zj~C&c&JxBV2-BqbUUr80KI%t?w~%j);)n}Ij6SaQq~HJ{fa8l;C*A3_XhnjyYN<$T zpV~dg?&JEo*6j0{Q=s#ymDn>ft9i2seOws+jw{Ouj^OOSC7b)rTH&I}OBlC|tn3p6 z?-YaN>Q+uhTbTQ1|HGeye8#S~rtn{zTNLLKldo zSevZ7wfl}{K0}XiN^Of7sLB* zc{`HgAbCJBI?5fL8{VX92FT043RH9FDvjnEAg*T9>^bFPm@*06V^_DwAA5kt$dAsr z>85Op9G#>W7*nCswWc9Xyo}YIsuvuyOYuc9nNSj?gY&*Egqn3tCPsBN_?_y;Ms=fS zwH7wk6f`DE6+sP)b2&z)!tznNBmY7sP)?b*?DNQuNOm$EZLtv;=qKtX%q{gSMy`o1 znt5O2_rboj+Be}twonbq>T1M*RE%jTMkUY$o23KV=Ar!-COKYpg&Zvhar^$C*t?~J zK18EK8i}ttt~I)>Vh3=|!?^Us%pK8aGKmRf(C3!m5zATTHM@%i6(<+(p7AuBl5=Ax z#~UGUmwBUm?U6rjD+&}5*eO8PeM!l9v31D6`J>dxYB&w5O6q^lPP(xidAJl&5C$sJ zR!unau-`|Pg^r;rZ~w-c;qZ)VFIA8$<;~i)p3@T03fF2KHK~_-%NwsIEpTNPb{Z*d zquZo~nOpY6n0D9u?Jw!`IIC3BwqSf66w=xh8iYdaQ>5+zWB0zAQSsH+SHyE*qna&Aaq;+FOKZnnR;`xxnLE>gm}wGATw9jpT9frU0o$J3EIYCWZq7X zR!sb+(vsh9Xzn}0nOF`>)>K#-`dj<5=pq9hgSVXuDpR*MX^h6t~ zG8tL3mPo7J@Cn64EkhRdq-j$Q*as)3is)!RW-V0~yT{TvY_qvJ2v4CU93X_=xLwTs zAaNpr>c0!3?0$1BB|i9GmlTb)fqO{mnrRgX@(-8mESe8C!tv<24#`= z|CIpP04hw5Mhsj-=Qz_+N}D3D2lATAVEi1FpTX07ot^9X#c5;0lo)!{t(`RdJb>9K z4Y=U{2Ym+cX7Fz-U2yt&c^syCqCV(raaq9x_0)Qp$7p@#MFz~x1auTGYk(dQoxv1= zh4F{@MpwbY|9=b)27qI}cVzTG><<1Iz&Do&#X?^`pB~^JwxH=<>)UYc3amP;bt(`p zS=&0lZpS$Ap_Dp7fl*1(XBa2Ydp;jPMpA|Jc+~{8SASOJu0AJ1`()}W+b9`1rzil$ zm)Tb7x3^ePPgy*+yv3M!xYI=}NyluJ2=kJC|GA{$0h(m+z{sQKuG`4$VcTJTd%&#@ zywckR_UwqRNocz@(nO@`QsZrS@~T|T_uTIekvYtcr>`tF>)DTeM&d7Da>GqoozES0 zM;UfU+#Z93w8Texgt27z?x&tG@X}ch=`b*W(lubnI|TNyJ4;;EWf@L~xNUCORUc3@ zBmEh5(}bIo!R#PKW(6I?f6TtWH)>N2dv=V2>tjaz|JZRXz*@e$)2uaNO?rpyTrD=! zRC*F#dxDG;jS+`%bfqEx+gJ!N5oRKM*+QfoTJ5sO1UU+B?hO?(F|(#|B(tN+w4|Xe$enp5!R|R{kG*`=l?xC(`hM-1JW6to3ok zgf0v$Wf>D|6+Qym2}5$A2e$0WHQ%EDLjMlvar)AU=q110BTa5C)GJhhRY?8kS4`8C z_sJ8V&KDTI@<{(xgPe(Tk2bjc6&U_!a{Ap7#|jgh3yg!jAxh(EcroN0z^h zfbXGakYm{^Exj6)58M#CmQvK(5Lq}NY@;kN!&P!~jM>f@Dte zQwO!T^1+UZa$d-1^c$DLibax8;z^Yf3adJ8$tzBB!@+WDQ1)Je!Ntouxsusx{IG7d zoz%noWaYQ6Y)HSEK9- zhn2^j2ATKLyi4dJw0LCan11%X5+Bx)Su4>^Y2&~wj`T!7JF1*q;!rqEweWGtS=6+| zk$Ks8&QrdpnV5tY2m%;Kpqi1z08WLRxEQOd&set&r>VDUh4-Xi!eEa>QDMD?gGacg%s$i=yBW=?FW7$TDh*jY1+2Ka0(Z-eDeNyg z^oD4-Fs^K7ds4_N{I5F+n+H%;jp>3~@HIiKajs{|x;SJqD*ww5aerWj{>z~-_#eTz zhv~=knc%iWApkR4Po(&tN%eJ;?T#;!$WkQ5-!Yu0XhvI1TeQ0RkSo^)| z1Lgm%nEyvjX9bMiCV&1vyikCFo@D#YY97I<%}Qkq@QW{mx#ZYhaolzsm+^B}Cfh_y z);C?vW!Zix3o6Q<#p|AB>)tv=A1%M%CEao>dF${_#bw-JE4N;C-w+vH`Cdv0I|8MT z6o(I8n|SKdtjn%HILXZaVHLyxinzR#;{R(rg$EYo$2A2J894(>4Ke0f+G$SB&dV<3 z>>dV{+EiQh=ui8o?DV1#Iv4}(MABc3dBCdR5>w2B2XW?Z_UNsux7(a-mEw_{WHm|> z1x$*TFU*1FJBii5CPlVzPg-l<<4wJ}lHM07;LDTT@S^bhf4%4cp_4Z#V6beVV2#IR z#4e03!zE{yw8{=;uahefjjo)=TWCi2D%V}6Kxl!`ec{Z+ZfEm z87zvh*DP7!S&N?XEUc)FKX0$bqqv}yE?Hr3ZJ{~jXFZ^7ScSDvQs7dLklHJT<}8PASU_W1s5#nE3<_e21CHk~m%7pwMmTl%2m{X>dY2kmJ*?#Xx0Kmd$q)dKrp4+W7NI52gq^}$Q z%3lEfLAplJ58Sj<^wQ5b6ZD3>{VOV9!+eE~;)=r3|0yc=pUXcRwA1Dbv!tfk}VBnKs+?y}dpeWD3D z+mi7KI>lS>6}iN zAyxkgqdh+WT?$_QS>>cwGPzbf1{*YM60*|&82~>AB6#c|P}JZQxg)tywiv{(&uyCn z%rV;ymM4h^6~r>L{_Lu@LBc(}hXMhRjU~D(Sgi!{51(1GxgZq&UlV!}^NRwRP7c8x*dU+kFVC`3hbH#THH;&6RB%)~eZbN4AHBRB;u z)d@B2gF{E;7_(`NXCCs!9R^fs_f9Ocb)rQ%(g@<>?}sh6yy-0PMVJz4@*x>2&>$=^ zl^{^Vk?(Q_<%PxWb}JEfeL|8c;}=3Aeh&^123>Q~S=b;D@tPqQ$SW!>1*1?0p(v(o z9DYZEG1t2Zkg$iEdZ}Qt&Y2Oh$E^_YP9v>ygM+f5+f#`79vQGMq{JqnpIv6Bvo?6 zMAGg(PzDw8{@aLF2EK_85BFiaFZ%uOk^Ug=NY z3zbmt$eQ^db-87-y&GoQ@MlkJXJRom+IV6y38qOq$7PYUhgr|~T=ZqP+2hIsR|v2v zKi=$OF$Dy4Fu&-cV5X+VyfFI~JhMjtnGsS0t4)k&wEMD+#q|Q`UBFZwYqSx?p!$CL zQVx7VI%DpjS~SiKS^`h#XHnokOAZ52#}!@FlYeS~P(52oi=hwFgV&H4T8m{LtzA5O)b#04%>9-Zr>E!)-k)bD8X+@gab)CZ}>F@Tjz1kuR0; zT#h1Rw9aQROkf<^bUcpzkH5*nUy~uf`mD1gV|6*vvcB?@N|Q=v&_Da@zxc2}qjy)i zp+Kywew8$ONnIR=E)+wYe^CHCcayFi$hOaoq57nzjtjXYv!H1+>OD23zla_G`ojLA z^Zblc=`<1O=^8iboLppY>Yil0lnu3zAKTe-gIALKQ3 zHefhL8l6_bxBG<@9c`OT6RF&sP*2bgL$bpNer7i}UsdolNj(u}Hhz&=0!SJYj z#h?;V&0?AyKBS{$>K;QI&}HifHfIblQN!n;3U}`W`I*qI42_r<$x`XJS!hGOcdJh{ zL9AHz8B157Ve{cydtKZvWa{tq)NWcV($egIF0P&YCt*O%d^1=G0c$3y3^U%!VXAd+Y)!<8KPRx+;}s&Wh%^q{ zlA*Q^BID7Jp4t>H0)w2nq7d}8oe_Kq%I=D?TlQ4(Oi{0DRF=N7ZmnyEQ2}v!@&d## zY&q3lKJ}fwKY~G>pX=JIa%p#%Nwx zG}r*8))(0Zn~GiNAnwOyp%r6+e2#8C0YH^I9Le;t5j6wm2P^2C4RH#yCjI~23WNGQli!D7UMnfKpHRwi?oXz^$JtxFbpJfpK zvieQvS2I7NQ3x|}3~Gg3j806P<&)obXl{S@X@<}6t5z>&vn7_l9QHJola$C69(50y zGpl&L#YC`Iy1ija@vZZwHWiAc8yC#{VX4oYvifPsw&HXz2t7B~{FhJ0U)dR2TO}EN zJ>fWy`LMDztq)(oPFH-+Fq(^MD_*L)9PFEma#Bs?MHZxFCS%&RHuT#a$dDt7N$rX1 zogUu$B4+C`{2b|6U!q2ntgQ}~mb&plIc0e3BPG!uKA_hg-@Mt!sYFsd%m~OYG-f(+wf<7bM z%PP#Y=Ge=5cF)3=8Ii<2$8LY>JXATyM2b>zED{T&R(iIyJ7wGuPw0Z~>co#5Hy&Ym zTc`#Pd1$r2$pxb)@@nBqSTNZf^#_Ew*H@Ck&{eviGJxrHyD9B2^ez9rhD-r8jQnnT zr8^@7`lf9K!WG^!V;13bb}Z1`)unIk!J5L_-`F__?Sdf;?u3StIdB}^FdAGc)1=&#Y z2>lMs9CWqv$%x-Wu`(f&A$$cP>6yj2SZrX)drz*lG+Y>*j)s&gQdE|1g8q{nbKzTa zW5(JOH~H1?d~fdUjXfF7@|s#Ev8ovZhV_vLY~t~AVEXYHa@@CfmTc+m-QoKjXgm$p zX4*rl2S9asYjZEQmqsFeA=`&Z?Z?~9F4UeC?a`m`H)DNS#wtHu)jsrLqW7X1&B55y zZjI$>^&fOnJvi1f%AV9FlVn)=(1fz!GgqsG z-fB}xq*Qxjbt&SP3L7o%P`vPrH)u&h*vqPnAC*wd-+vVGjbAMPIv znK{0?S8((pn>P4O<*Vik@~h*J?aRch%(^(M+zgsi%j8IkF)J{|bdfZ8M^Nm(QppEb zjLq+R?=X&eDj$!VQJdQoV77YJ+?qQ^c4$Wn$eY`(Tgij5BS&s>MyWJWVvJCU^a)dT zT+U%2D5S6p?_n4d1z;TcT;p}o_6_6Glfza47`h17SBMTJ1}{s?(}xp4OgjawQ+G|O zVP9FLtu&y?C`Z8tPfz1__kfy#sep&qUzj>3{K}(Qa|>vM$~3cP+uL64z=!2Z16W#Z zD6HO>-u@S}r07v-n;5-1N^Lg{=@EKmsSA%_u+TJb<{QD(18>kO%J*P(WE!4z9JBEw zh^@-8mCLvg&r8+~Nc-qHPX>?D!zVf5h;DSfI!DccR9X3X+_Tk*9}Z4#Km9-|^;OR9 zly5CvLZUVNX2YJSrV%Q+)NO@k+_5^<%cVhkD!sHkJDs;^teH*B1F z@2vu#vu=drOj*zUM*bFIk~VD6u@LfumV){@O|wR!X*MA2h5$n z6gYY+?WaCPqH!5?40Gj7jjn{4-T=FR7MMnVBViE1FE;PNvN@dQ$2IoeZ}AD8VFMJ( zl=QyWtQc`m)91bix67avbz=KH);6hGc+6X6%dy>22?MZWSRfS@+pMjCQ#7n#k7{aT z{oT#kfPkY%hx}Pj(2QQUQD;N8TruNKb`;F_?_|^`FKDFWgddD;ZE4gaAt2~>d`D&H~QG>${EF}zov(G`JDoL86v`< zY9s`3wScB7PQU}65{_RA05HmT>m z-7%)KXnFjlup^zfYj<40tYX6n>A3+*J7kG$_BartXT4@ov|VZ_QoHR>-iI7T?Ds-FNMVdGNdHCun$6b7;}XF4rt-1% ziL{!C9>PqD=nLc>wbS%tV)XpXg}ILQsG)L5iP;}G><@}7&(-192V6DZ%l;l+S9aG2 zvkaZF5@R)EYGJX_&?Ixj3GbTjH31#eE>8!i%~OaPy{lS7M&qUkGtA%fVIz*J%c$R< zfIgzs;T!$?^mY6sG$ocWZ6Dr`U6T~NclcH@^%}Ot8> ziztdjPRCF$W*K}*%i%$4bP$_$WDp(L2ukzzYEJIkSvU+k=4QnSnuQztD`?8I@szTU zR>!QgHDJUrgWQCn`LC;CU3^2q8!7yd#~FbT=7a!h;7Bgc59hg-*csE%UQpYD)!Bjc zB^nXIxb0f@KRpv>-gLtY7i-N+Dk^rI&zEa%_aicV@6{ju2PS;dpWD@PQ!an;FzGDu zcRTz&?B`i+rNB#JKQl&Vy9WJEOO+`GVSGM-UiFINw$0fwrJ&!^9h*N{&p_fNXHd%Z-$b^5*#vg^O;Wi$_gz=>dO z+~9BK@*e&d9Ux~L&7|6!u3hboY}wnZ*qRrWs86Z=u_Ql0tULO71|L+aKmTs|JPiA= zyq?0vooFC^gJlxm_7U~L(qS}4sfKJ`{M%CJ-dM-mrhe%C9Gb;DS}f-cC-Yb9W#?6m zHJonl)EhQSYL1@u0$3wv6WsJ94wTz2)`#15mUz*?OPz{HTzjJ)AS_`s>TFICwBj zDoL88T*sRQBjvs3tzX1M=_zGm{$r2|id`(dW#>V@S~0W`RK|CqkMex}@Ivd*CWUb| z?v6vkgzjW$#kA=US<6Pi9B|uUs0c^e#Je_7MS!>Pwit*d_RG|&Ot``~fOcpF(J}_k z?NQ@Gf7HzS5pIv5rLE03K1LX^^>)6c>2s%&qVu`Gdxz&^Qte2OBX#&<9;_ooxmjEX zeB_4_PKqB2Hg4J?WS2EWf-Qhsx1{6!wr3ts&HHB2==w@0>TlN7cZZ5opNSk_?mK*E{`)fu4=>7J4r|NP&f)Ix#Pt==E3r{KhH2b( z7I6g|uI#?U_;Js?dcnULjB6#oJki?R`b))WMS>y2VsK(`9rPBxGR;&t?ka^vh2?Fq@KuJXQ5DWLP= zZGY>19cF*+MK5$-t*@5TLuFu^H*c<#;TGGoe*+(Hj~_CC45_Mpjue0xMC#~s zR*Isf7TTP?CpgtnepBMwp*~C#f@&EO_9!~#9&Jq?=4ktPzW4LF zJ?w0(X>_bVyX@`i#YY_)-$K_cy-Cub7QR1(tK@%aNttK>`Rr(6%lsk&sg!ReD%R{! zOSt7>{OB^b--;uwIH%*|P)4u)ZI`xPi#TP@Qa;(0s$_9G%}yfX^3&u>%G*e-iJ9(d zqhMv@9=2@0)9YA<%N~hoy1gk z3lnQu)-E${Q#NVBuFPxanK-WlyzM{gk;t0bTU%Lj(n5TiELYS=sjKrY1;d(t^bIxr zXnOnota@4y`d~hHI1V?4meA2wxu^Xp_PKxut@HVDe!FpSAY)bvLkAkFZ!g&OT-&#w zQ{I_3a|g|aS&~`*3)_X<3)*SpiPUJ`Sepha2Uv65<}R{>nOkq{49}(pNf5UlUt`5CuzKgEh0ZG5kFpdw=f^ z^ZocEVQ0*S%*KbBc1vQM)ZXTL^vxHoO2(5^0vS117^)V5xrl@8Df`<-o z*;V-9{kpv=IwSjHRf{6E6+omqG6e?4=b;i7_<}V6D&2$yl=xvy5tTY%2`mR|qDY4VSFoM(!>V*PQ7{ql^#TQxfl_rtPbd zoZD^BpoZTIspFIXd#}e=7S8PVp@V9)E)MO~UzHF`7p*>|e0*`D}zVLFd<5Xvqb6LJ;oo-YeGcRH?S zcojH*&RfMSS0+$W8+$EH*LR6V@goc*6o5J<;M^)f#-+wwJa11|&TPKPXiKH*GSe7f zFA#et#3)6-;Fi%qo;h4@AS^$CO4M>i6f;|BW35tSm7Fy%D9tF6%ry{0no^@;m?Yc` zu9#u63|eL$J?pyO?>e?xB1!dE z9*BJ`r$X=S#>~2ABD%ViH8p^vOM4Z-EiGe&pn38^ueYc^W^;=#s?UF!h6X6+Cphn# zo$q{c!zG*G7gX=akHHwnh3UxXy=xGVrX*FHE%+tgp+!9{Hwrf#=*TcU+c5dZrLMPD zFSm5V5A`*p+WdqwJAe8pahS6o7KZ7OA7TX1V3|jAdSKg%fT2l zAaXcn8heg1JPli+c8rkq-yg3m`EY;iRA<~Z3uK+VOmkzbNfIMZnzQCtr*^a1UZc`Z zPn2EMVojNSKOe%m?MVkw$ocL6L33(-BHnz`Wm9-%-GH-vN=Lg2uC4ff&`#w7+eE=- z;En}*yVoIAlhkewobCk8Lar`(u-vA?wogP--Ly+IV#$8CR=rvJ5)rjlC#69bwuN1jJnd={WW8p;AJoK1`YjXMF{v_NXM^@ z|2Fzs6u48tkr{^tS+8yCEK!^a6z5&cdP?Dg{m}dyq`mHDp!V*ENbu1Vwf~Kop(Tc~ zK^Rt=BeK3BYG43o6W~H5(;{B|p>fo0<#nmyjxEK^UVW43$PwP(fHn|04*U74tfPYW z&qB4;fmik7+eqPTb!P{|g#r16okteOXwuXFEulXXOYqsyb(=)YPo1(70|f1xsii3y zRZQ{48BE?oFx(v^UX&)@kMwj2Fgav0_P?rPlB_utr1BARNn|AEM77-n71ADK-eKUkpKw z4<$C<&wecjh;wvcb}Vd!S!yak3q(Jfnw}&q+&jC74p1|y9C~P@DGAty+D&t%`25H$ z8&yXUvWI^?flz5u?ZSZB2Z72;Mo(iY(bQt@y7tkDtW z6l+)}Tuz=6m!k@qw&6AB${kmh zorh1WKoea7;@d?eG3IF?q|xfR%9Hx;MKWiBeOX6VEw>((u)Un2H#}ZdZr&u%=pw+e zdY1_~9EIS-#`*Q2{$CF|J6DzE;twP!B^k_e-q>5#FuX}eZLzHkC(%)Mycxd959Ee? zr%zy<&nL0^LV{(QGOiZdXrvG-iBq#fMyQ+0fM^OVnHW#f8P7k3!e_nAKG<>q^?gBn zP%~+l;9iDY&4vb%pz-|KaEppdv+?`D8LG+fP8-L%Ra+CK-#0MICY>^z5Ncq+8wuNg zFL_bC=RI9{_0BBx6I`Ps*n+(GjC_@yjEJOh$1 zFr+Z;KhqDeC`A!$`4aJhhP50aJaIwjnM*Mn!r%~1747pT`c@Mkfo@d_Ck6K~<+k{h zUhc#_N?oQ7JavI0`bBD0`=|M&(nVY@>LwuD!0Yhfsi8E0>Vu%NrD~ax0EN5OH|xzu=Vi8=HDKU{n@SB6L*^ zoItXVl1{!M`|R4D@h;nl+5J<>8>!_V= z6v1K3=-ORS^>u|3%aj#5WrbKuO-2G!_~NX~)9U%pSAobN4zOc=vne>rrp((y0*Ybx z2#Vafaq@~$#;61_IZ`q8MHXx~>l9BpZxt&%h3RKzYgMGMGx|&tZ>*4Ghu8>fzvT^) z72#}E3Yo+qsXU7o-ec)Bo=aj!4D0=3W*^qw`7&@5g;}j3rygrV$}y^9AYQD{H2L3y zR!N(;s+C49&iQRBD#}xD4@!^=0+A35WBMNyThwJ~!*}Z0RAg^MRhcw!Bc5(`_Va@l zx>ncDF1Z0gb9ksp4WN{En&sv0U^-HLf>l)!I>7U2N6b z?V=1#ekLT-PoSunnnrX($(=B3wvh`>&cs_d}vWRWENr$ zqVXMP%2P!S|BA_gMWKbi4RNoPdpWwU0v5<=F)$tC=I?oIH&@_TZrRB4CIx9~zA`aBvvvXebueJPA#ol} z?-}N>f#9q<`|0<`-#Ds{y?C(cYmE9(wchM7&MZ#<-Wqs>dJv9z&K;Nb>7shlx%t@} z!L-6Ib+yT`)v*F5_yeW#JN|r%8?Pa*Tg~ZE$O=~y)Fcle9J8>mhXE2(!2eGQ5_ajK z{2vxTMs>`f)#KKhFyxa98en8Pp^&w3Vg1esa(%8ie=wJ%=h*Yhx-BbGk1Mke(Dpg^ z=0oa_8LmkMt|Jd6-O?mC-QC&16NhSW73M-8Z|&uARXY(~4}2Zkwjuq%HPg2)ubvd;uy$0V zBGt+v`%yKeW&-E&q#cC!&bYO@?F?Kely>89FpYgi2l{7aEvvm61skl8yt@HF^;51Q z2tLR9j2esZ*T~hw0heA?;~{?Ri3COH`cGK4w+GJF`Z$3upo?WUX<|!y*C~9t$YS@j z{gkWwuVS98?uA%D?Ui#T`$iSr6WWcBAFl5pR?iw23>we^b9qC86Wy>8g}D<2{cOFr z{B-e`FIOzDI3`MVnGLSaR`2xU*A;5>DACeu&Ssz9)~dhW_pO#ontR*N7SE~q)~@a* zA(xdz)^2Gf+%>*4;JYX(81OdPyS!PLQm+Yff<2-FW2jqXoH;{c}MRU#fs_i_BiD8DJN*9MhezJ6M@yQMOH-en57bj&Z7BqQ7&qe{Rzbl;H8NJk+{>x4Y zN{sI~IQ~?)$d%%#udKj6Bx5Io3@&TN_9PAnAkqEjT*wbtA>fFYS2Xy!W(^;tuU>`o z?0Ufpgu2N=w)-8S`XyE;qDHuid#oMM=CmQEyrE|8#khrsqKwkxH%-!aUI4Exf~F&y zq(I9VE3g*2fjyO&71a$U&H__4ZFs)URNb$2zZC5P)jLf%0h91-KgJ)X)I0TrLE;kl z2^XucYDH}sPv88!-cHssCY@k50L(29sw*1FsvQ9F@FB>K#dwVyPyYo~N#j=@0qq%{AZZL0BKuBXv>`z0g%C>A2i3ZQWG3?4$w}$<+1=8WL!$ zVw6t3-EY_qUl`4771oEEB{o)DC!}$8{~>BKku!H1H5dH^K;)@jLo`Kb&hWPa3N&qW z)y8%c8`x~74;Mn#N$G`dqnXaRwtcn25{6w9R>*Gx<;2rg%Ztkp0mzgFR@kF_5Qh4-eLE^?t#{NJ z5QFO>464`#p|MNImWxce8O8g5mT7L1-_DM6DT^6h__?r^P}M496KEZr0?~DrnVfW5 zXGvN|{CIM{Mx^6i0gqh4($L8Gx;w3I2spHR{cg zs2m^$e?N+`cDM`wYPg`5i4A5%{LQ8LN0Xgt#1^SdQZg`3C<-)0wk;zieNM_mt?}X@VCafrcsMH?z_()`6uypgZd8JN}-Vf?AJm> ztH|lXA6RG;4JksXsaBL2Qbx2($+!u!Xhv+1*=eH6676NRZ5b&YBcCri8qo56ZIPy+d zqjx#Br2ahgg*Y+axK6U9@ij*~i;sfFA|X=w)fASOye0}RFdi8(93dRhv~#0B12Q*$ zUZYq!Vq_fwP{?%VJ|B86HtC5ArdyDGwE)gfw z-Tj-J$#rci)?u(PY6i*F6x(fWq57%sN%qCd5+G;MnYp-49D9i2o8b(B z1@>C?S|hb*t5Qw58-{C|+`83H7BGUADs<1fTmlUaBKQRo#!x@fuflBA8*pMkjSXRV zO?N?6eb{Wp)9%-h{$p5`l<^xws5K-4;*YdQgJ~X!=T_M%*6ya#V7nme;g|Ne9Rc&f z@m8yY33Z={@bofT%1QrjmiH>!rLas;t5m`?)9OWzKLw ze$X*HY55@4TahlVcMk+j&>pwhlLdi-TZ#e`V*ONMf03@+s+X*iH{rRq3Ur;?www4$ zmYTj|ElrQT7!Jr9p>HfGB&MJTvL7l&3E>=}7|Lv`#nu)_JEmzyl6>-*Q_uutAoDeN&iiOgASz z@4AA!MlCX8NVRWQ55x0Cv0ZLd>fqGtPy3oQk^I|~YB)ack{+NHlpA`H0aiXrkVrL} zYVp_S9t0L0<^zYRUDS!$np2y`2|GKzf-Vg@fzh{c%cN~JCNm8F?Rq0o>v;nm85XnR zhnL>@@2`--03ou6;93ok;k~O}w`Smgw2Sme@+X0m=aD;@|Lk*1l$!0JsXYt*lax=r zz9dLqz5cOHsJvEDlm=JzqpD9C%u>Xtfim=g)uk7ZL@tAZ0c=PC(#}#HTwtmosAhSH z`-09!dobdVzu=61=fH4!I)L2B58La;QHT2vOZ(IKFrhSG0g^Of?fl6Ay82zLWj^0Dn; zVzDmPba%a;VE7Y?0-M1S?*Tr+^M7c%2JOI_W;@Bmwr$(CZQHhO+qP{R6WcZ?Hvi0b z*L{WVbE>*_ZB#|Ioa1E=CCStqULf7#EN?51{d-c&{u?QBJqDMK=J5T7Ad_{}V3R_h znQ>Pe6Zq~$!;$r89P-=|uAMRCn}=-42^3A%p~zfKUZ7Scqg!}zY@#d zmY7+a)~e}TZr{Vk1FaZs?G4-hx>nl_6zBgENemnPiIvCC;(hdj3O49!Ax&WHZRtp3 zS%_g*RpT0Ld-$-0;D77Oy&S zI2;Sasto)(KkCIg&T~t!a!{6WH`%o zyV~Db@rp!V`;*qH4HHpyZ!=>`gx7>%V3gzu%YD<=B9K;CK6Ktq3Ph1T zYJEaRg&t~p&;sVzVagm=u8K=3a_8QCrF_S66ODcdeY}eMb}A4e0`ku8^T~N;<R!1O z?MtONF^DX|2qX(&1SO)OhqQjJj>XNpJ_PhVsG^wgbNCO?AZ|ON`p^mq2ltT{JsIvf4Hji*pDP2!ER{#`a(nngE{V zk1q?!J1(M5DyxPZ++%*lsgk21*bO=t-YpF3jkr^=8l1R$efEU-VJ4gt`gPwcCV>FEq1BiPi z_bi>`Fi zsDpr1?9PSO;I=7ma6E&Al7na5GLT5BCn@Gp&Liij$|0o{gb)E3-rQEGggvN>=2BMD z7I}D8``=Kb_OIYMb8V&z>Xm{G6_nvmY>4UIN>M}R6Wj{GL3tj-%@HT5i&4~foiyUH zq|~zVYIhsKzvxJ>4yUD4?YGekpZmYWumtBZuPi`LB3K&3Y{bD=p$`M7nfe{VI(g9= zU~ff6#GgWwJxyvC`1;?H%|^|GO=G8a5bO9m!?X@G6(h$oW2t7x{dO*hh# zUt&tp>aWtIW15{fmflItZ0zbc$y1OAEiqmu0Q~C8X8fx9k>j>#cb!7*5D@TkgVK-D zCU7i_p8+Qlk=`~l{9-*~m+?6($~mKE@SzBTatbbsk%Ue~o$J;52IlIBM|Pbhf9sj} zON_%~m@&YhkR^FJMppJLImEhxOJ!9cygH2QA<2R`BDX`zRG5)F7~-QkT5b>&S!kwz zI>*nHiVVXFf(6(b{woQc*;K-hx*s%HBJ#-?ZdkC{nA$*I_ppY}(v5}kE7@I!a{O1x z?9fK|A1Vw274WihDP&dHs8l%u<`GZ-cJ1g~7J0zra^nL%^2{wjqQ}eq07k+H6rP$k z0?v-OYl2{vppiZtX!+xeOBNN5#lZK%V!CHuH?WaK`U=0#L>W@~Txxkcgjlas_ zUs@Q}__O({fx%Ev#Je&BU~Thc)c57}*q)?*Jskw}KlC#k6;cX)xQTHQ%w-)pcYK0l zk@AzqbfqM#iUG>LPBCxVNOkemt38INut=w$Ys8@hJZ@b#w63;H72Jg+%rlCuGu>XX zG8jz?`a{Z|qXd6_sX>)!M+1aW0W6!LrbM#X&{%%ell~_op@L0*m^paK`D*E6c`KkY zTqTAY!K?{O1)&-%#GV+1&x}X&x`3mPYSfrHz37MU^PpmcN&=-Ji?>K*badRwi{uPl z!e>_Yo5~Y^cwS^nNjRCcfn4EON=IA4)uLzUP7}x|T}`M|z%Q7a(w+kJ`_cD)XW2(- zFs2JkqEgHWSU#)XcoM0u-ZqtN zuREx`hI}aqp~2oS;G33=tZvTFn`7Vi((mjXHNRXj=cCNtfs)i$yA~)*(uq?|o_P~= z%a(`=^sp+LMYvOizZlBPs$MNo^X>&C*upwJX=P+jT^c7#2gb#j0mKlC4(Ni`LC{@H zmJ2>X%A9jE&HzA@GZPVo=+Ku~+Z;SF^HiK%XSluR-0o|fKFK&aLgFf+^zZ5$21+_z zyDH!2h8idO~;_+S>%eq)2OFy4zLDim9cYJm(a3@%7C-m}fZO z<>|AV%ALt?03HxkEtPmwpa}jRv_YWSy6Ey0v;K=&vFZr!8=sx=yvp;B#@*afMjLR) zm8-%+&U=pHR2h}nMfZZpbLg{wYt93U{lT*D1CFIH>}ffuMwC;8l6CohmoD7#zQlfw z`M&6WC9XqW(DhH#dSwtam9t^fw7>Vb7*Zm&-&anXx7=Xmjx;zm;B&ZG?fQ@`T7EVTQ zgLm>AT5llF)8_|7nh=Hs#2KgN(WpC^msBqht#1RTXcFY2Ki13tqR_<5k%xIShza_% z#O2&hBaT`{LVtuLnvy4La7IVBo(#L|>JphToYU>tj^uFt?Eqi@g2~8`38A@U)&}&q zXu9-~NdjRw_o!WyOAe_prxs!X-Yi)(kwDg?%yVeiDkaFgl`fA2KaT}h#s?9QZg ziBJF(g09@JLVy<77?dGJv@IH~D5>;X3B>3ym|qxpU1MFCoGDU~9j+VEBowlM794yl z*B$s6%FX5ym~ilf0{wzHA%9($k{dInIEmO}Lc$be(2s4&$POwu?5Mq>PmTkE0be_x z!Yu$t8gue|w*lgU;}(tuHdp8~uz$sup3U9kGfD_)Dr7a zrtH~W!Rn(V9xq!ZnT1#$`V+l8o7%ub05EbhaiF_M#3dz>dp3n>ywrRr*Of%TJCfb(K~tF?SPfy15T z;iInPqLY!xca5cxnhyZr{+HUP1u7NS6KU#%n~=!lR*=_lCd=&cE;>rJEuafvQMLyj zN*}tU?vL$#2DFtFB4%Ke?5biuDd=F0p2+=4x7Khdgpx%%0m0wX~m&Jm^og%cfeX zXHZ4KKu|+1edUGJICZna66z9uQ7X>(gsUPqcSc^&VFpK~G%DD`Y+f$`wrzY@u24L% z7LU;#$6}NSIgGoGrr%N>-@*WtRwDrS_YRO#i&GS`|&xg$dbWU)UH>ELZTC)==w$GZH;M$QTDp`dGtC$hSqce{t?pKr7SWY7yf zm?e!Yr9CieF@mq-k%S(A!qi% z+5;Y3-in|{Y>3F{IVupTp$g8J(V|Pg964}?Hvjb&J=q_O#Y2|kO_J3ykjBx%5KOnj zM7M{l?EV}HkW7eEiwwf^LC7=9+hi5iBr)yJE0xf=L8*EdIz}KdCS)(O34T1C>H)Ln z6InUNf+o;{z;K-wC^EmHGF3QzAsko`$E|84yL_9HM0%3@@8VyE&RU10aI@ZcR%kJ0 zrwt^LLS`mao$fwXn9#XoLw4<x-kPcXT}pZ&t|2_W-Y;p#(e7KI{Ddm_@cNTTr!s)3KJ(%~1A zT{aP9r@;L45yB4cXyhn_u2h*>sDTfs7lpQ8pB*^8=|_#N-*9fq33=D~Ck}^=Y;IkR zoPxaVWdC|^`<;v7t_!>xoT$WF-Xi9a4W<7vc5D0~H@7yN)*kM8;RQJq#uHU%0p2VG z*03@mLV7@TbDX`RLnYAqfNiw9SkdJTBXA@{QNxyADHkcD`eQIxkGI#YC*e;mN( zZ7vb_Ax;LJ9J2Jv4pv5WWk+$RXQoBnzglLbVuuOM(J$56n4n7&CPS#F9OnK}k_d(R z#Pe|VOScnA%w@W$8GA^Rcl4Rv8t-bf_kR_SY%xZGt5ecy^)Qd}y>XPrhYVOV4UN>c z`t<=s6t52OAxzDU(>TB{Sy~Wq%%*5Zgmh^y zPQm-Z+c>F>rn8^2{aC6Z1yYX~13HtMgHdj|RgX^=Ajv?a4~s-IR7FQ)Fp&f_jCJ`O zL(6)M(!%C=b9=o-?v33uVd|imc?Kyq<;T&l(EIGwlTI%_yw>wlR-O^zYR8~+>1Po& zEY*dQ$9LEmXXc}^nmcsCv|O@U?U%)>ARo;Wywcet9gtclDU9M@kxgKVO&AY|0v)WD1UOfZbPD}Dq`-fR5YQ#CfK|Ve=`Pz;EDVSM zAW1IPKbW0{I5%SU%?1$TQK5T)Mw@Vve_Ybf)}14j$!ydlm$Uo8I}Vn9XPsB)xQw!h zQ;3LmFjazv{q>?c3YUk48Yjjn3gOm@j7~1n+Qokw3tCV=Z_g;@+(rhJ6Q3${KCu^Z zbuu`vLFy0|9oLc{9{0!K*yz!t!7E@;7w{yw3&=@*2xJTV{yLudIw*2Y!s3aB)oEHA zo4n9fI+c6}6%TuYRhg`>3(;?H0cST+?ay2bZdi~XpM~_&tb=0dZGtp{3 zYPASJTmIuzXSb+B&NvNWHP6K(c}lAR|RV%vs5+1U>lEP7;Avm0}J{L6@~RRhQ~&cext1A){9Ok&|K_yv>@cF$5UR(g@vVe_vJ~fVm;u=xL=v zP}+R^Y7m;H2zj$O23iXf-($a43wtd{XPeZTCxBCU)tFM3btEqWM?$}!p5omo)w&kH zb_uIRwjOk-O%@s>DZ&M!8kPvXYQ<&{@+5s2eXkT(1Q~&@YI3o|;%{K2jUkaH2WkJ) z0yBg)=FnTvGW_LY!+1vt-!W0svU729YxI?=Dz37CK$h%+Nj#Gky>MVg#^mlVlD&^# z2H*uzUTsBO6;UhwbDmLrjZk@{v;nzf%wP%zOC1U&o3ol={#cToW}19@7f8i=)>-N$ z91U|c2#cbO9W$|llZJ`@W%Qr|oS9}Sz-py22^?Y)vC=-PlOOPkWZE|YX|sc6ZI)C$LZLmypkr%6i@5kU=&vE3kj zWXpxZI#en~i)wH!_vu^xd7xZBv=p7sw@xrg7!xt`K4 zr=klWqm%()6dUbaz1Fo4443=m(NIXF4X*SwGI7(RaqS#mo3u;UB~lM;oesuxGpaO~ zWCibQv!GjN)~WAfpbN`9Hc9wHvi>-!9C8XDVg zBN394vPp5@JlUV`rp2FA;*2s>5t+y1^16E%eD1~okv(`Uq$f2%yPHLXVTMP@5Du`3 z+#4tR-mGYU|5>*C|Hu1#E+*CDegX#2nT|ZW{Iq457cyLhugy8xIx2B*p+F)CC@L$& z<0y^}#k@Eru#Ne}M@iB)$sblgwi%Z1us3UgbdVg2GXRjMaBh2R5MY6@nBn)q4s&pK zha?d#+>SS?w$TqyNQLz7!K# z()XwF#5DgU90?jeC<{$CR8WyJdOOhQAsq??I0k_kGe z=WTfS+chrNaKTVQ+X!B8gsc$~ePBOpRQ+xFLMc}3gBze4<*e|H=HG<1M^(f0E9 z*CB(J5n*dg7z| zw1DI;ZNp1V!1^|E<(?~I)q&$$tE*zYCWZ2wt*zAHTs^`a=J! z>9aJhbp+&6hzaydaHD5kUP;ELXvfnP{7mt9Sl7&UoMif{Y z8Wf-a^A8l#?+bH8(zE?O=+HSavI|sM=lIlkV9_aft7dDB9Lnpz)K2>?%fTA0WPu(g-)>kQRud_B#2~@{+X-G}^7?p>u-+MOuD8 zbsyqkadETx0GnSy#|EVC5Si0n;TE*Zd&~*1O^;Zsr?PSKyJyTw!c*VRlF>Ud503{! z`5i?7845P#xd#7U8~a+HCW`s7^uWbYk^3H5G@4s%EE#(Py}d!#MAQ6^Di9y+13?6# zWBo%Xa;<*sbJ6cUt2O$Q7!8C23iuqux6wA7;JXRCYyDWi^zF0w_O(0^k_mnCzLD^l zQSMURd(4aP&%!trV^CviYYcaj6`jP&iHxHU(eh@-JzMG8Cs-qbK-i^b9e2oJd=@J~ zerf!PGIdIYtRdTR&$#i@w;cZto&H9tw9lSrbc zAPqXi8S%}1p&*)@?e(0&bDqi8$cX|OrVqaCI97{UbuVqfOt>Vq%NJDHS^eLV;2Fr@ zQ#9`yoTdf?#(#s4uRH0e*$Xp$N93pVSy}r0Ou`jZR?~){WI5gHLSp9l<5lC9wu(!8+ z)Oxm?=WB=c02_gqj`U?Nk8-ds3n-s%M)@eTdqs49fJYZoVv3tLA!j0A5VKcfzI;I# zH%%*il7?q71GkC6>FQy0IYQIc6jKof?7*HjgR*;~?EiS`rSZ1f3;ChGP%wP^bR+Ef z)T)4skf0V)#FikBOjVRpS@=-Fr%`OD`>zZL{#y=&ZU@$U!NU4u!kJ;l8^6Bf05b-n z1gk5`5eb!nf)dQXi}`W4PD6}}{#rjGhQG@2sH(2_^Jf87&{2vdGnEZdFVXvdp@d;2#vQ~s?6POR`T*|=!v1`d!EP)qCI!Z)nEjDV+t&Q1jFR(F*v6Hn61TU4+{ zrP5+tU9vyrFb%O=&5n?I z?k2rpkIFmaz*a;nY6_;eKzZUo{AHk<- z0qgzDy|elg07XMdn}{0_C3Ty_tv*W%mmUSljA0AoYvcOhlrgfA=VOVd`%WmxZu&8n zwzF=j^fJ_Rq*2y7P~pvK&?xV-$!U+;(Dzo>AWZ-`!2au_SPUrskUNp|vRcUQ>Qf3$ zn#rgzFPf1HW?37H#A8(c@_JzV4H{wEzonH9!PkdGI0yoM9l$Tbc#x}v8aA1qnTe#~ zqbjKTsP4rWpw--tcL=~ubo0DE63ZGDPHY}Dygn8G zR`q>49A25x`I6^yW4IWieEGFs7KoMeo z?Z7oG(!9gVkqr}2>#O-W_??V;BCh73I~FoRCr6F=86-PLae#d+7Db9it0#w=ku|`KS&Irv_4F*)SRFJpxsylP`uCns!+(zkR@BsV zkH2?^H(g-zf{gQZxN+xeZH(1-7uPp9!vPNa1ZUf5AcFn3RB|mS&I@*8dhk4%e@OCp zw0^Cx8~VX=D1D?x=nb=z1kz^@!yjv|7yrTiihp!ozaKW44caCNAc}@drto%c*94ZS z-9B^M(?kOeT+p()3)87-cr17g9tr^NF~p57i3sQb2ePcm2G9g%$3!tmW8kJ#+9c%HEMWd;skt@C*uT&}3D&MwXgL9ook`bwwEzgaY>WYZ=t%t-@aI zEjB+om(u65h|bFKy3}Y4UyjP|%hyI*pZB+o-t*6W*o(kI3M?e#Y0)_szCnV!0U((% zpnA;E&dLGQsc>W1=ID55KQwc$i(gbxP%yFuU;4^gNk?G5QB~24ZFEWOzwc&hhARmw zmkww(d{xtaz&AV3fAkH{)hp-e2$8T4@dQeZ@T=_M4Iva#o!(Hh zt1{eIb0rb^2nh0tPqV%{5}j8;EWS*E(Lh$3b+QPZv*ga>je=v){#hwIYaoqEXe*@g z@@k$);EO5FS^d?h`4eiJ?ErtaYnTl-ED@ZR8K2klvtA7)Oddd+j8rv9X4CEowekS+ z(>be7i;e^R(Z^6x)5LUCn9$yjMBVQrU!RXRS(%N)QFEN#B+TD7(qZVGWKwEeC;YYY zIv%T2#%!TzD}wYO$_N@s2aVhkZ=s}TxAztwEduU=-AM(s!3=}viIp+LKLT@q!@Zxo zUR4FMs>S)yZaNEjt+_CMJ%t)o{bceqwp2N7*xPIZDf_s8eO`a><$ltBE5ex?Bh9P| z5;IUOvI)&GH(e$QLbYUAyH<4(x|&V|R#IST6{3zbP8b&?6KZ~m8<1?aY*RJE2P9`b z=1Y5a#oXRUrP%ye0V{r~OdeqYpu zDroC)0a;X%#dV2;50SwX%{=XM-WFy-n*eR+UFrzLD5sp@Zhy^uKlp+rB&F7$h7Yjq zqQjkbz^r;2Ls&$no&^w3i{uqrb$%EV#*mkBtA;@S!Go@Mys~UlC?#0Fj}sU*Pn_v0 ziNn@0j$zhQa4i&$B^DdkhBwppuY`_+DDNZwI$x+9!;zi@(sa`Gc7A#6P38G^LBh(C z+29cO_&m7#2yln_Tw!w1y3 zeF|aa4cVx>#A~S?0sx|2$4vsvm8A8j7{bhZeEML%%dih!$eMfRtFk!Tv>Yj*YAn1&y2j6_E{^ zb6Zxey3T1jb4+@HU$}pM^;6cs0Au(%?d>|jZnp7H8IZ=(aCdCVpGjQt$0E25|5oY6 zDr?}`nsBp0W6FCx(Ycc>sw9Bx4{@cX*Aaj)wt#CIjJ&`Y1L$s56dCEjWmMbAHPPpi zHGc}FT?BHirNv4oU?=Nj^bLI6-^Z^onJ>;cO(+OJB-k=DjZSo_CiSYaTw@r)7yNbHk zq_Zls>3Bb7%F4>6FnE+}ByR_&!g#)~CRsX?Uze(X+%s5qw~-`PS7(TE8NU^oYh%AX zzNe7w4Y<6O#ll&v01qx;xtC&qMvw$ldTH#3VQxv&zDLg3ME)K|R?PVYfU{3|p ztyc=Jj~2ykLLLYgd5B&_+#KLladd5TxdGj0D4h$Bi}$6)p#`)YkPOubxg`7nv^*c6 z9;Bh4sl7PKgW}1+_G&7MEl?(>wJT)3mZKD#^tg_-%t2}gS-SWS2D5+5Hrj}Zx0%AG1zB~I>@fitI_k5QXzh9Y{bR>$;&hwI01ZNNZ6 zH&qS75ZxYLkiuIsDJ%xv#^zniW<5agOA|~1s+jaB>pjL-G_N)AV(+5-^E01e(t!ll zHN_Kh+dIze&)mEwj_Tii7tj~-V7h{@VdgD4AXux1L4fO=3LUp^Kgp(u(qigbDF z8I-N!3r zlz?iG;I7XCcddALbK$w6_hs0vvmJQ zqCw%%E*@kR1jL^-w9R1g%sXfMX8`}N{zg3RAu9;V*Htl;B`++Caf#mc9ox?7r&}wU zwI6P^?ey^AZLBW?8;94M2MYAPNmAbR{4VJBliX^tZQtO3g-jz7eWCKG;=rEmTQx?c;b50eRr&@k1>;veEibCv9 z{x#X^NMp4~%R`IX21z4DSc&u!?`y^L;iJ<7c!w8Px&c<}q0d~KO>9B@UMSa}oF7%VbILlMcLVQ$uo`a+8YXgN7B zPy=x_Msv;l%w)S0T?~M(u%;ubD2mCQ@L+Cry0q(n&w1tcg}b^r_R4c7x>IZ{{76x- z*e#~Tyz3@~8oPgo97#~e321$<{wDXNY^5$cL__1}H}ypWV>?LpNFj9y^2ekufwCme z*xRww`n-j>7c;Py@v;GWpnio!XJc?C{E*xvN<9hYLhp-ah(V+@l}c%bVVcvi3G>y` z(wQvc&6+^PPgtFQEFJX#`n@9m)UHpEsHbI}k);*wxqk#NXHXg+uE7H8*iG_j{3C!K z6Osqc(D1IPJZX)?K9I_(it08w%Y90!ivJ#0fqFRu-@GceRSg1?!b21jHP@7kdN`)h zov~QDL^lGM1t{dMUxQHsEGd8;4hkRhqCIQ3O4kHeTYCj6VG;KAWgH%xBo9-H2(&(F zdcDaaco*_O>sQ4RPDili#U@=2=g_wDoO z&d2a>YA<_k(n3QF!Q@p+JZ`3^YO@b^-K+WvjdxAv$gPN%Yi)(LYYW%uFD;RBLMT;G zGm8WNOp=LKEJ%=O^O`;{EO3QH%*VnOG%AvJ!bS#Aa^vSS@b(Z{>KhUF6=u!!`?3B zi%V_P3B+2P-R0qE%g0uyNSTYCXBdB*EUP9jS0E2U!=D$G-A^r?v`5rKZnjN}#f9QMh-}qIXc8z@lS87` zRci6Zrg_qT-9%I2fO4aN(_E^vs`L7Ea5Hi01s7SXZSf_M=Zwp5!F|*FXbbD0*Pq5c z#>T+NZincnve%_$Ab`Jx{EobIOYk8rJW>-(tk?sHk?`of0VCliFK z*zdx1O~vo8)Tq~RN^G5mW8IKu!BjE9X=1i{87SE~)VJb3@80aOKx@ojqoy~R)%#`j z%@s_J4k}Pq8wx$I^rT!&V**7;>VUb5w{${E;gDZvR}A@4Z1LXO+s`%~cmH3su?2~<7oT>2z8w2<#!$3>#_ zU^qM#no6g4q@eRv;EMZ*hV^G1lj>q-EMT)lhV>L1=ihNPv)$$UZI<&caUR7KyS50C^;foL{*xg zW)O#B(jSHRu!b-jnI2~=QTo=Ndw}h{v4hC?I4-U0;p9K{SQUxH>tjP(zKIa?R@a^3qQ1d#Q>R*Z^Lg z48oNL{vbT$d%4{Zz8T%F#i?*lf!W#N!tzX@KPr9H2Y5|NNzs@hEJs4MNHlVP(1M*0 z;{ECqC%cZO>2fnwr=J8i*Au9h=sgn}ZV|Gaxa?vM=+q+Z&jMFnYI<)TD1o9xsAeAm z4BH#C&M+NWM=|S{h%Fm)XU_vo6{NG0f_V1&TH|;3l z!U8(xQB*nVd0Xl{f?PEjxmu1lv*yJ{A!sgsVrT~oR;*agn7Fl$$Js8b#-6yeQMq)N zeCW))p6Xw6*=n4uuzHfMB! z!pLD}29-lv6UjqeT0!)W5r-W>5X=(a3{g|V%8uFjWb=4lxZEGW{N~BkQo;3k8kfIJ z{kr6OjPOUDtWmX29skE41Nt#f3W&`!uL0Ku1jr0hXu-m#7IKt>8fUQ*-Jj9~ZOF5z zT5cKOeVk@FG@39a?CI35II2gEXx4spGZ0a75m_VZ3?WcfO6~nIFX(DKvPQ+J0C9nIk_%jhsCZTQ$Q(Mk3g`#|oBmo+u@CbP7XffIDSOOhw_O7hEd=H#2gw5FxrI=N zARajtLJY)wYDiwVY7uWNDvb;-_ha$1iw9QUl#lZ z!TtYkm%x@A*VMrHd;0Mq2F0NjcT!T7JQ2PE#jm^@V`4Xd@&9hNa;#~#-b^Xah49zu zfkn9{9hPh{lYK5(>lz&yVqKV;NzKGwK`g8GzL0;jQZDIn&}PdeU(^o(WvxvM5zjqx z{$rB>;?`Mtzmu5QRT2zhp4nqOtI`ZAPvX>WU-!3iLMiqgzV*~v&^kqM!0vWJn4f9br;dFig!7QlK}5$}9oZ7I=(a~G4rAxF zFdkYivsZZdL+n_lG_gN5tq1kAskpuT7#toZN2#6p{;SJ+)ReB}u>k3n_Wm>Unm4B^ zF=?pPWaNZ1VxU)B06w!%*+2jD%z^!{YRH+kSMYEFVc7p36l`G`Ksf} zb%*d)InQpUoul~LNx#yHFB$pzVmdpUMpt^qU6)A(lqsS^MtYJ%nca2FpjY zk$oVyo<029hi1d-p{fdywSy{d2=?ofWJh|Xri5WrDw;&~U9$wK48ePizE zi7gH9hf;R$J;@KxII5B@4ayRWMkOslMvj(+phWWS&&Rs-%w>CG^Cbd!Ix$ifSI3 z0OEihZ#@5eE;G=3_{9x;3PUHC#&*}oL;HpqRw6uIeHw{Hw)WHBVE;Z)c;-BRc?eo< z)#seM%ov^QT5){c2r_a`z`&Tehx5X7W97}!X?FnOH!h{H#PX|?`Kw>xw_{Q=VNPvsEWFe0KBIW8+{hCA z^jA`R2XHLF+(5rNQ>DriTMzlk=vhs++WJBH)UrGe{-xd0{ODk>ytaszm#j>huW?Ne z`ww>Jc+Y$!O=JIvTKaD3;x!d{BEJzYi6D16v!+C0^&Xjgj-^W{7jhcomGk{Xx z1HFd9W&UP!ax~5Kh(2xmXArZ?^3Ms1WzO0Cnj7#gAoMGoRgYRG7y_anPWdoSm7#`A zE?guc!b22~-bz5hA5#@pZ=FD@3JDS2;JH8?sv!$_NqDNT4#sYu z=jP;H1(na&l`e`R)NOZmc5n5_b@y4js`MIC8gZFAeNhFSz(j?CfL!XSOsX{oP8~1r=XtdfvGFA13NoV%Fkk^duRZg9mpldZ-<~J z@afiLcmdE>!yVBOx1Vxtbm*7_idZiQFnhA%n8TpfESiq`LJ(vFD6`HiW4XnH-pA+I$@rd@LKcCG5pW+Fo z$eoLjSgi7b_5L85Xy9?zPP#DYCPZy2am^p7g8 z_fJmV@04758YuJ&I7V)0Mu5)fNW0tDxSSPcb$o}&3)C9B1)f%qhm7&m#N|UAw=Skj z`5x7vos!l1jC@w7B6v;>tcjYVd_zgAt2ml0gGC>0$cGl@jh&Mwm|XC=9FU3}CHW^t z7Fa2<38SflY_&%5oHE+S4qp5EBEvRoTi+2a#97|NyXp|RaZXqfr3#AA>LThkCVN+WyE$Q%$Q^FR0EA@(iFujLfE53CoM{DcGAejpi8dscV5%W z7_NX}Kj7y$>PW%xZs~goTEF%-6dH^vS3*2fb9N-=Pi@7!@M9Xbc7|=$XRdlYef1Kl zwy$VCBKT%aJEL{_vl6l0?HgK24ZM}1D=&eaq`WmX=9qsRI7_R0ou`!xsJ zyJ9Kyzt)tfUztxUNSIME56%9L`v{%!53ODv2q}~+SBQ@n1sehes=!38Wkk%|?}q3j zo|haY^lf+$i_FytP>o;HQ99#kG(dsV(13(+Pc&0zpeD#_FJTG7fft#1DGi#VDyN{M zdZ7>lCP50!8jvIeAy-pfBxwNgRKajsQobrgzuxo-MQO!p-147jr4`I)#HWZk0ts8- zFu*5~cH@9{hy8V?s+RKo>fHak)BUor^EKF6F;(X@`Ac?by+7$%u#EE2(XU1}#^ znStY-9#=Q#^Xfa!yYq;u>b@<9|4u$OA-k6HaZe6oH1VpjSnhx^x2NC{i!v*j73J4% zMHEcHhj}+wLWg~c@PR?KR%pqnv4`j)_k)_k)1P`i9@)AtQMn$GU%8e=>jdG)>M2oR91La4-XO^5ZI24O};;w`QEl$(AjykttQA##CcB<`E_3i#di-Z;Y@4 z_A<`Q-0gQ=j+C%-9w1kf_;tX5GJxH{YBc3PH4bd^Dbh&w(dm^9NCjRGgusZEZza`S zfV~?FIZ^x)x2)V>M;Cs&IJA`@)g&N+;4K)*LZfVp%3gLOhcP*#fGs@nSL9?^2yqh9 z(59jZCp07S<$94DC{iJ{enRi?+vefahAXa5o7eZ}!-MJdes*Yr_y|@>zz8AvbG;y_ z;{1K;(umi{ZI(2xqhIdS*aO4#PnrDzA~62VjLy#;BiukhNRDX>G1!n9SKjyL?-q&p z$^(^JZM9%^m?*eD_?41vPnpu9xIB|IGxJ7igNg7DklkXgG&v9aw zk9>S`R5mgdSaZ#4|D)*|81rhnZIi})V%xUu#G$40 zaL#D2S$oZ%H3|~sF}q;yLu zkM{%)H+=?e_6}Mr7NL!m8E)uBB33l-b*nQVuLISi_ABgMv^%UXe6*h@`}tkEE4-}_ zyh%UK6MCzK-3-Pjhqll_kC(vQQ6xAj?xHN&`F%2SdT_cuW5`pMxsP%*fT}L zdQ^~=5W2z}!~;)hEmQ0(a~g3aW|YD?=VkX@@Bv!~}8TC3fQt9Gk}z z7378r4}>jWD|ug12mQ$#{Bo%>@+)UN@dmYUV~ZOpdlMRFEZ#N4)~-Ets>{g>JjZ%9_JY%J@}+_~3w=x!@oM*?8)mWX0Mi zg4`L|VgnbiPF4JQxXPA^d{dQwcp~D}{axn_`3rwva$Ho8HrNk7B&Hq+osGdrwfTYmesL`ESy=t^>4+PU79=BlibT8=hK=L(DOWw!TK9X z6rM7tP!r*xi#T-{wMo&_t$`)oLz_dbks*k9&IvL5-(dos0EFoS>w+1<`` zZA_r)bydDwoE11*%S^v;jxGUqDuW(en(_1+*?fZ#eAnD0=R>O!${2>Y2HiY4y3x~q z*)hHd5YuAQm}QO8#>=&-4ggRarVPxVM?VhXNMax9%Sz6i*AI;yT7BLJj(2x&HxP-f zv$G@AZLHbV!IKYmIZw&HXXftEBG8Q2)Bbe?RH$oI(K=tAMUi&ouUlMMqCmhl97(T5 zX|M^jtZJI&$7Ln{7Tu_;SFK-W3x6T`UiPyeHD7uU;jMaRg&Tp}aO(HzYI+>Y2k%QX7Hkc9R+Keftp$eo>glH$nouz+@giZ0@S*21GP zeMv~?*HkaNoq5W^(92aA0sZDjbb|Q%Ko7bE#nZ2wYesQ?-7w&tDKc)}VcZcDW-5lA zzJR#0Sk{PLJ$5WbRynuZ)c7Y%7+k%?f>OF^<}oyTkLIVoxHc6D*qfgfNo^~2n{2U&(S?FE{%+xbBwP@ znM_Ek_+yJcP)>~Y6DCe);uR0!^VmM$N^ImV5?w zioX*L$ThlJbDHVZs$5Nzi(c&cIdyE3dY}RceTPz#t^bvv5^13Q@&%s6hn!9ztQU z^M=bU6ZkBFr-mhK;U1O3!+0#%_AVb%5l9&o#Bg13^+YLj%TJKY_*@$9n(80Sh`X3k z(AKo!dv##KSW0J4*{vbfNK9Zt(r91gqUnhb4FUp^!KVFeoEsOwejFfH9jV0%^7W>< z?4l9Z6fH_cPvjkID+~_Cwc$r0ao5EKKu&TB&~3f)d%l+_a>EI;QbH&?YfN4jrA;PFE0drEr zq$5ymLOhtb^HQ_;JJdaBM#`zPbL@~K*Fi#5yuX;)U9wi2G1S5F0S*W%60lAkf>&l#1zu!4hU?*zWOm(32waF;reI8kMJ8m}h||A>mzk}XRF+7i*BX!98W!Dq5B5ydviN(0s5S@cGV(9yiZAiD zRu>8wp^RcLdk>~5{4x)1`5?*8N*IRXqNt>Xmk0lW*7`2M93AyZGI|ZZ^EZh~_QWv3 z)YPV)uX{=y!`MTQ$VY+roQ~X^8O*>i1K}8k`&e`Ec_2{8wD~!m5JT2Z4U-jbG*Feg z2-{ByH8Lc#%*}$X((Usa5{kr4n(gQMY>Y}yL`{a&ZM>V_7COP$Ia+v?$WLDH;7tPE z57FS&zvarx)S$8@#sl2#-99V*+`hEa%n@A<0gg+Z+%AVAUBUVI!R)01ufe>)VO1iT z_%WVU`JXP&Urui;{ZI!DGd=LTA3dp%*CX@~e-EW;Omlj1aJ_lBzE@-VT^3{Laov~+ z?4bg7SQmL~sba!hulh!>MJyH-5ia@4f;*kMD+hOw8%UC(MX9`hu@Ezxy;Bjj9GRy5 zdLbJ5rt+k{h(-{+zz`efz^SRWX+cg;+{JiTA4EAW*Zq7px@~ZK80IGMDtngoVx!15 z0k5$(hrO&W5uB)_H#7(|hrx&V1led`$EHqlPn9?`WL4~_sNp6PMcv3dXX4BfH9Y8$I2~_}2PwT3s6dQ?^RnS;N)F3X> zXT6-YY7@iuvBK!{?FUcuZ2Yj=`b| zE^!}SWOj|D?|W(9*e-HRwcaU7cvxK9pDCLnz=L2Hsu=7lj=o!@gL-p75GOMy1AOpP zj3&V(l?|6Wyo>OATn|ia4!_6KIIviJci_nPgYqvPxFmN5yQv>0UY7V<$^5Ka3;}uS zZ&;DR#ecriad4m1QZww_%gLHGKIKkNNW_Z`wE_}oUTAe{LrCbFjibTnIq87HYN#zz zMk3(*(DBjb`OqkAweyOLxOH~Ao{{C@yCN-S3CA(~isy+P5Vi_YzsxN$^tFuTW2fx2 zWw1EwRB_MPEIr?_l5|nGz*^oj#h@aH_T)w55SHZL%Lu+^Kki#$5IZmZ*BYn$QS&|VYInbJydmgEi9K9*|u?lkDOiI zP<&D47bbC$T5go1w`}X-ig9KN$_9Mv@u)fmjACe4yTM!dba*J&I?n(INUU*R9+F~8 zX=&bAkfgt%-X~af%fFU|!U<>aA#Q5vNr<1|13Ja1+U;Xb}Djk%gh?2B7HLnZ2^Ne|(nzfJI0gFu&TCL?ehjc?`?X%I1(Lvk=~E z1&{Cs7)I1pRe+BB*nF5cb9si2Ij7fk0GN2pxC7nAMIH>CTgic5v#}fXE<_V%4Ew8|+XAz1r}IN6H6# zJ~Nm_2Mej!cusA8b|h<+9SM!(m@>brLapm}9jJC^Tg#yC#;AGu)vJwCoR&)PEZDUt z5r&($?9>zZ<=vbxlk*Zf_MoGTnAWJuTMkIi@dfe23TSqEn?^UXoDPrp0_qB@)>-;W ze1ku!J0e!#Ft2TS>L+5d{&Uvx>;@=5g~p0zXc9fup0bi4n#lS6){oH}eDt4D!;PEv zs$e>C!Gi^ja-EGvQeNZ3LWOiA#k6zs=>)=V47&pA2h|!&?~tF&oEg6j8U+z>=qXje ze=pP&r0*`o7{TFkc_T}!y7i*!m=>i{x2CT^f>Nic!$2TP^!W}io^U76rjFG{Df;6p zL^O0l5_N2lg0%gTlq$GnAEqHn5T=sdw<=2Vf{k_M0X#$6p>SE}Ha35(f)GVfy`2<| zT4Okp+_U&uNYn1LYe}y3Nm3uyU^y=&tf=s|Lcp7eq5D@^$f*T|rKgJKUkxE57fsys za<}Bdx75z~2KiOi5>Ry*Hl#LgEa<6FSHXKrY9W76GEHJc@Pun=&^KnGtc3z?Q%o! zp}@fkr;epM-YKa$#y%!+HEJz{Z@ww~Rfk=A6Hxa{N`iiB!p|lBq;UT|=CuRYriH^7 zt+uVfOlsg6;z4~mqOV5It$?Q$2aI-aK1U!YBdPxG!mPc6-wpY=;~Yf28z@^D_$hN+ z<3oY=L!xc``BtgYN%D~X9kNV+vzj-R@NGf^5@&%$duJxw8{QR$#iI=NatBz7A z;y!H1$4ew_8=|yh8_E6z0@wV;{DzB(sadA)_P+e&IubsJMVql*x7Nm%!D<_o*FbW-Z zXa1qV((wKZMW~~&F}ehC)m$7 zAOAZXV!M73hmyeH8D%aF&m;*DnsM;G{rmSXPhjjGB^}y?M2yh_=T-(mTX=eKFS@5#o45<^_kZRrkTuHczmT6$|FvDPJT#wPtpP(EB08hEJ- z?gqwT?S^MsdI@klj%(2#_fCz;d|rqUJhdIBrvRd>)UN6-%tWW4Um-(L_3xxAWD)) zS~dg|rE=DaSen>hvl8W5Kay~jCHybWqz%gv{CBzN?Q4=}-pMA? zEB~6yr0RNZY~0RR{a0YkwO{qgAETDSFuD!9GT8=H;Cv(^0p}x8Lc&z}^L?rsz*BXk zYvU8lo-(}j*O0B&H&@rKl7!u6JO=+3?w56NSXL}KuAe&@HnqXAIz0ecnPI0Pq$X#z zKh`es>hfHU*#a*f_lV@*n+qW>?r3QMMTM7U+e?ZK=sylhBxB?oB!d{~y+l}Wi$`h@ zru7IiyRou6PYE)~x;Dv}6#6YtmpkE#Jp^&j;Z~1UZ%L$GpgJmQQG~gKxgP_WHBwUDU*I@jcF*wmStZS>xF!ptNl|VyL>1j*~#H;drqtw?geR1DMZ6s@E>@+2M%gpkJd& z=4)`HFn^Gilu2A;AXg&gI~bu3Z)vhRK9R@l1tCkbtkcfUkV+K0Vc#PTw0<1c_%7k# z!Xe7-_8~FplZUirqOXXl77NRsoo=fLlFjyx?e;S^UMAupN**c=LE98z-OWXX!V2|3}{Db4GclT}f2O6#myL+iL< zeX}3KlEIU1j1#T>F$9&QU@D9OZq>;X7eN!@aq@DgCCpr(Ea8SR7|js!+@>) zuAtI_19Ds-%-GQ8HYXn3$-|sWiTf5lS-P`2T}zR(G!WO=F#_bk+uK0tN2iiZsm$fw z>@ecCz;lBxE^YH5PhEb`|33d(?mb2<1j=LwU1jwP+1*h1;2iZxb@RKCr_>Jms^iU* znvmh@%YutjM^_HL3ULNY6lRNF3x|NT@cDXDl!s0OXU=9+sDG4+jpI67jU74d;?ZK( zfeUgG*?M~%t0d7`0->+l{M0;|2Xn4iGNzK}vb^j& zS(-fujUZh$;;2P(WX{qM8o6dQ`|<@Qs_l{zL-D8C6Bs;`Nni|lW!ktV1_uc>I)jmx z^^5-wdqSM}JsOZZ8+V?Nsy(&w1NB zE>bK?UX+G9iEy?KM2((R!Dp!j8?&SRCd|GHAK!<`Cr*vwpk`Niq^I>&Ct^}KCo9f} z1yvqZ)yhU1ED&8Uh&p~YV&ue>weQ8RCuFJ}$oaV2diV2t9wF_NEI-*}?Zn+OF-*@@ zjboUH(%^Q#;(5Hz;(E>I-N2d@WWf0T=fp8S?36{-yROK=?wUYplzJR$s|_ylg)u2Z zmplREx4sfOJ#B@x@5PB{;U;qq|M*god4@n7Ztq&f1PS88Zlpgj04PBtdq)7R$xxfE zWDmv*rg=HJel!ir&^g?H0PBVWwib@cS5 zd6N!jhiDMwK71j7-X3CtNE+b4AgloutNPrifiSv_4;qr1Ig8?P2de56`_wscT|ZH* zD(4h-xl{DD!y-o#J}xtDVTV)#v6*tx%$_p&Q2(ui6Yo9()%}t9{L}J{5x;lc zJ7mi-oSB~2fvbjv(FW!C2ges=;a4nV21pY&5+!$=Gz|NJg3XnT(cceX&=R~xE#af- zq~a}D(E3riH1&X%g-T7}DAf{!sRZaD{m_c}Z@_fHl%o4v<1R-X_WLy@V`V+X`0|d42pUkBc-g}eH4qHl++H_leWQQGB8#vXN&+H0gCR3j4CO9= z^0ubwMTJ3dQw{6lYSlg>CjyIrEQGta(2ip%?&submWP@B-jHDY_c$ zDEwZ-65Mwl#R}GWlp8w>hXV@dDXhW!EzmyOs`lAdnog~@e{Fq<@}I0Jx}l76X8{A5 z!XmAlT+VU8qL=2Glno7oXlI3UR~!?gMP*)WDo$Hr?f37stsdv7rXOmr85NWO zA*8Z!$l2K?bdKZG`k$CvwSOZfamBO%7TRr(MZ(8dz@}Z2LY0EPkuqA5{I8t^phbL? zS1<7Mosm#eVdh^QWZ*n+`fw8O=YkS9MSlg}o1!b$%T=Cf zF7wE+3CRvy&>_hUTGDv4nH(uARxs(SzoJaC4NqbwtuD+X)DCfY#83r~k&P4$iWdud z4fk6^y4oio#8RXP?0 zID~O;{Sajv|nuI@RMy6yb-u%Cmi}-8n_ZBZRS`XTmK`(0`y#^q0YvA%SzobQG1yr&ILhNVl^7nx+SU_57pVeX;uU-BUPT`s>0`#V@ggxaH~DMA`GzwiFs zb~My1MAA1Kq{Rgb3T*i>?1N!PDKyZ}KMv+H8{mF$VP~1Y2zA0gC_*uTro)ArDKK|#SzAFS6Uo8|4o zuqHs5mtxH9JQEAF4u5*ewJ<1#1cL*v+RBkWG>A*d$mhOH)M}Vowjcj&dB9u%s^hM6 zprP_mfBuSm#AZUfJ7XNG`C$po*mcfxj|S9ilrf{0wYG7Dquv0SzS}-J#qC1udHmz> zm7!;P@0Kxre}xk!QSfx(JC-38Mm+A+E2L9dhw{6`rhfa>LwQ9{bax~DR`esCMy3#P z>1(fZK57cN&g2{GecB`kj&7NBcJ;OH6LZl9-k_fs(<6in1RPwOvuHO%HHjMK{>qUj zZJVtgOM`}eJ8^UJA((aoCjZMwQMJcQt5$ZE^U~|2fdh7l=FY-#oVh`uqD{4Xo&J+EsRJ?}O`v`KvqU!oc~1t^mEM$ZgG zvd{9JZr}+2wyW=o33U8k(ZX4so7n8D`$Mo8%#ZqEBPfc2*4#mN78_Xw`~_hyOrzY_ ze*udc07>M(QrDF^C>&gjAP}RXagdOxF7R^F^R)JH5cYmDhCsdpaD`KrsrzDP+EbeN zmG;31%K4uCBiqz$zzVMY?9AGo-m{Jae>%F!=zYI04%98@LlV6Xn7_QBU&%xEy|C)M3vUoDw`2B=Fnt)>1>&FQ+_gyJik--)W3 zhLmu3fKp{c*h81%R|CKxq<^M)IG<@Aj)bhK+CRF21t{f7p4_)D67)tA^6|H_(;fD} z-&lC+_$5deB)kD@vrH1|hIw#F(HpPeZrnhUWpL2VNc~OagdY42tNmG1*2v|MQxoft zg&=PXbVpmnw4A;lCvTSr!IG#{a10si{3vQ<>a?W<;%BaW^a=Y%1YeUrdx;uQCD=<7 z{vbBF^SxOScSMYxs^U98)k&St!+-!@3YQgZgspChYv2F_h>{Fnd4XXd#idM`>jcge z)LCZUg?jf_u1CjP1O%_3zVPD+UQ@xoKF*9 zhBx2DnJ0=b-tC_T))MtQab^EXeand zYH6r#>A7&f2Xozi5~hz!!)pGf!&n=aujt~VhbLO%r#-xtRM2yg=HSXFn~><}p~K%q zT?U&gaeYBQwJgTTU7R-|h!9XbgK@v%rd;s$ysr9Tq^)1SvCzW_FmbFRJYt`>3G0NX zG~w?^+{{8e|7oN*4f#jf#9^emx|ji7jK-ZCjNhf5KZ|rV;Xl?ZPO|;mw1=~LHhU#& z=;JajRz&anXLF6x(%ZIA?}Lv*%P+HvNq8a=OpH7iUGS!N6}rIfYQ zI;r$eN4WbI&rNgHQb-^s^xhEX!rb2M)>P5_0|+)KTI$KDvBu@Ar%9@r*CTk?WGA)J z82(oUdAnz|iObHUgz#wk748OF%AXt{`D*H-!LYl;0O~kh*-m|_xY)g&??`u!*`%-1 z6}o3t_tHqfH)DV+I7n!bp8DSpV8B`pY5&aBOHYDSwt>kPdNGA0*oTO{a_x%g*Ok55iPGK0U$%{ z5-_gfQ~iVi13IV$kavuH!KfT@-8DiJ4_O7%rzf|kgSFPN0qT@c5p858(-K{__RGmA zZW%;K3jMTA2embwOvg36@D9rv@6V8<{A_UPj)LJo7+cl>yEHI+%GH-xt|b4Nq)`bn zR1`GiblHOB3;EZkXGJ_6)##}ZI-l~`|k{EVH zIj0Mly}W_ZdllDw&L%G%1pxiC|bPPD4E#~KH@RMDPPSkbn+ddH|bgN>21D% zAni;IxVvAsyFK=ULF>^V{0LjhBl}b;8$OlFhGCH+#eZXEjAUUwE$q-@5Q$#3LN-#d zQ#z<4u3|c5qJ)-@Rlwl*%bJQvZ-zd-MIbU=p+J2|^Uuk8sa->W{I$CEbxdCu!yFTtqzDP|^JFiF zLd0mQ=~G`;wV8)O7*Oa9A#+kFtr%GD59Xu3E_5L2Cd);8v31&>6Ibt9&kQ-iVtNWr zvNuh{h$w7jlUARJ!J}+)K|RtLB{kXReZ2QZjZ}8%5u|BuH3!805Gfx_AD@+de-ps> z{m|cfy}v`8fA!YEMbL-Pj7LwqmB8MHi+b&ZB$!^1mW}qjp{sJ(tz>yR#NQNFe_{@n z(}Sw|lbVQm8?4Juxm#W-A(p)C`?1eK5ZXgeF;9*86$>Rim6}(0hWm^CDD42D(PCrq z+K%np(@xqA%F#IzlF;ym6Vma^3I5jznI!jpsIc}6PTf73dd@$O^GeiL=CL1UEqlS8 zYwPA0E?JyR2*RLzPAd$ybgA$h5$xALcd8Q!N7VvLmf2L&n8FI#p*-IS>*o|b;!W3SHTOwc3n#~?I} z`8TK=6Gl`LX8^rL4$q9(Ss-z(+b0OPiB#kLFNA(W88|qz8@B)eJVm6u86?WG5=$5a zW23Uup~(96#QUVl?E0G_vO4Q}O&T~#`nqV}(~V#cY{oAyEBzeBCx!!Cl>bo-t51qy z6($F+{x4n%0QMJyWPnx&0#NroxP*=9sV%vhtr5fc?LRtFsPQog&(OipW5f~mJzyGZto}6 zy-g;nkbUx*u+Kd}B!15Eze&J@a_^`&|9;8ch?!wn@Xg_tZg}tHVQ8P+5d@Z|PL)Tb zX{)tiL-8%yT-#GlBaIQpnwwsB9S7K9W{38J4W%ozfG%-h`?~Ob&pQ9Lk4D7caHvi} zUsarOX6KO?OLd6V9~#l9^Q}Q}q}Fp0d?Mm_#vU2MVU++M!c}~!)j!1+I7q#Vgt`10 z@~kSl57G4e{h#@U{6dV^q`Su}j%H8wCI*qEopLfW>6S8k`m2jIFJQVj6=MWiLj>Rt zGAV6p4#8%~5u$httE}8pn>Ystui6^63Vl0hiZ+0ttLCAo^(~KQ)zbbZf#aGZ@Cj75 zj@(a#Js9IrakNaco5vY_%nT}0Qo&K6K3!0F3kiyz*!@~GHTr6xih3eIMVh)u2^xkX zj~>Q-m>VohKDubJD;O8Shi2^u)p#(G5!7psFXfYXl`{{0LBV^G=TS$k#{T>%VT3_T zNF)XyEkVp2n0mM?b1N-N!EwO1a(7LQaz(f zW}<^iI-IjN(T}1<_-F9<#ZU_##9mFEU;>iiD@&F7@Ez85KWHKE)69jAdG|_QPr8BJcH$ z9{8k-P5p7Npr8{%VsV zqVdz&C+^8>EbfgRk-ci43s(&x1LiZYi|jolA*>Uw{V;~i+>EaAWqwL6jcE$rT(1cp zf^E{2Ios_OQt9Cge;G(jA(w*D=_k82W~FnCaKbm>+F@ut{ggO@2Bea3Mljc7iQKJpdk^AfI;^F^~66z4|`E9 zLQK%JC=axGxDEa@PnHY2O!^XH%O^RVv5*qOq%NtU&0!zBh}e(WDNoMR+q37?pxU43 zLDRg}Rfp!xIhwRKdjHZ92JtVj&WcDg9dZN29iAM|m%3@XBRiX#8dm|$v?@bxA}S_} z4cLZ3_?fu4J**Br2%31ps;y6`9Y_rc=8L4;0Y8LTJQv%uG0)Bz_Zyc*9GJ^| zSZz3*YITYk&26Xqx=Xeo^u3abHnf<~tjxznc?ktd|0HrqWa=fp=}~CN@>PP2q8wv; zN_I){q z5DXT!kCEidAmh0di2=Py%fcwf*K@bT5!Ew6v4OP7e>Axrb2v`$8<*o3Jy)=2uaX=$naN`wivf2dHDe-ZnL>q$gng6>p#} zOI?ZECr<&a--3zja%HU~j6&;*VlX*r-07siy0LxHK@c_aQW3Prk1Seyjnbk{qGSojFs%!&CE?x!GV*}=6!!1l?Z|o znwWrG`S;Be0y(EnrP`$FPT5!Ryf#{AT@JP1Ma6kVFm3!SU@{i`GF)|!PIC?s0llkP zd%+1u3x?jo55X*^e;%J~)!#PWEJvL5wo2Z|!6oMpiD%ddx~&&DT6(C{{zYnJFAE7F zeUIFp#2=tD>5~U{H}YAhc#hC_`@aYHR*02&r#;?50oj=RikYaq;|13)ym)*!8n%)* z53?OP4P?5COdNQ?)ym-$tiu1l)ruNWPTpNC=0#qoXtI}a^HAIxVX3EKScoy;I?ra_ zVsn3Q1azU}5cp6AS*_aFsdn-SeUWA{6zT-cTAtZYWU`)Fx}ZHB*$of8kDd?5%=h?Bk5A&2TAxw?w*HO^rv*ScPXN(#kPgEPXR4H$m-*bTh`tI7~KCiXDX;1rl zw)6fF8dbG>1`S$?P*o9#B2`&_z$$J$C4sT^K~yf?$*GX7nMx_uD82KpnM*;-5RE9O z=WCS^L}evlBULH9&UyIb_+te4@HXFgppa~2@9Jl+Vl!hov=-#Ql#0>QBH@~(f&@&) z?B^&WK_N_t1g36Snd?TUdp=lJSqSO`dE)e&zj+@#rrn=A`UBYj5xEDm zCbj}m&*r|#tG$uI<~vVZm}f!dz`#6)BsZffUORY7`IFC+0Xrr;j=o;WVlEUdQ7-R{ zwEywBCBWYBJBcv(uh5;nGA_f-%oov38*|P0tdHmAaeZ~Zo9E<@o4}QuJ9fyqd{*iU zMcss@iR%Q-*8;^Fo=s@2B{~%lt_2^jf^>WLgJ9XxWq-^Jc&XZu7YcC>sz0xSE5*EP z$d#3Js^fJOW;Ns;kN#_a-KTgVy4glyPu83}2GA|O{yZYhg#tha;@vU+_I~?mGT_$` za1OCQwE!(tsb4l+Lms%HvF_wyl`UTBD55(qmG%E#03Lm1g!OC?f+;4IShs7C<(hBr zONTsAEXB^MBQxO*j}_&Ff(${7HK~ukqpcP((6JOqk8aHMJ}QenT$LRX^?6b>sN_ma z-X9}4F|5man&9*2NQ)#Zwz**{NMXjXV;jGZct94qXevO%e{t1)BwtR1cuZXUQMB^6 zSAPmIZ?HHFU$A?HzF5_W_}j@fO!u9$poyKZhjyaRL)^!X`NzHASicD}LO$VW@ruDR z!4~QpJj)+v?4gwDktcULxb0mf!R75EGQpW?GOnewm^XkdLiZE5`^{MA#S+10*5GJi zC)3XwV`UWM5W6_)cy+Txr@Z*-brZEl{84KXeK~eNv2+Y0ZdCB#dv=;@y>yv076%52 zJ^eJ%o8<1(9s!BqDO@=xIcoCeOac5$h6=5?cVsa9ldPupvUf5izw2fA4jmsQJ-UB~ zxD+7Zf9ji_^ofe;NE?J5N0O?vRAm z#LuGPM)&PgBF+dDbV|036r={S^MCBdmdM{S+GmyOmmFTF?}IK|m!Yg-ygU<r zSLou#-@6holo6)lLrKM)t|zP$sH?C;58}vLZsQ!=^Qm$^KWs_5JSbj1&`GN2TUo29 z|H!_Vzkg~@76P5mP@z`(!&b`Dg-uq7DwWQxVK6AX@h!fALrWA=H(Q^q;^)~UW;Uu2 z0|mAxq2kPtlL&G*`gsExL(_$;`~zx$?kDHh#n(N{_}Aj;$S-mq>!yqX4V%MTwzVbb zwL)=Q9{@DU$tisy?>gCfzwUC*nFd@Vom6}GMS+)gIL}~G(qGZaxf?KKw})S{v|?ta zxucpK#B$+*fV`$K%ljW;Od{HLv2g3?d&S)sVrzfBns#xWZ)v8LUNo#9o?{ZJsT{2; zE%i?$m19Pc4cxz-WQaM-by*IXb-_;v^#c?M?fO6V&B| zwd4|#MJ6v`^Cp^IHb zJy|>L4Cl0yVg-1GqCn>AYbbl7t3YO4<{HX_EdJWB&|eOND`ANN{!*>oMdbFq(<+g< zFy^fug4i1Q`BDiA7FH+UQA7Oo6&X(y1i;|6Xqy)QRg3Yat-wYFr_c#4lDGv9d6P+*@8p}2XfoaEj zS6lIe7YpQZd#uwX{t~yR`4iT-(v&E1^6=lWN1pAzBaJgQOe3+uVz4`(?Iuq6@jCTU z|Dat-@t~VwkE?n3kkaS-`X0lR zvHFJQsv#4QO8Oo*Fq3h$NW1H%#{t7`rMpQe;Z39ckQ#H<^cj8OBTCS;@Ei=%(z;>=kD#T!F zSS(#X8N5?zNI$;acRN)tyX8hdg4e{-!QHXw@C_r|rvJiN4})L(UFLCl-j*Ot@1^n+ zLV#y$9#FZ~qc;P(LjzQ~zmK2mCx)TRjrf}tNvD^*k_E|wy+tiJ++8CQ`(|zthgskm ztOcELGF%#bPe}%|;VWI;7y!$aPL2$OOLG4cF$kboXCG%!aoDhvuBo+iWe zRFdW$EDLxsSnhqvbh7E9T|`Uw&s6N(U-SK*ueno>;J1klX|q)w@Bh&eTwq&R^0hE5?8>EE?<1pl87jP(9cU<(nY!|LSS(_OBo$e z0S_;UM*e4KWqWAJSiUcyCnjhtFN6lXc7ttCVCPg&F2|x_#l8Q&iAu}>vWx=f?*Qij?LH(r&(fq+JJF z*T7~J2N0u{I&3aCtlgDwZ-|5MPr((~o;@IiwYR`NiC(ax0@nY8V<9kXSm(W-08KU{ zS)E_c0*fw5M*{CPOx2N|94Lnd|!YLXg(9EV0BGMD>E6g&U<~5VESTurl9IzHf zmFD0?BfsirLPT<$8)h@UyEwc^hRh732EdDKugLv^9}pV)r{e-^clbal{he6KIf|#! zOQS07CMV~x>CInxi3VnrFTBsIZvUWmF}!ydzW7QIP=er2Q$gLF*Vg`Pgae5;_C=x( z8!FThJ+uuPXdnr#{_z-y})Hoe{ZYioh;tKmK88b@ER_>VLPC)@FY_W7Bg*i>T5jc z&*QR66Lkhmi{!MfbUY%&rY~-suVPoP)2Kh0s)?RQ%`EKhr4Bl<(=jm#X(SbpEeEq- zdCJss9e7|C+Y24dD`GxGS#tB7toKFf>r&+QfG{NAuIV0JqGdqD9GZW`YoGrE}!LjXdK&}5VLiV(;dOGf`q zVXsv1njg+PYOkiV(#^YZE;WZp>5wF7NFE@i+)N8=5O20WHc`%!ioJYE@THI`;u5kA z{CC@5y9|xWt)>${WROyJNoyg3zsTI-xs?$yVU;%`wRqBXA?>z9=(7Sz#ZJ%0GpTlw z6wW&Mg7CNl6l0JGp+asGAojJyzn6@Szy(BrQq_iQDbEAE<9|CVZnZ2)S-j6PPKXUS~o+KHt z_2q}f+})qs$A*8qtCX~;L+O`%xboxgo(WkN|7*377)9V(x4j6UPz8Ma|UKow2diPQd*t%a*TkqSw_yy@wT*UCB%i zG%F;_#vaw|j!9)MRzAY!Hz1 zf~r4QqED6NU6=0)%^5LfFAbKzgPPGnj)eUY8-LhESJV4FcN3_5|2BB1xw;z5Cujdk zw9gg55*U8}7{8+qAb0r4b$d>Vcut#10^~E5hY!* zZYv*L87sicX|H*R80EA;knL+Tm)NhNm?z>pa-uu7&ZqJKoQ`cSAVNNK%`P>g1waz} zW6!4JyFWHyh`2Kh?1o+-J;|CCu=92FQunqa+4D~)9mlYDK3%eRaLa2B9OLINZrmO& zOebizM;}3a43X%-T_brqee7GY@G>cm)zrVLz}>47DXO!6&Py?{4CA&qI(RG(>XaQc zCVAcZ@tyuGq0LTf!8Xpz>Oz!(G+K&Xj7~<$ihfgHzq3hI{N3npI#HF2tucABd!sq> zI+hSvTgVJ3lHw44Ciz>zuj1>kOt^WML<+`Dadl?R4jQV5d_q7<%7eua62K08|S35(0ocFTilZi6#1nu(r-G zGaNT1?|9|kk$^E^Mzsrv1y!H=+fu`fXHA>jhS1&>!Sl8S=LvV<8D>jYExPTBqZT-L zJgC&N1E1=j6xZe(cMAFOn)Ng-M-Muf9cw|(uwRRHRdhqtn`H@9x8-t4PPbi-mFr`d zBGw~VrnybN6XN!dy8{+*xYLFTPQzEl>{EN~cIB=6F*dpyi6Mpp;R($2{qmO^&#|l< zPbPjo0mT|l1{IAbR5oFiarPzgcSQ( z1c5*f0rxkr;t=zANh^&o(?B)!Ym-~ll}VC$4sxm}9gVQbg=D^KSOS(OCdOa>`S*sD z&IPCX_0q)|3^JG!wD70IKiuD@_N~SDQDTGwqs3Ka!UjxOyQ66*I*v6A7H(U(`V9TW zeZ8YZb{#GQP$q{a%mTD1h0a%%Ecb|~cbDa04?80a8nXMpo|Tql^@3)eL~UI*&OBKC zO;?|KFwhICF?YSB41(svTIH2)$Z$~jc=~t}@B>!m(6c30FiSm~Uk1oCY{CB@QQzQ@ z`SyKXO`bSuvN741Y-_S@+qN~?ZgM@@ZnACL_V4N5`+eVk;5p~i=d82#+H3Fa?zQ4^ z)w@@RPxqG>d*$WV>OL3h`U>%U>)a;5ktd0NpfDstK`ZjgE|`)tRJW>(k(29X?Be}3 z~m*U_aa?a84yRI?mhJ5_R`DZb_R6w2Ha~mC^?zE#!ciwnbB9w?(43De{pM^aIvKElDAk++h_G^w#6yMjpmF{e?gC#ot*({@LMDdE`4~7D9D^(^c>?S zw*%U5z}}eIEvWtrKbXM#Na%jnwjISShV=JEj`O}g7H7e`!3Ro=#pp*=xbiZr!9TOQ z?{aQfHfdRRzpHJJyJ!tU4Hma2sE|dfDSuZ_zY!=}d;XIk0fd~a&)_B-%EiiQ5NWe$ zG0nL=eHm9+R|%TW^7PGHjfDJSCZen?`~CZz8n6SXmT63Hm%WU*uVeYNM%H2IMfbJ{ zz^l_CfiV=u@_m~M1%YwZ)lywq!oUT;*lF+5cOh+lE^EFu^D~u~xl{D(UpqoQw60`7 z+m{4Na|zRMX8p51_SRP(d@Xe`a#a~I%?_kum$17iD`yK%chO4`18dg47-3Ddef&+E#GDaEW|nb^H? zGdh^j%I-+=8Z~a6JInXW@jx%FrORx@Emn;`3E}>f!V)n14m1`BfRls|OU3lfYW6fE zr<(Bq!@^4gMHO^5gsJRYL}dL8np2(Pzc<=!Txc#<6AlW@uOtcO=lGepS?$KSdeJQJ z=r!ZRcTm8ea+Lu7vbRKCJYnzsp$mV(Q+OAEn4XxwjSdQ<{v~zSap?<> zo3LqxG*crUsPadwv9)Iho9%Y(dcQP{1%R5EEqW*Y?^zY_9Jnoqj)k;>DVk-D69{;+b@`&dcx#wN?o@v>*f?^FJzPHK^s^~YzsQtsk z=7cqA@L;D6P@EV4HDWXEV6tY&Sxe7fHC`rRvH&9dQOkqr|NRp^}TqrKv-(keZetUiw8#{{?P zC_8J|e93MTC5~6ri|A^T@iv|DUi5r*ae}u47Uliowd5vGH`hL_wLup*HdEYmM6*(6 zSx{IxxIJ{bM?JtTX3U}>A!F9Ch5}6vr3VqIVVRRWVX_tHV%pU4t3ue^*rtx8B^;MeY3lUj1BY!F0x8Z{A3}g zC#&08vm4K)EbKLj{8l$8zc?4Wr=8<-qIfVSgJi#;`akI~Sg=fTk_X@Ux*z^!IN@Dn zgqyNS!eLl*<_rprzzYKc?%F)=5*3DttH!z*R9VTmY zYC7SqpCfqF{UtB#71kL<_=$9j^3>H`3J2nBo`Ob zW4>n{;A>uOb!DNFF^MyJrov4~&8`SSw4~CvcPQGP%F;c*bUU@%!;bk^ z7&qS6cIwkQH@)j=vH*ZK)lYt-+~y(KV5Hut}WD0==vQe$MG2_Va7bbf za{4SP0d7_+;^sC|?%Zs~=~SB|e5j9*zIEW>jzUWR%Ls#fzhxSllW9I&%m7R8tW&Gi zb$iPU1$W>msdm7l9=Z|;JPno8aW!*|)+T>8N0a-BoG4V<^TizgE%#0VKG%i86__~OJgpjjI}J`wB01EgauTOG8n2KMg#1B=^sQ&BS+@04Ro@S z8#?0l3nT4uIIgP6sk`~ShoUaE{8fb{Zv%wtC>5sfeC2M|P{Vk4kz6~6!DKWR^N(P^ zP!09{qS_a;ugWhCb&;>a~D1$V7q_pprvpI27 zPebgfL=h5dnvww;WykNM#IGk*G$Dp%_j%V>Ec99|&YI+{;k>LmqHi^!X8&^p&?>*i|b_MJb-*fr!VwQ#cCL1d3j)O5x&8KE+^fSWsYJ(@VsjGzS+xA4S?-nDsjhZY{UpNlX2JuP`7 zUR;&X=ap}mQV~z~to~HGw@^<)l!?F?X%B5Hs=lheVcpBV&~>udsf(6-K$3BBsG?B8 zGEZ`|R={QtHk9q5`DqD!<`J-@&m#WGBWU>g?yROzdCPd9L&6qAE4$4PIuN%ud%k#a zbcBUA*Ybq{AiO*xLpw&Ic+NF@`%w{Lq~l3uNsyJc4bzlzakJW<+>jppDMa4TI+9Cm zOWU|yxQe7{%78~J`wS1~HcRUN5l4X$Jr3w>U(jxy$!i`1ABLLcRNBbUom*BEfwQCe zw!qTba@&^xhus_>M~+ChNLyG(QwL~Ppx4sVSbz%7lA5URk{Y(|=1%U-t+6QZ-dF=E zR-ul?Nej?&twA+4N-osJ(Yt1GajtcIf6#n&z$|eMD_PnU8c3etdlGliiMTn zK})GMCb1#4(K~<12#qshsTo(;Pt}oNYZn|_J@fC5(DF0q)|qG1*NBs~HR?tTFj(fXOZYy>3K+13YU*Ry zZHRq`8@~xO!@$lmucRA5WFE=Jq?JRD3PE}E>nMq#%r8OJ)X48HPGe0u(R>+@0MINj z`U{C%B@){lE%F2@%;jzT?8>K&{tx&!hQz-$<41m4MG3Quai;YY5-W{vS>yq`?&<|* zo#SC)xE#nrf_rlNt)3C&G?=mP^8%#Q93AU2jW9k9&#jzJkUt&Y;;8q_P09&FT62u@KU?0%Irxp`nF%GNHa5flBv64SK@Oz!-2hR zusoBnE=r3T(T}a4_?E~1{?xH;F)MXoI=jnb#*cxOi6`l#yvb+lgw=UHc}4tKu;Zei zlU6Qn-#QvQgv~S1u(M;4LPWJmE4q;<#X0GJZBvaePH0>~3>(#oA-D-1_WyyK5JoOP zhp8YcIxfb}c3N1P?UWp0@qMZGZ+KfRE9>N|$_z#1Rgl@Py@@Yx#PHxWEL-B?zzxA= zbKAK0PqyN%Qf>3K@zMi7L5+Xi%F%qu?+hPWA63YH`{%U-tUO9At5(Qx)^4=ntm-#= zL4-w(Tykf|*D>fFHrg&m8qA7r!|B{#Lg9PN09+e)hzMa?@mF1#!vG}PE2K)3F>4uw zv$=s!Oc+3ngDrtRr3H(S(8z~k#%^7?xX)t2)d?(YKRmYg(vV%+7B!>`Cz6i$44;#0 zW2asB(UBgs<@ec*)p~|GX)~SBzln1D;rk;^1%<@qp?@6@H@lsofsM9Wvlv{ddlD;9 zNILOBP5WcNb?MMu3m_S4rUSGx3-RG;Ys@eW7f1ucw=Qp@NzI7+ikFU^h<5aLGhuJ& zh&fuT7qYbAf}uOM7Lwj=+74+uoMhoRq$o_bAQ#>wwq!dNZE8T&3f|(&#;uXDiL*7m z=|q#B*rM~SJ`P6)E%ol=_kt6Os({{PUZ#Njww^L#kuX|V!fC#q)z0@w^WX+nTEtLVV^3`xp6>@@&TDg0A+#W1Q z;c6$5oymKEE~Zg5vGLf);j^NZrmmc7!r}aYx6@-oZ(M&LDgzpvG(8-+9;~Ptlr^`J zd|VY`HoVfbO{qSQn}dG%hU0Putj<@1W?p8(_VxhmY5lOa{_*J!a4Fp4|0C@ zi!8W4v^4k4%T`4Q`v*gYJ!}Y*$npZmugAss0s1Bw$gH^8(jY0!#ypebL0{&PJ-JB2 zAk-Is_1xaxv610cM^6s?K*aIJCA>F|Z~W;Yyn69$ey0I^1?K&Ld4dx@r4vQqa-sy2 z6wx?^;L0Mb9}|1I$~Te6wD339@Gbnprjo zyY}0>=(VPUR7RmTZi<5jC4=wP>?2(%V(#!ST zMjb(Ho*hygfBOcZ6+}~}a;=>vzI?ahqIk*YOy*W%@H>JD2^2%k6}2DQL1w6AO5A7e zHH*-;#kF&1j#oTAeHBjA^ z9xfv01|dcMIAVXhXR7lR`JyYVC92-6;hef_%%9XH>G~Inzl2t5mAHg;}jg83WYa0+fo4q{lx*Gxmud z!CP8`lP9*azIeYbhjyWUsqqolw!seRs)V7|Ju8ZG{FMGkXK$|BC!W8cXwHY{2by+)C}JJ3(;7s&qwmX5GVv)X(WrtJ^_eJ>hheB zOgQ*JOUt-nA%VytBjwRGTh)8%!+sUn#rvO8T1#&wC|1CiBs@ljO@Cw7859>ZCSkdy zpT-NBKLB&#@7wbX@IQ#Z5cd9BNyWPmp^4V@KUe9lhIL1qY0BEJz*9n&mUH$pj7`5ew~oTb#K<*!j5y?X+~20xzS4dwQ<|& zxX@l^RrYIg;|s5O^gqPnQ!_J!4rB=@{zF~XVjBZvJO4_E+mbABHvR>{PV>i{=K%U3vIwReUJYM#JU; z+L#%gFAOQ#gMo9s6yp*7?Nj&8npbH+yD%ArYp zHeT7~wzpW`;;~o&@~qG5GFBu|6GikG2*a`w%KCEha`I%GVilVN^R&xpT%Ckv5us8t zSLiHijrffAw!Jb1Pfg-w9wXi2AP&zbC(p+oR6E?lJf4W1*w11zS99=Lu(#Pz3y=Q< z!U|}`N9#@UXbrCOCa26keqIa=bBOq4H!%+K;Hq3xR*cDFB#R->~v3+*+^fQ#wW6#?G>u$ z;D0RuW>%DaebsYRK=PD79rT~UI~ceC4qIPV_=!kRt4N%fswSigRSloE)g5+* zNV4k^U)cm-@k#jcf_?WSZmZ(~zqGiDkZ5>H@n~L|p&hT-1nEy8QV>60KZrwWEZ*3_ z745&GGQ}56h69i^+xTy}CM@gLVODE^CqG9?;6YjX$*qzZF=b64tX3LG4L3}B5 zT=bH(szjwLY(^e_IifEiIuCu_bmj;PU3^h9YhFX(Xq}juuzUanwsVWQ{tl8zb(vgT zISdq{cE%SAl`jZfL{BB`4)X9T8SwrdI!~7JWS+4JzsXaDc^+FTDP~V|G9CUcZ6{Qk z%urErci&DGLQnJNS4QX02tmCuFDasGVYOri)Zu!e@qD`K-EED{?`}a)DavBnii|fL z6LrbeaexR}f#?3&O+{Fa+N4dQ+6YLF6K;jC6cl^KN#ujtl{k69(K7t~r;90OqTkSl z6hBtQT|1Hg0Jy^l)x59b$0UMK3}$%VQl)f_E)a8SSk*Y_yZ^pEO?*81ydK|fXfD4y zIj(Wmt6WsY-1OGAT_vrreqo-yitAv2}jDe{^`dpa^gHrnK_e&5wpX**V7DNcdhl^aJ=^K$uQS zk_Qvx;?i~Q;EE$>0bSZ3%&t!G5q!Je7PiELg*cRFrL;4FGi}UIB@2?*ufZox1cb9Y zHz+2?n>Nl1+_UN*OaxuORV~VXdd$OCDOdjc$zJY3gU_UwakU<)WhuFSGlAGLz<2J@ z3_hp=MPnt}M%>BmN|9|!=uGYRLwR?)4?RQh}` z>Z30oHtVK>=wY)iZ90jMa8a(YHAt@3Egq~o+ZxF$;Xc`DNt<;CUc{U)z+348lE3ud zeNXoTxq2wQamh#Y2@kNts&1%`BqFR|X$;)rpcht6j^V zGfl+5qZ+YPdc0w5q~#|Ir=AII)S-lTB>{`4Z;(jC$HD7={Fbp z0Z$y)8DC$5zfg-9#r30OC7Egt91W$f4}HaVH{QsZe5VAPi2k(7^MF(S|As5Ro%A(v zshYCZw9v&wZQe-jZu$)|xQz~-$Q^Bz+kchGMoWD|1CK8GRN?+R$YeXhP7Y_;P7aW&zkW@-g?j9I_MS4YSX zYNGrZCy2j_wj0Tr|IOZ_q+OzhIX(S($R6XRaxj%~uyzg4GNv{eO?nA_QaRZb&zN#~ z{lfQXX{73x3He=-@Ys0qpsp{?%r9jZ>*E~<-hRc_s86B z(~m@Cd9z+eE%%FUZ`Z|{6cD_}>cHI^F7Ob(_LR_!&SKn-1rDw7zVB4|xMY5JPfl3} z<%i8AhVpggu^@?e;PAIgKPO~9NFon$?h;kpo{r~oF;rVLgES?y0k-M6YmA}Y;q1VK zL*UVr&0{|cnC{B}HVusUcy=jrH!-iIB$f;J|FW-dMtGb;0tS$xeP|NmS1d5PxBsJnUK`B&Mmo8*nVFq`S z7_AXm3eHKl&muX!PibH8OWyXs&!`48Gvo+ufW?fKkKi}KCZ=8wv{Wg#54nAq{r;1Y zG8bOFd&|%%-Y!M0AaEbYBqi8Zc;~75!eNnx+vOYD!^y?y0~q=+H?xt^f7!LHxW#K<1jKvbYgSyb5A;YYX+3#$&8oUh_YSXNwQ&s0p-)HW+#*YCULe zE@7pXj7n>3cfQd69hCPyaU)D)JSKXschN>omRRavYZH%9&WkZS9|Og1+hR620yk&o zrt4)t#J!VQ*TmAe2A5CCGRzmu*NW(@{gLcB#@1v*nMPD?1x>B3(ID-dsPeB!rW=5K zdA}(P%4_CVWNSca%^F@BxzjW}IDncE@sKd`z<`I%@^v2!-M{8lxg$Mlt)8zHEs=1T z=?}Afcl_in0gpS{JtVbq5=0^aGA!kBR}K4#qXx1n)TasLi&6#99{P`kEf{1uYvZ}; z2GXeSWl1gU$(r>qP}_lMNs&QuT@_hiJGcm%s37u%4$otwiVzL32-1SKHUwO4UnJ`g)UJ%(&Q>#ex9VgGc@KZ5*m{pT4u z+3YeXXWf9js;rrn3w zrKSAU&p;52=pAXRzG(OUR0occ4{!a>nbBNT0CSRgEY{FqjV?J*FnyeTOTTTQ#v5~e z_PHsO?Q(5{ViFsp%cH9FvbCg-)3T!~PW;Bi+(fM@D+6flauj(QQ@N}0wDDi!wW~$A zEOiB-H%(Rk{F2grBn>vkn&)$$c~6aUu{UNrvzu8%89FgK7~X)DY18}n)=GI z@L_aS1kI;p1afthGL18O$}!=PH#}adqZUCg;_Yp| zn0e(1(wwfp8xL{IfA$fE9wobMU(OPzn_F_QAz}Wn_n2-15??L`>F63y8Dj`+T5KS& z6%e>|DzDUKTXfG_p}o~Fn+_@TBZlRTO!9;RpYLQ|S=qYlAXJE8vN1aI@pzwl2>xeP zeN1kC=RX-gAnV`+qX-%sf!|h?sT{B3eVt_Du<>>U?7`)}-pmfJ&N@GK%sY;8giXNN z02Sk*mn@ z^*zK7GoR4?fC^Rg_E4RR*>-ht?M3|tRs*Of1rBhN!UfaC=VOooM^O>M%%2h^o8JB7 zDYAk-Jr6RriZ$#1kEY2+g2+@na?F&_OQ9wlJt4BTMn-J~xNwbQZ_}U69%eT1?IiMl zWZqMspEBY8l#QNAVC%l57?e+S0tf%$xFO3WvYbIwt@4TYD!mn%|3;@TNSFL~w!6BR zM%1(&q_B?O@3>{Gf?@m=BTK63C(2ycH|NjqtmvjDw#j}Z#SID`lbmm*K90euc%ej% zSalYKYxnS-66MgAB%f%oTU{de-)R9}GvsS|hC+;O zIipr@;3EwjyYKufr#(`iNT>=b$RbLsX{SqNN_D{y9g*(7iYZc2c$5*y6iR%Jvjr*_ zNp(H&wqa`>`JhS zu_HRb5xJUW5M*RcJ3KwgH1l|DENMKbdEY}A3jXo{fbn^x&8@xf^g7(MMP7N)4h{tG zQ0mSX^geE9f7~p6JksryXOr7(@6xAqRc1l3VkWE#J`fr7eKJn& zHb5Y$E`ZV-=VErkN&3QkSCaMHuevnBQ#9>7)}eT5!mIQsufJ2_ML@DVQ!2iRx_KMj zGF{&sYRfQwXxB}%g*yz-dDY;=N3}v$`Pf;XOD!RLZjxzeN$Qj|x6yMp**(AYzwekc z1Ozr=VW=r#Xy?bXk(#0rM@NU$4t(^D3^J&K6sQzF-)0~IVgb|Nb#xk+;{VoOOHlAIwsP^JmngE0 zP190u8U?XYX6>VJ{000Y8Wy{T4dtZ~IOmnAY9}B+UqvU?P;NFw(|~UowOI*4SQi6% zap~(AEke@$5jv=KQpHh2p!KJlNr8|6)>=bd@1?{23JOKmq|NY}F+F zU%qHKVuVYJy50)5LLzRpv?vXs=gg4D%Pwae>3foNe|HPqs!Nv{$#k(o{4Mf%Ez@2Ne%lf(`kddD84q^$UcpiUZ=ibz~Qbf{K zt$(OS4~9jRweW>B$O-XP0P@9AO%Ia&lEYmT^^!+%o|D;W^A~S}Md>D;j>+>RfUvgGIaY6Km)D+-v}oLeeyX9NQJ%hkQ8-|^ zriPW;-#*Wil4A(4G55%)jsWt~B5~-{3&WbxwKI=L58Y+G4v)j0^~e%__6+%EdSgcJ zAgqeAv?85K1eIZU{Mn{xe)R$Y+rI^NlcZtRgZ2$r!}?3b)cE86mCqPL8tVmS(N6H< zV~oXOps#zDH+FjI{a;l5C!2eiRG_*&$;^;?)VB;TzlB~+b4<+#-nAyjI`IJMH)TO+ z@`*Lnet&`Q$D7~An4jbx=gA*(Xo1l4(&*}jo(<^<1%smQ%U=ACqp}TPKCpARl=&1_ zS$IAo!Tfl_qFX2}MXz7x#ckKu31Tg5K6Js7OHRgDo04OtV#TID^KQZ|;?pcU%H@=Q zeid9L>uiy5d)gn8-Lrmq`M6i`VTI^Rz^oOO~UgoWyU@{=tXsA~^b@wFe#EM1M^Aq%m!K~KQsv0=%0EhC>*d}_+Rki1-s5_r*Uc=tr+(4M4rleQq@!&2KtsR!l#UQOQ~rM9>;#4 zd1I|5y1Y-5$J}(iiVZHW$Tb(1Y z-2jWi5cM6%NOuQLtnJ;s9xNH|Dea3f;MHPRt?~O9C@H^TKf~{gMi`Ggs(|#Y5C%f* zXI*d$Jlfd*5EdIOrtwt&Hu28>d}nN;7KxG34NGbNcI4FzTL(NF@4rH+YHx6L8>L6S17Q2n znUkrx7f(+sN5 z|El-@6ihod#wYp`xiFi-_CsG+ zL)R=Hb<+0(Jr-y<;=#tIi3b6a<6JO^Wh4(93zuCs9ES{lkWc-foDwh08>u$heQeB_ zz%BLg0z|QYsbLSl`tfiO3qY2S!hyU3Ua4(P!e$~{z@g}2sQKr7CgilHD~+v&YV7M! zw~;u)x14vI0N-3bmR9)l&{XQNJM2ZWPb6O&R$czWmHLZ;2V8UYe9R-E({^f{oMlyA zJNtCZ34LeLogwCH~QjL1lmFppf=`GkQ5Kc=d9K{n>wPVA@WKIT@uTj097 zZKNKjsk}rCH*QM*l4`TAavia#Md?m-@84sx`*d5?j*mUoFCcb)==a`aHIi`6qwBFS zqxG2)@W~jo9rk44myNbZM?{pV8w=>_j_D(c^dp^g7cO%EO__(rFMfqjOWq4h(Um2? z{{x}cNM98S?6V~(k<^WbNmyJ1E%{XHbi;&^(iHtbn&EM%l+2)TC~3>@*RB+ycIk+D zzf0B!ij+`;zL&sY^xgc`QHoqYn4r5)6;ubacjHJImL~FOf#Pgf! zqmZo+3Zh!b*W7h<_B!yl-a7&lZ?}%B7Lvg2R~JG*vh_T4Hr8=sQ#X|KpYIs9&E;mE zJ>3UAmcvzIgn_R=61Rm0sv z2+}~H<=ZYp50jlOCF^=)>NzMc^{x;_UNrn9M!Qq;OaI*KMRIU-n#qn0im_{CQa6q{ zWuoH1hCU~}imT_r>+Oj!_DQ;>(&U(xb}vn}KCm&$sGoy-Y*oiZr+3jCfxQW}(ZIHC zW|frPw)L>L=Y)T_e}UtDs?&&r5|W=ykFtnzk{XJ!TpQU>o&TVhR)0)-1N;)IVot~G zYy2S$SQ=C;j?=RTF5XEgMd0QY#9JB8s$&y{3yt)S1*j*DxQ&kPH-!YgEo+b4OC@RL z85)Ek^m={}u~?6?{~_QE0jY*ao($fB~7l5l8 z0pFjf*d=LN(Tn$AGB$ns?IvLmfn1`xF2UWw{rQgVm0%`v0`QLE=pt@dv5D(V9JT!JJ@5#{C3Ef@&mRC%_YAix&=@c+_%Z zE_U{O=7|a1Q1K6Q%H2x3W|k15qoo;bH8Zu{&~+bdklpC770B+gIfKl?l9ZO=W4`=rDe1 z%8^gJfhrWZFA#C=mwNaBJQz#H@2aL{otqnX^d-f4C_5=RDbdZ)%n;$^+6uXfBJ+S` z|6XWIjVStS`_~wwwYc@3hKh;U;A+ikCUXOHbargc7ka_=#+J>6fNNGUSB{!DCmbIN z3JOD}Fh?D$p_HyoW&B$WmW)UDGe6Q5^*-Wp@YrQ2l5Bj=+R0xxyf5|7o}~UR8Rn`z z1Y*@zT&Zx70rt*mB=QxfAY~y`W-alY6?3}QIT+m=ahuz$XZ%um$n}&>lmQuGgywsW z0W~b?Wu*GhbFQZJRE=HQ+HjR-7!hp0zc$zZ6`6uFpjMtB%P%SaWf zoPpM@jT}`n#h(!3q|CxUAkZp?SM(;Ql5vFMY5BMaYg#5M$rOkB*m-VwwKxrLm@GVs zb#j_;qIsLu?moCNQd^3U-IEc}=KZ^%`e0_-=lLn7E%N?)$nv$gyNjw#98@NteOZU1 z1a;(+*1;{nx(oSuS&Jj zh^blM7Go4RkM(o3CvcM97tmnpSlt%xf>OpN%7~}kNZhzOVZ)j!ex9W)IWsyj5aBeH z8*4qeGX9259B|tnq<#kToI|1jV^s{EkCd&SoT$UW@=>M&Wm`~k-#;31s%aw-{5bu6 zJ>@bS{#gM5G^A^ukidh22<^8Q0hJ}L$J6QQrIOtEROmnvwn!Xz#Yj$H43cRIGlI>T z;dvLWw>J24Fus*uKt^qHF&4SlM6?NRzs~EW=wg+HKk5%2R1Fa?q%8qJRKZMd zKnR;jjeEV`zxfq7*^V9zIN5G%*iP+#$K(W)RpXBYdYlk|V{e7f78^J^*9*I(l^GWd_~!(UJ6nbp(vw5OJQ+|0ip|1-&H}7FK5tPv z82nnfE6B_}xj{b{YI9U-B*pLi1*z8E&LUUs!MzEvk@6;b*!kbgybg;l^N7TM$AiCI&Hv|cU4*tC&_bcwRPx70dpM{Z z(Tg!vLxbcQFNYuoH_~+Yn)as6nzbgGCof%Dg^_vNG)e4=;!pDN#v3sxQZ3xx&vr-y z)d#}DVKf3Q-a8vap)Yp8Bdmw{TFMoC8A=}vUDY3~MLEs9Y5`dg=3CNX6tZ79m1A9m zH}Xa`n2;zXJdvlTjZV{=pRG~pPw}fPGuz38!n;>@DK$1bn3H1-Sm4Ai*uqRuDF$tv zdhWJXn+VXw3vyH1*T&TAV=FO11H+x03O@e(w9ReCNvJ@1;++DPlmAPhAVs<&vHf6J z5-DbgkzDF=ep6&Z^t3}&w?Fj6oy6t}=ESJjl_PeKn=F=cVy{cH{;_;R?Zz!#lqR?y z9swq&d*No741W8!R>%?pv3QrEc9&@%Dzjn<%qv%X6k*@Ae>tq6V@Vojl1;OARPJdlMHi- zP^-()ER^OJF;A#wFMaBP+)y|3#14DmqkjG>TqgxhB6vQ-gzx6U?k!QPqDC#HNPJ!L=Gl!$|ruI{-OuWcd*?s@K`Mc2V-_!oR_ zG41xLEH?S&++JyT`b$E{Tjd=xUOckjDITXb3_q^#cH|xHc_Q$5VXN`m|0Ym(7q|__ zk<;ea2;rVgwish|Y>Ex74Cm z|BycpPqtqVwq;N7en~a|wm$9VI3B0 zB;rsyx#x@i%tKs9!3gOcBh+|+yhJAkdHfQS<<`5VraU0t!rkn`0qoji29CE5wI`zz zGO}7hIH>||KNm@9-&8QJI`1z+JSLSh{XaT!AA`=HMD6B;xw|#&&0!5r{UU^H@T2L#bHU zSgeaZ^26hgq0{9q06*|$V$h+1aMdWMH01nbFsksF$R3&to1`{PN%H*ZWlmQ$>F60>Gk9v5-xS(PO1aM8)gW7!>9WD{0o+z_k;OEx(k~+# zXahHls~?9WQwbAoc`)(0!th&O*Y*p;FFZzvH*gA=A`9dQ829f4)z@pyyg~M)cF?zc zIcHyPnXu2#>J%;cV{^yCMJz}~NDOY@!zE7ghd~zF-`#)T27sR>x+y&(#?vJKqLWqm!j_o7z2ZUb zVnt?5hk4>Ei=LIF(N75auUp`DHF_h*d@`yDLKoxqq#7id>}z`gqI`xEWxe#;Q12-N ze+O)|;Xd>;3VOjT{#u@m_52NItI?UMg)c@H#LB(EUjIG>$-NXd74b`vosoC>>G12t z)u8azeh=@@K_Xm9^W|``8|M6+6ygrU9fHD z^|bnJcC-+4Ziuwq^>wSIdoV+C>G1bty<-{V!xNG^uZdcJ`bbFW*yyi52Unso**>gv zl684c)1gQ$ZwmRbArDVW9k5Ou9SYD*(Z@1MAtR$_1l7YSDidt6=@4eh%<6^)5xmG8 zR0GsfX$ggV?q7abE1}AnAae=2dCJ5JI#gD?St&7S8r#4FuiAkBEEy;FDBwphTip@E zl9^G!aA%|Px;G3l1AOmysvwR5rjVHX`)p-21DKl8!wewj0RHFb4_(d7=U4^p#tc_? zl);sgJIf*c#ViAEz8IFV2VxxOQ^&*0GgD)CmQRWL!u3Zb*TN{zF_Vhqvh7;J*Ux0F z%&Pl%O>2LPvE=W{wja^qZKH-q$K8{plfb^>lwrL1p)lyp4u8yL4M{4+VJP7C6Z+NN z8+}k(38BaMx2t&O7^I+&wHac={G_Y?!u>IA#i$Yiq>prE^l0$cX|#&7xdW^!)%G^_ zMzKccw8%Lvw=6`+7!FDQf)r(j;M%2=Zqc6;J#^Ib^FfmqX>#$ifC%=v)bUU$C5uKe z)2oDeZ$B8oWQ6O7}AxgVKcbXznZ%niGQBT%K#k*c5{10&wat;rD{pdQxg8JRn@y%uX>_rvM_ z<+G{ZO$TqQ)U~@-z9s3=b3TsozU|)N!^OY3p~wPWht{owiEgKl*pnEv9_cPdsyy+e zD&Kq^t(SXT{M$bYM~zZ5R5u3yZ1oMZ1&)0OTmd(|vUT^gsc=!|bLssyA%8!{Owxwc z2&;b>+uxzSio^1qrn!z@aKZJcm{6zk=gqa-0B4XQcC$Cny??6_D&3`oTICqtIVvK3 z!PE6yR390$v}Sv7nIP?miKX71I#T;7iA;jUiC4Q1a4eHIwn$FDPw!O+)z9KDZ?u7{ zULjp9NOJe2T+W5DPMj=noYOgh+Jgt_c-2%8oWa$C-_uK2g5WqFuhTijnBH>B0MK)m z_`g>NOsS0wvA^wPu8vaa>av#FL=NOj8e40za$WA%+G(mj+wBFpvte#A-`a++ltoAQ z8fO_quRie@f}64fj?~5`Dclrijf9vIlKZD^FX(X{Wf&{c9Y)Eu~JQCbCK|j0I zE~!9PTq6;5?dh%Y(k7y}*_YYV7|n6jva-Z_EP@84^HD~@?FY?*M2~I)WOL$U?9+@S zcNl%5xQ~-pV(bL$Wwnu(Gy|7&BqZ=yH08Y|c@_@rD4s0oYi2BNzV}r{CsalWQ7_Ue zL0+uWNIdTBOmCDSDP+22kizA!CK)Nb2$a5@p9EG5`1YI=ZCA4SclpO&<*$ltYex04 z2DM=A(`JMSWHHc#A{sTCllt9%bG1+0^M?Ftv~nqQFI_p(l0}?rO$jK+JhrlXBk72; z@!lJA)Y!ipgCU+^JP>VgzSs(6&nNcn2~K7j3r`YZnYft&grX&$l13Q^9?yzLw%YHl zw)1TcZ0M*X4zLZ7-^{2?WPQXUtAUvOKI^!E@Km7%N6`se_K8a<1y1sQzdXX&!RU-S zMlo(pnzGHaZN5}n2R69{@_t-U2OD$5i*t7H%5acTHx|hI*tRjTZQHhO+qP}nb|%gp8xz~MPM){E^AC1iwf0hX zuf_#g5$ONP>-opIKSC49G>(-t$A9wQCQ16I!PRSTJ8Z>4P_w+r-|QL8HI4b%s{3|- z-7iFf5NhD~3fM!^Ry33kgpBZPK6#m%Ms6%oV|L-JCSiI{V%42-@RSnA5?##1679E3?j0OhcujHO6eiFN@=YedfCz@zazy zOW?7BqwzG|&=5_t3hTcl(7%$gBd4QXdJprD7iC)L^fO^eE${ah@PEPBWpXt?Eo4N4 z2_1e>`@Ykg|NLYOpQzqKEQvL}LRG;E5$WSN>Mga=STvt3lL65IH;rhCnFU}Jq(T4a zcr!CiK8u<#Md{DVBv?H@sY)P-G(yT(Am)#uftldmfe<7em^-6sLUgWc4PWS-sc&h_68_kkGvMW0yZ$SnPRmW#C6 zLOTP*I}i^+b|Cmr{$bekQeaEoEN)n~_k21h%9>?aXoP+zqepA!RR|#Y6o%-kd{S9o~(#7;=y4 z{2&y6S#)7e_c^Oe+#Zgs>1SW!{*};{u0iHZyu8%4;0U;CQpWJjMVM8zg3(osGF}K^ zE7(AH4ESfqS0Wn)LTy!C5I00%r|Xy0?Df(uJTe|nXnD1smdkTic}R5>fuD2M;z30k zjuuJsPb_|>K`YC=T)P`bk!$VwXqQlH!$Pfk{gEz9`h03bOcdJr(N#;bJ5iqa0{286 ztU9<(*nYVrt3I(-ledk@^^wSYrhg1rWf-=9NGvcbhJ&)M^>BtiMjah%+W`D<4HiUygp3$`7 zsgp=0zH`jlkF(F@OWgmWq#`rJf=F_YJUIDhv|%UFT8c(V1oQoaP^*Co69%$d)fds? zDPkCJiRzaoK#Jz`62f3=X;OU=*H;FJv5K?~hTLZkEM8?BtxEnhqfhsU6>Tmg{jziW zUb_DJjKcpC%D#{tsrsD<8P7Bo5`zNqM;*PAQ(Q~t2Puz-CXi+LY>dc(CVH+XIoO&3$D@8@zJr(UqV z8CC$S5BS?(3oE`>cWczjzT(!h>^`9#0fPEB&+1&gz8f@Mx6g1L7&lO0h~T%NUC8mE zQG4dNz_Et2RWF=tal$rw>_$h~GZ2-NarUg4-wI>3B3kP4Rwf<-C_`{Ol(X>?Bk>9d z!*kQTt`Znr4EFQb$P?mswXo>5D$dXKLOUafKU=C1LRZLtCZPF%^JtIFV@{D-qDEDD zQS!sLit`up0CLU_H8$!{@xVSJ>_kxri%-6S66o9 zjHxi&J8TTfN@TI%3C?d3rEAIDV0V=GurfYJ-d$xR1bT^U>T5PxK(fu57-4 z$?G0uO5<3S*i5$|tZSDZ2D?Ak!s2FKc|Ci$`CtN*RE;Tx@He#_9~^v~JSNfA?);aFwnTzI+(7uUR|+o5ha zvW+A_S9Oo&yy|;@jKJ8XsQMV$0sokKr3s46TBUDJLF-%aUl+N2wLk|t*a^gCNINf& zdq#}h{Ad@3n}>WXX+ePI#zd$%HCV|*3VNyl(zPGTCHJ4CW=qr6BYhh-CyK(q1}Uo< z0ZDQt;~c#Dg2S&q=AhP>v&**cM>B6f(^&%a&rZ$~l=V^6g&zSm7NFZU@<3}gsAKaR z*wxiondF`8ZdAM7bbk4|(pqfm*p4Z~+r1(AgPu{G6@NB-4O5ccr)xBEUTWYqW z#yV9cDSr++rnZVCw!5im?ZBGkw<2D45at3U!&13aEEaPW*usEpXHyoOC~zoh6%hoY zAXO1gK@li?ZAd#v9~;P(jfdR{-f|GDSCzM03d1^>fOe%&{ig2l-WB&?KBJUJy@+PX zErLV!QtwLWPIB+x18XG?&6B@Mpz%%(hYzfnw;R`qjc~KSqg47z#!$(M&~6C8EI$H( z!q~-^FHg8vFfz~1yr}_q%hEKzblxtddaVEL*_dlf%Ik>B**K?~C%e$Z@5n*TK(41_ zbk#H*L!pAPM;im0aK;JAnri+nW?L%_IgpGkyI_f|GF06KUXD~)89Q=&o8pU9K88cM7;FHQq$(*aTS$z}H^4LTJJAc_wR&}mLPte^RefliO?x&m*JCxTY zOU045r?yQt=9Szt<^7zF%FX$FRh!}a2H2;Ejw6V`!ITb-=TqKSPKsIx?ob`WpZ!~#Eb$bBQ5dG+Fe=wxC(PR;`b*2Ecj$BxKt}+ z;?QAjW}}}|6GdD}Gl7_^=d;K=vd{GkBhN9{2hsez>OQ}tWsPbYO^$s1snPDG%|~lP zd^6&I06@DOL^G~|*PU;P(V%&GL%i&8Vg#&+;tSP~f)d19zeodi3l-$Tv1wr{$q}q) zsm2Hj7h5yQ%}N^kW57q8v&0?rOEhA~}ocFsa6T zG4I@@#ud7vF%{Ml>Yx@13@3>VsaP#oe5$ZVLkkG3x2@U!GORH)JHD__YnG#PlMx( z4KTdFa?aEp}ugr1jm3RixOuDTmNXB@CrM-|iIZnOs z7_W{=;31glucJg^3nqP!HX*8;Y`T9PcT;f0XKU7kmzu`fdFry+i#GH zx`F)jN`_pIaLqh+{PGPk3PFnmk%R?0#qC6E{IX@O^LyX8b_tf`@vHyc3#NCj~trc44cCQ}Y}_ zfxrc7&b3l^LMA#fUV*CSoGd*YpHl`)%eW0e(1eW18tGP_U1msA!3+!UXzD5F%p>qe ziwT~nBSD)gQd?{G+gG)f#`q_}M>uOp2Et;lK`{D{KWSy{_2t9Z%#AM=979SVs{6Nm z!+LStI-G~L|dzD{A0E1 zMA4=6o^ie(qptI0teXE@$(&iG-%?;vpMEKAifrPd0bS9mgI+L3|oqf2Z!c#D2?(AJyy-ZVxdV5GnKh^t{x-c8C zY_fSUR`EerP1V|$;u(zNu&8m9Qd`U_*_~U$`gfWJ2EOc1XyIA+LL9Y?4bc}S(PN1J^E$3G^i5d0|yDV+d-HM^Jzjl z0=Z+@w*>=Df-E#xwM0UUDfl@mcuAAHb)>9jQRX0#;8=^^M1bobO!9xKGC~KJQPngP zD`-v``+&ZQmq!yi7Ztr7qjsws#=)bMyWVoIWKiN&LvSsE+A83Xe?^==ew6!J>?QgO zYQuKlw{vMT$HQVFc}M2r==YyuaSF6@<|bt1WXi0&pt4CxC1pO@Gxs7emgT{-tLM|Npr1>qAa>lj+-a&0oubdgIPfL^NURYi0<{Q_D3 zveKZ-F`KMYAfFBEat3`HG`Ljs^oJnOpf(A_S4i`2Za{z`AEL($mT`v!WFIKW}xPyOT# zq&Kcd9JN?mJT|HU8Au0-e75JJ0YWcptCsirG>0cnho4t!i;;RbpXTRNiSt*;g(e+Z@4hk;~>U;He8-SQ8Wm7@E2KH zO2&BB=L9W8gR$GG<~VG(%=>AQxj*ybf$*e-y@%;^nc8tk91MnlHcO6NQy~t_U+O_| z|Et{yC^#DVxs58`ZG=~zC(T)tPa#xlIa>k=4sVyCSZFM2Uq*N4^>=eDxO*au z#;mx%ihcW_%6i4aQFJq0$N!BiKkK1ORgiY2|3G|@02N1%T`RspP|T)0XMs+1%BbH~ zy}e<7LO$p1BwLbLlB!{`pbP0Xm?z^ivuSjaaij+%265z)$VP5FDLr^ln}O{+g>1T1 zyh}@B?9(HBIairV0c1-Ly9Ipx7}?b6Qa!u&^6uoYTsIPB@YXo53(<2p9SlbskJ`g$ zoE&p3jtX>CMp@^=UU9i9bgF>dlq1xVeHD=ARy1}EeRb(+I9+drX>bRY=e6n0teGfd zV;eQ}v27S*HoR}f4f*3`he$!>^!njv`ZatW&#MxzDdneZiy`?vQnv+;?&S0?dkAY( zkfSZPs>mJ2bAL#{+G>y}1YFf(~%`^eB6QoV4c@P)%2S zZKpuI;MWTRZ2aM(I3zT_8`nbC8Z^ghus(P^&82P??J{v_{5TiP$8K{`43f2X5)7WI zxNmJJxYXwdUjN;R9fZ~YSRi&7^5~nNqW-+A>iw$AOC4Eh)awQ`h5?%zB7Q`4SR#XB z--F+8eS}{b?-rpRKnXXE?6vxZDZ_d>7!M^j4GQ(?S(m)UsrIfs9kiBC;2oN^OO5#k zb0hO_PJxDQ=YFR{i~-Y715yL2=!jnN!V2S@6ZDg2`#I%nu;<(GUylbXc8l$T5qX#% z=8(<8wgjSpv8!3Ts3pzZ$xAbuRZ@5zu6bYTscXfPFqgSjaF%vW=050nE>wmG!1%+% z#m@Wjj9<2}fhwB0gMD+%q%GD>@2y+p{d8q2Dku~TjrwvsMGqWvS!VY`BZaJ69^5lu zW;b5yR`OxMP)BCf(!CqmF$g;-9?H*C@RgsrhJ4OKK&)6xUUm0Afg*81` zV3KLjPYSWp>1#2oVxbjOC$b@%YmNRnDwCcz4f*xnR=N{Kt5h~$IxHdekE<;-fdelq z-d$zXljT=_;3;VERC=cQG<*^P{KRcLCdF-&Da*r&A(8B#0d0h$CGo504uY9)z?Cn0 zhuEp}`%1S*2+w^jusS&&x6zTu-S|M&HSZh11Lacs_|@L+nwbh2JLpQd!6t1i>y>{D zyHpPs^ooMt$@u-1UU@knUD~s`>DxQ;iuKWu4L^UOTLZtXW|p=OE&x8*zXtt-f4s{W z2jm$W*qsUBLZ1rCH-rS@bSozxY8%LngYs1mX(E7!bb^p%gn@XzPZ^a%p9Sa4 z&W3E8Q+k9`C59&8Xtg93Mh&gSZAumx0Bh86#-T{h7rk!~uk6bzZ32E&jo4*xHH~wx zS?_f?07S-r?GM#{fww2VCy81WP*6d=PF{bjEd@IF`55{7aZp~_NU%P90_#M)^pI_E znNqmj%0J>tlv|z{GXJiIaB5DE5fe!Acm#ioy3tDM^4cTxUt4-QQ7;b4mNO|ctoHm7 ztof4lpblnPN4S9=j%)!Q<$^>K_35~^vgNd))XRgpSyM?LMQggd@N=K@-r}ONn`EMq z6C&(ggI6w#JEo{Q+r$v4Q&!X~qfaZ@WzR%F$Jnsk4PhNf2PsgtLz4TZSeN!l$>AS6>CUK{Xu{pKc5LXHQOX4BIsRzN+l2EWO~KT(mfV zY#1eklet=0O=--y!mGxuJ1ffqGnaVzOPQL8ql75-q(2 z@0wl1B0f#fb^Z{^#Ju>CqW;XD5w~u=Qnb>rmRS5IpIwm%ufMi+l_ENgL-6@`jo#e2 z9${)4(xD(8asywXRD_QGg4w2jn@>+?_|x8oRzp{eZ7bEpZ1VbkjZ_b#*R9j{4&Na@ z>_<+VT$E+9aO-Y<$&#iNP5q-ot7qGKpWbqz@6DcrRA)I0KsKrnwC4RSkxim_{?Pxi zYxSlg=|%=^DDSiR^^q@Sfp{`}fl-iEH6aAE+u7O+lHFIBc|I74g%Q2$vAGZkYRjpA z8$7EmMCGO2wN9v_m3nHqRP(8I>|ak;SpRUB_w!wABC%M$TEBf6TtJq9|Fm`$4BZPs zBwRJ5)(3i$BDH^-Y{ZfQM(6qdPR7WN=Dxva>K1|N=Jruj=Z87Vq~vzX+2;GdFswRx zxG6BN(LFqSh;D4)t>nyx;Cb815puD?I7}4fNhrvYW`tR{(as#g#nI3@*k3f5)DfSL z%qeq^rMefF!NZ5^gtO4VV)+J{347J{qg%I?;MNNDd{GR74YqTm zG|J>Fc!-e6^d2(R;X<}s44ul3p^{!3bPUQrYjU}?mxj>hn2 z^$0&m<77`yD1@#$MBRY3=cEf4&_B2~ zug=|nEkgO|Ae}i<#3dE2bp&TnumF$!((Ig`aXCG5d_Fp3nL3&M3bn)yly=p7L&kk# z0HlJo>>z=_t4G$v&fOV8yUhLCJW zB4E$vV=XqCaRl(cyNt&!HC=pMEVaq;epT_?Kv-t&Hb{oorAN7eHI3SN1#F}4r(8#! zwWU`j0gEmr-|ZS?7{y)A?9Z*H|8n)D*$f09M&#MYhB3X z-odau(q=PY19KIU8l&S}05ek$H-9p@rtsMU0gV{!kjx*;gMI6i%Y&Ft^9Qbny|QdB z-VjLRn1W8&HxlbX%nxuSvYI-Sy!Xj$WcDitbuyBdNf-`GdSi#N7bZ1r1ug2{4T9VH zKg!bTd<*@fHmU(BsNV9HxrA+@!H&W%Y`p{*0=x)ynF6#XUC+Vf$jn~^qT{J0+;+$D) zzDugtE?P&@_fQohGC|PPC4`d07#y`(hbdwz3nBLjy;78$s3rFPQq`Cy)7?4!(aE9= zQR(Squrqh$n2)Mb{OPDixu#E2Vh9DHcSZ44*5*L4tg46|e;Ys58J*Q<}NK zL|Z?KM+}%qWq8<7Ehg8->K*YucTP4)u^=TZ6!Mqjn5i1@S*%r78-ERnBaM&||T=|;NVk2whznzH*ocUJDV z$8RpL$43I&#E2nc$zAGd6setbqhqVpWJD?CRk9STYJn!FN{ITP()f~apX{1}2e9R* zoCDId9ET$&n0rE%<3lK#H`S2}4?Y|wOYY%tz)H62l*h5dZVP&G zqtkt@-ul>{&P*F%`-$KOCw^K?0h|Vj@<;X8!4D`txqqf>!@vdD2mwKQW|DdEntcId z`FtE5ZpD@P2D|&(Q;GapvZ^V`sY+BJ`xX7#Fhy<9H-e#iT3*g2o+6S=-DsN#9ky#c z%`M6xowauGKEAesVBW`JLu^sOM6NX??4o*D-shwR_}YlHCwIdHzSnX~bGo>5rGmJ@ zWa-ozkCs~e9|OD_9E6%9e7~Insu^Kb_V9;r&cTKMq^jdcG=FFTK(t3Xp{e6f`wR>8PgT_EYX-nQzA1Clc1Dmj6{XByH*D2PzG^VR-iS?9CudlmRn=?n$B)zC*X^kyG zMngkypUF!jWM=2}8yKAM4I&eF3$Uo7_??R0?OV@PI;n-Zy_w$cQLlh)u%tnDicFAG zmo!c~ylo_`%_p&_B${#8vgLn=tpxgB_-&-$w0Zx>sqy` zSY5Bz*TSt0;)(dJi9(%xD?9ty{RETSl!Hxct3lH`A*jx$LEtah2oAW5nH#3NZR0RH zUB7#}eZEibY@Zy~eR{gto3V!Sj$}?J!7-`<&oqz(GaCDqj6}fFc~`PZTAGbm|3|8= zSB3W1|Iy|931vO*87d81v=x_|5t7uQZDQe=91Wc%TrBRZKb;LvJ3n(TE%G*U>X z@I2awFxBz~k&1^RZe>Vj3*&Z;BH^rJ?^o({zc%LTzUH_-KIv(~47QR*P{=%J3SVeS zrmGHhdCIy28=c?+WQgk-Kxk|h573bLOSPu$un}ibZK|%KkXWb{6UZ7cKBKgMNqxSS z_8R55K2Co)|J6HS5=B+CX7Un15p~9!%r-<|GtNSuK3QEaxy_yhuG#464WDpUeH}|b zQ>;b-q=Gd%Lqf`@kn0Y#ZX$N&Hw3h$`wvGD>OL{ki`=7$4CRJD!Prj1+ImKLr9hTmODFg(O$$S>a&h8nh&&Wg@E2Fq|lpZMZ`nvYO~cwBzWQZPn@AccV+ z5t1|~WxjUwku3#$j(OUCVF=qV!n@X}KvpmL6plgsMxy3)htgGh zQI9UuvB9qh?)b(5`F?kbo8N<#6aWwbN3@2G*mWWbxb8A9-Gr&T+pL9g4+dVGJa0eO z7ad++sLFqJ(5@bG8F^kj5(`}vf#h)duh1OR8G573mk{*9f+8e_y6Tgp2sClV zd(a799Sj0?u^Vl};Y^lz)D5W0mVmOR_zO5>J7ySgT~y^JN8XA%>yOK8v!vVC+qs8_ z^vA@hgVn4Y_g9bhq|jW9&G;I$f&<-Xc}z;5!C{vB~pA5NOb>y2PLtM9pOppGEPUO~7Q?SHn7ig#Uo9*~1h>}DQ#T@(v_jYWc2HGx`VU$$6+_na zo?dx=4cm4GdWkq?!ctKsXkOOe9Amr*A}5QJ7@&f>OB`s(2ZvlTPPi^}a(n>`$p|0i z!NZY}gD(xQ7yD!AK5~3}Ad)*(xi$na(@g^R#T2YFVqTops~}D;Se@B|w_?uus5caP zmf80iezCsJQQ~MP@~2&``LD{>^j{(Ue<2Qh4A459J&SQy;ViUA^hdG9kqfWyvRox|7N=i#+1Q_nqgr8-KGzeQKu$@0e()fzzqXwfjfWrj{dRSo&L z)K>KJL1av>I``Ni10q)ltCBxW^45fwacTcPo=gr08SZAcT#r7 zZKsXiP>o==NISKaYY&trW^qCosHfwD=gkXUDd`)cN%~OyvcJvy!#V&Pz)H0it~GOX z@~GpeTwx(h?CN$u%yQ|<6k@QWCi8Cz&Z#wmOx|{)6A5Bg3 zxI5_-E2K0Y>Yo@`f$}8N%a5>&OEfs3B};ahVE&QBuErp7sL|JS>9MQ{!^o{=8+%Qn+2>)M=wI4++FNkSGEr=hfoG>U+KP)#(T6o#SDI1-|RPh@<4{sX6tnPVP#G-{CvGb;;uhZ2*rPBjce^sKOr(7~yADt+Nj;2SFL z1pY*g2CX6EZY6kv2JE(nu!;uiiEPfc)JUV!wdU=5mJ`wk~X_n>s#}p8@VN zGHI|-Kg+cnwH2ztlMvTEzP8u)+J>nWH960~oDLzQPPdqh$6A!$0DB@l%ZsvKnGN3P zf==Zj>>o6rziOltrNM3eI&i=w^~n^MVMtUHNQOfq;ouC)>5puHjAb;mFQN34{&_q- z9C@RE8=j~cF8Wj3mpY-$u!z}p9FgPo=vf<4yUIlmVA-jz{!P>w1(q=pu(prL@z)-R zBZ+2F5Fczu66c{CwWebJ9i%kpCNG|8z;pK* zuu`m>11I`Ex+Nr^3mch=o{n5CRkW-y^A;Hu zX$8A1sj-5H#Mh74-05wc-`#%S8~*qE3V0oSY_I#1x1%#Nm~4v58J-S!1@9y7SVKB> z=e%$j%nGs*#>pRKijnYa~s z!Oic9hdx|Q4y-(UT%I1TH?)Nih28Z_yZ}RP2w8 z<*JfW8Lc$RnT+3BH7Jzn%}dBQ2SREKxeOBum3%JX9jtVWsh{${2I~9pQTNyExAW3G zemoHjkHEd_U}S^z+YN3lH~M_*ulH@L`D0#>y>qWa2#*ft(CR8VPz6wF?B(s^y5|(J zyJcyBY;dr7=JFCADQGVJc=GQ%AVE`W3b5Ta;JVnTSJ*aZG!tF6-~zg0{LlPL+B6)T z32^gLvX+(b3WyJ_rn*GSTEhcup&$`ZXrsH0|L z>tIXBUxd!$N;s=vr9&S;>x7Do`GwD-;=j8LE(8_85c>D=Wf`u057k1!P*%`l^9>P@ zV60@tgu;O~lwkHL@L<*!ev=g~oTd%Ygru^9{ZhoiI&gkRKq~&ifb1U>-cb8oyAu~f zDj=25f6WRBs;6vZoG~vJE-XZ`F|cU-3psqMxp15o7ZvKWc0W>g@p_u(%)tRG_EQot zbYhtwf}9VAEl|l^V93zLxmdhW};Bb;GnAMX1eG$cMTy zhNGADB6-Shz8aQqtZ>Pv6y6jNDIBB7Pr8m(0WeY%nSjc9E`q6btItW@ZkuD4wSV}W zKkpv=A8=T~s;zs407As88L!GNeoX_hkvMxwFkFCN*;Ge3`~PnNARG`(CF;PZ)8?22 zi3Q(ARbs-1%ES&?pcPjm&cj<`)x6O!3D~mvh?Rn3mYsR+SvdHQ7JTDi35+trhR_O{wRn2ArIO$a3bVRs6u9+jDuV=l zIQWvg=H#ehqCj+cMZR*v4Y>?X2w7%MuDyC!nKS6J_w9*NMCmcQZ>#wmV^?yIlKzj? zB0kT%qu(_^V?RflOosbG67AnFFnP-c? zz+q?}N%cm)5fzzu-JTEQac=Vu>}G7i?Ba^9JpG)gmXT5zXb&Q^CD3}(Gql&VzBV~A zHQG6&e5;3oO9JDQ%4Ja)>b}G+YG6C5U|uNT*3H>7$Ee5ulu2UoNzeDOrfq76+L8r# zg~DKN177`UP~xF#cBt;xJdm#rR5X-Hr6wW^!!s&+4R$Ur?`cBl|IK$p29MF@%hPLC zbB4W3Vy5797V`|uBj{F+CrO=H7}F=TB8%L=vOk_1D9@CUzdl4az&?^{;ulsB%MrRO zXQu8z-ym7o4y@^u(D zLV$Q6bcW_7@*0xg9?r{ydnt5_j`#}vhm;%bu1=%)yr1^_es1dBCt-HwqDmz89~7@> zkW~5>TPgSKi^!&Otha81rB@O_I~GkBv70Dro!Dp)G)JACT^!72Wkb;9&*=Uwh@`SS zDGaI3SsaUO;xA4%LqSC+E58+PE1;awH8raP@A4bS2lGXSKs5Yite~tH@o<$JX$saY zxthHijW%e(L$$&8^NTXhi;0pmk^s;pLp2*@q) z7tungLKS=!E)w{|7A6zUDxF{!9ZU?LV`_Z>CV<38E0;w?XJ)pxHgNIscyKaxa8aK< zQ#q&ynCR)$&lW$IGqScNIH)NDd(WlU7`}*t@T`DI93;1BJPZ>@^||Hj);rwbCoPz^ z-we#834DHZyLk0UaGKeK`)`neyP{o!`{}Yp-X$)B+jQ3V6dXARg<5k75m+`uoW;bo z02`VE(S(de==Z188-;PCe+WYYX3&1GJxpEC=zZhVe#%!)hTA2WNGEt!Lp- z>cBz?_=aTj!yQuWSV`Ar^4usOS?h-oRhfyD6hIx>nT~G%V8z?pYXhknNOXmpYgyMi zEcgI>)(@MH9-f!pfX;^ZP;=WcdpKVj8;Odb%j-giE80Y$(e3glCa+Th3zx}2UH5v zGYURyMg~E8CFL|K_nw@h^WIO7boa0Bj;@ScUP(FlYHJzfzxgW#sHxH_Fm!<%f(8=V z@cvYpo5eXVsnRM_xbPDt|1qDHN2i5ff73Cac{h4aB^M?XgCC_xridBKFM#{Z{2O5z zm$}EUSi91SMby~hBqVze%oYFBUsH?)n^YO}2Hy-Ugu4r|4$ z`)8aB69P`GmjWSL>`m_mOjW_r)WI)@#6U8f5-_7Hr|Xx5a$HTRjG`r>dMdITnt5?* zo6}C~R{a-VM%F?W01HwsHjp+1VCUlYx_yu1eXW|ae{SwalROb4)LgwOtg%lHK>7oO z>ZMxpSB%y8sQ_@Jje!ZvCeLdqA;9l{{U={D7AYbF#9a$-WNYoVYM*VpX(dP3A3}C0 zze?J@wUS9ackNGRi?0-Rh5B>bYrHI77~ZWYj+7dv+OTd$POs-{l^@jp%bMJmFZ(}$ zpssAX;~4a+I|Il&)G2kV9Y)q-K2)Dj;0$$Ac+uP(mIzaG>Zi5#%=8g?9a-3RPS2Mw zBd^RTcr{IRq=(Xs`b$3=xkF$ja}j!skc@^)mK3 zrBO!}CEegDsbZz*+g>SA18#pZpU%QtH&m4{-EMp+X_3c8Xa4cviuE8$p-skR1WnrX zHlh7FgmwGlwx@^oZyXS|Ab;n$A?o15G-OKYZ^}lqf*rxqQ>wx84BVcN{jJ-qoX@(U zm*2JJ-qx*>2cEgw3a4Otr&*SD)V|^U;&aLapkN97PRhLI()#`MaBHUQFa)#v3m+f) zv&O)dV3z-gJkifEWB-QM9fB%`GT>yL-kzTGUB}1;?vg`*{E}99K`ceyvoV+j*#Qe% zE{7OSL8J7G78ACG`laeuNUrZFfTIGzZB_lz0CYvd#ifw>HouB4V|SZO@m;8xpEiqW zI4{)Og1u^^eZNr*ZkNo!=_SO_C&?9uKSRE-TJ_t2(eHB#-*3xU=Ke>%^LaC81~i7z zSzc5bp)Lv@@LRVpd!V{O2r=r)(3m}kwOj= zct5xDU&l3m(Z!aq$25>69ETp_R+f2)Lj-e<;c@*1mnNztLSvOUHV?MhwS;WB;B`x} z)h8TLtJ-aoo!&mn?inF~r3q@-1jvFw_~2alk;)>{33dTOEYEWdJ^Gg_LVLop9~G%X z#1<0B5xkpDW1m|3OccbfEi>pToNh@82uRZC`-Xfka zPxVW^nHqAwH;z(twuq@bWXmG825Nsf(b;sQ?GVPCAdmhfp6L2XijgxJDMNINfclAB zmWKlP;oNxLw?i{J>5QKz`~8CZ@wmCYJ6uWun=+Ycf_TG6R124M6=2GC*#b z=o1>wU=kjN;3+a)Q`BxSiX(AL*257mI`P8&Yflecp}p_@m$GJ4sD~4E;7tC4@n)5g ziGQ@~_yvFLqke?m#Gt3VgCe6=l?re)xYj%K?~$eTCK0!S!zJ&9K7>UvI1NpVQgKq$U-^|yGUeKDG zi~~~1iIe!BhJ68hjWF@-iGH!a%KqaC>El@(zD`^XEmWCST4=KCPDTgrzY{!cN1k>J zy-BvSL`Ia~?d7&-h|I#)!11`HlT8414!E2$6)gk+?{Ui#f z8rB~q?2i|N7up6AnKn|8HsT3FPLYCfuBT9}}m}>h`wR zb=21P&ir*UzECIY^qr+f_waWBj1VC^Ekf2p6nu6d z<<=9W-Lehs^)kI51qck5q3zcX84rPM21>yC{6$I*f1p6vKNbNTe6mO6VRf=vFWA2~ zkWU#J(Kg8al(n*e?)NY{D#If{r5Pwi-g9zY%t*pWr+1etyJrL3^)=D0S>|@^hyg#c z-Ioj2_vv^X)>TAjvWrd{(*v{H_LJHv6c#{kTz9#u8_$J50%#Wq0zb$MFXD7^Fe44y zg4fM{p`;X$BBIFF{DT9`o>7?5U}rwgC#Jup6`+&h2>Vh`fbU??vlFmFv0}=tE_6I(r!jG`bOK3^}E784mbCCMlwTL+h6+v)MYn@7lh--|ps$ z(DC8%{cz%PU&d{5;2YK$xwxp07GB4**5;^IoqS?1JCNz}NVyt@2s6b56M$lVPNyzM zcNJfrdnR(96iVnPDIF}?3)zqu-?d3kuez^;s6Q5!^*l6a?zGP45oL9978Y3d1i?w6 zE@{LQ!{^7N%zmKK8_#-hfL;-T)P8OcO1g-2a)LMn(~Hn}mlg3pX&a}T+h$52MWr0M zggp&+9&33E12vy)FdL~qY@gB>2Gx#)nf1LNZe5AxUGqV^x|W4|+;JERx{eYPlFZoW zzb5AZkUHY+Et3f&Kn?{l-LM~=_;ZfP;@ z$izeQ&;qHx1-jmpF%fYD(hgPB89R3FH}<5s2{bmX`XzX3G_hc6HsuPMWS)Lf9F%#Z zN6t*9u_yyZ5Q8HmsGWhUb(lnxraATdW0OFVYSZcNv)K0M02kNL(GcFdSdjyE8J30q zu`oolB&`NaSf_e;4_6Y?f-$B*=56$T6m11f=AXR~JU!8FFw0^FY@&_ z!z7`T66&sf#u3nFPh<;;1OKB5NLkhz29D38>g@)J)^pWn!9-U$l}BK*vli^{-C6Q z|8So0yJIfNe*QV%_BRCb4Fu0On zaGsychxz|ebq&yUHC;PwY}-lWG&UMFR%6?CZk#4*Y`d{-+iq;z_J8|+`1`GO)?H`K zy?4%>XZGH+XKOBj(b9>vU!1&EvWOtizEgXe&{%v2`++jr3YBb@PtQI>_XPXx)u8CB zDmoc#$SAB$9QL`uUe%#wpS0SI zHByXt)Gr5-2f`4B%5L?>^_ojqD}Gl)g$&ERy*Gx{ee{)S>}nRsQNaze|OBhlbRV4daVp0FQ* zI7HKY)pnfq!gzrT+U8pkOGDvsW>Bfj5<@@i2DtNB1vmj$2elh-qyo>Bcl+M>RrPhz z_$r#A9k>2w-|EP*-HKC?&(=zXZ}z!EHNz3(`n+u9kGNBo|`t zNYs5LD6f)!t^?ZYu^I$QpADh`&6rg-e9JK`(jPmJwHX@a_F8+2>iOp~tO`XJc7(C| z92BFCXMxPXV6aFpi@b%)Y#J;Uerf-OVshg zQP$t_`dS#0g&3SmrO*;53KP2sOz-k4Fr*|Takw4qBYjIhZli}l@q)>aXt9&MRq!+l zF+#!;UA9trKv{xa*;Om6Y@=vok}DCGd!IB>!v5xuQ9&-7{9eAl_nMa;)%mg}K;-ps z@Vv*N|78t_a;hDE1p@l62y>+B7dXNWrQ*sqi;_-XWp2QViV_oUdJ>ZdJ95a)@X5x< z%SV9T6_w8mlK1nXuq`&6_E4SZDSaP)LAI2!)ZYH9IRi0tB-3-cpD2#8cR22+*Cy-Z zGnggLtXxW+LD2vTD4d|5S2{ykHX!hyKoTs3W?y*I+KNqgdUaocw!5CRs2$F+6NjED z`k!L0FDb?rI&8ec_6a#c0i7~GI#Unk21P!^Ez~>N9K^F$dJa0=Df>vVMHxdMf8&vK zhM_4AFBwNQgl!ok2Vjy#WA5O2vk3kk`>dSa2A~npbF=f79*34Paq6x69{mV|k9W4o z(sRlC4ULCY>p<)KQ9>uTB#5kC+5T^;46+x@?cA({&xu+t{j#t5R8Hy)?!{p}M@*O=KYG$u1V}|Od2UCZ zzVExJUt&!8B~|Kl06}^k*MEJTrP6(Ox|k&rkCaBbLtJ8 z3QiDJ{j^}Bq!g1Fz3B8ALE$qaJ2uC9D5gj=z>$O*bD8dC5?W4vgkEk2)DUL=U3K<} zoF^lz9h>Og-Q~UDz1OzwC`C0gr_8iQBZB%i#sI5k9X5!T4pN*q{c*k7nk`E-P^?aH z*IrbK^B5R#9kgx*mU5dCZ9=mAbRzK5Nvpp6R9k4_wH9491*I0KhSbc*5qvr?nFfht z#d^EVm*=~Pfiud@V4wPMnajxc$(t>aN3Yk6Zp--FIDAQUfnkgsWr+%jp8jaW0*>(K&-75 zjh$u8byD;BavMJFyc?_-w9fb5Fy(-n&yng#Y*#{D(-oL}@6C1KYNXJ!CFUk|MJ=!b z&tD;`Mr@*XWq1SG0oCwr`Boabr37yED$LN}$*=aLHYr5T_V%0fgGUw#d2>Wx7|V6E ze*88KTWnePT$yi+7hfq^A&+-0fLdn`$g|)4oDZ{D%#1&OHPIKfl=h|22~_1|3&{k` zTK4x7H^e#CACqI{vs3<22ngEhA$nIztW({~;qdy+q=AYu+bnZzM%TP{(b#bL3yVz0 zdC25UTar$EB^Rp~m6V$pg`%4JbCfjvkt%tj%9{~)#>>rYoT_do7oWMm_9PMI^yhA^ z_@T~2I26(m173xtU09tw43vc#8zYVg(T%0dZ{Ah}mm2WPWU!*jdTYi-eyLhQzoKD8 zcJ1Iu>_RmK8LB9AJdE}S;!H4igP(Sj$0=uoQhDF$jF1v6B(jr;GZx^6z8lIBt1iLq#AZ29|jK|%~cH7xjv50Ve z0?iv6?e#M%=*W*AaIzUe8#D3@zRYwFX&(%rBI1)|DxKyf7`iORkZu@`3iAk>YjtA8 z33rqJ1QK7_mx(L2VQ(J=tF6oCc=IM7%s*Hg7^IaT9|C7o1({SlUj?Ik#1jon1Hj_eb8w}zga z@UNNXGKE_NfAKreQVwGnx_P@_B0;S+#*!JThU8VGNdt-*l2VVXXpoDhyW;ehkXBQu zY_uC$v>5S$u)(;2(E^0{BEKv<&K;I&Gzas3DX}K8xTxzQg-5ASs&`_L@8ggcsQ#3p znkL+{uipID!irMBSCtG6NQD2fblV1ha3oG*AuGYnD2kJLK;VZwl@2I^zP0`E(6jE6 z8YrINV}-a?b!-0}BaRdmwAyCk!|^3Kuma-ifW5MAVv|;)PuXBDPoc9IG^QfgJ^fr=X|i#(2hNyv8js}cQ=Vd-3A$kAeJzQ78hEze^CDgniZ zPXC<=x@F@UDhl?H84ka$GyX=D?ZP!z&fMMsb*~=5e2&_PsF7H z)G)2v7|Bt7PLo`TLuFr9!{^wx3p0k)d}CCrY6VuGS~G~=3HarX(G10`Mbyz3We^FM zJQ`CD$W5Vm;&QFi+S=`MzE-&?4Rv-aCxvpwLF>UOWR3}Zm^JL7_SHnR?X?3s zFr_16Hs|>#joaegX|47(Gl506dS@8oO;r6hD@|(Nl(ZJ?SQ56;N|8H<7DeO|(ND`@zvQgWv~Fu%c@7R#6EiED2d9fAOo3$M0*Ke2n(2R~ z&5L_7P-6X|9@OdCdKSe9a5O&mUGS&pO?+!Tdr$2YJy6VMYnG_8y>57}PN~^jht=1D z(Z9#YY&(2qoBka`gS!?NU7qJGT(}#m5Mt0?6PFP#KV_sNIfb#O>Rb{;s==m>C8;!# z;5dXeM+`1@7jL)XxKUO@?(syKWMV?P(!k+6H+d8{B8+Zh_lr!=;d5zn+Z9V;sS`Y% z9CrJh0sjs`uBq|^`BdWdvJ3B5pD-uJDNWdtVmm``x*e1roiUIy{>25Gc5A_38ohhd zsrh0mO0EaM`H%2Dvvls1I+Hv=RwK?XgP79HQIwVwqCsgD2-}7I14DBe92IQ7?~f!41-=che|BAZZK4Xdtpa% zS!7vnPzO{EGV~b>F<|V?9wgJS4EShf)?6Z8LkB0W{%WGpbdVBe(-il^V-&+?#+I*3 zOD`RChpi8R3|8rdGC#IDw5G$YT^9ou*T;R+oTDreZ{+hbNbAnVA8=!$Fu!d?F-yaM zmLk_Putg~_?K?Lh8ax*q*qnb*K=Zgu)rGBPcGHPK?%$moy?>plEw2oZdv?C^HdIkB z79}v?;*2e!KyZ{*L3IT(1HEw9JmX5}^u<|C|Mn{Zm}XelFFfC;ZmJ(*v_YG}CZzqi z-&K~6I^<(aV?K^Y^Z6QWwmU=>%}(TBP@(bckM<4S>0M@|CmO$D3Qv!p zZ7m**<9l$;9In>?2=W{05{~2i*$t`98YRwM?L8$$?Qe!deNKz7p?G1#`OE7{RTTAp zj7yIGsDdV&cl1pKF~Za$r91kqOs1j%-pvRuCSUWK&VgP-&k`azTC4LwzyveZL|4`r zt5_;9Tsg$J!H8_2ld`ik- z2}YI;PHFy`ayML03eh^w*TwLWX`zK+=5ruIZ|aJR@O_fxXBsLvH&vY4`@S9fTo+$42}~Vj2!GW9uyTeh{e_^V0F;&~C>RxnBA> z)wNnL1GfS6h`9?=o-;`fF9Yw0WD&LRb!-C@2zWLbdZSWdl(-SrfuAb}mXyABN)NsP z^&C>LhKtzp4>q5*bAw^6OrQ(L7le}Ydzrx+Ty>_0+#%7UDWzif@%(D(lR9&dvcQCO z30wn~vyd9b_*}RS3uYRG#yt!>7vPnaAm^w%GAnube8);NE~nlya)7j&9V+N@i99zf zM(%+tYyj%6rVGI!bWF=kZd9h#e)XNreXa$0S1D?f39-DUpTL?VhPIg6DUXW*3jEk; zNf6x@^OKC!LJR(NxM(Bcp2vAbD`w+Fc|(9X-kHCICU#1>7Y)XR?CP;YLS6Z4Bg$<8 zafXs(Gq>Ne)QZZkweJK>gCjTp^9&`IiN6F?6Nvni+U6j$N9Ut8-YParNTDcy*GlX2 zTGLQ(NhQ7P!3AIA1wTFaC$DrfwqGcZC;s%^ib&xFH6 zD1Ko_W3F5KNK{kux4Xfd=i)Ivo=`PCyM~v`ofZy2`_65!dHlSRIoHIkq2}9X1fc}1 z6+diSdbgEjK0FdVB`X3$7+=Nlz)4W{ z`Ug74z(v*$Q6q)iDzYhYsTP_^r?rB77Y(Nnq3S)HJdauS=}t?UC+f+U^*NwUV}n^r z?K`BytG}I7XjnqwCvvsPd|H8e!;^& zE!A|NNb%fN(^N=pU0Fpzs@%d}Pn(|XoNA3!WBg+;afz_%S6~no<`26w#5f2`?`{1! z){{^j9}(N~w&O}Aqn!1UgZ}0~P92D*b@jrvhNVcx)8V5}CClaCBAq?B8?Dx>rM=y2 z6yeY2ND4HV>4f|D;*xym&+whEMz=6kt-$JT7eFz)t-w*)uQY=?mu)9gU;%jz5B;wq zWS{B*O%2;ElXaI`9XXA^2QE-8nB(-kiRgRIDmxuSXw6xUATz@gtr4qf#L>q3H zr^)hkep_@f>7PZ-{4^dWy`zemkO!`)DM#-TG2LtaR&bJ=3=|Qg4;$F??(6En%4P@c zD_b;S*}va9C|xlrlfL|^hd@H6w-i0`^?{ICk*Mi3Z6>B3`58$o^-QXQ@ja5$Mwg%! ze*^c=b_Dm9VJq0rSKYwXXVeDmXw@pj`t>*`$EWslC%p^aOqvB!De6??Uhi6KxS;MU zU;;wtKG%dkRQgomN$c4&*< zGad8^j2;rZ(m(uAdfD1{Apqt@!R-Ys;sYV-uDbpqe}vAs-g;Nt4z}*p0L63^X=17Dz;k4%`aoFqwb)9JB8az4 zTEF?$YGxzo+QNPyn2w4bAU7dF12e}utSf#SY%Z7tUo>>JT0S+?z!9usq2`sDZ0%j>?y5ie~ z8AYGv#bI{2jDXHCXeQVAQ0l?8?W(1In$hmwvv5bkL!7sID#%0~;b}mDWfELD~&DQqfS6F}E`V@?e1v zw1iHHX7>^;uh_ho2KE3;2K^KF{japSKS!giM*!ix7#Irt*EuQwjS>!@ zS~!pWpJ?~7?KR0yR!8?4$L$GYo#MY5t~(51!Cw1Nr9$Cu=cCcEv@_@IIAw&9PXiC#TH=H3{j2yoxiL~xNZD13exXr zR5$MDX&t~GA~u31XDVuSN{U=Q6g9DLBQ_F0faOwi=$G%W*ga2XxsLh_;cRa%`WmjJ zG+Qp{+&HK@mMY5W-_Lh0+Rn|X9y6}AzX;mm(_JmwRrTAh6i>H^Ecka)@VE`$YXWqkSo7K<+`V(GJXWSmK1_D+QQ=iKomRKisO&O#-Fa)`&*x4za)WXwoya+R zT2$f&NklK`WG=X~uWqEtApRn!Yo{~o!(_$7s!9+YXKYWrRp-#!lzm^_b0M8N$C`Hc zF&CWR0%nMVY>?$d@h6@xu|h7dcIl-KA$=9N#A~YmQ-%Lu#IVQ<=ENzVPK-PZy{ma; zd$r^#$siW&8;i>gOz*=J5)9FHmhPxB<*ZXpLU=)$mqCBd*{JJnipV=4{d4ZfKV&5s z2BHn0M!|AHf0JX}@3UCDGfB^XZd40>tFpCdbc!Bh{2>$Eaynltb?VoQb~$v7aIHp;T68Ke-opvuy4v_!B|+y z&=g6$&bLvg_4}W#2gq(j5a|%lUhuwmUGGt$b@d99+TVJh=#a9a9P;A*j)UO8&0aYoI9&ihC8uEaC*auvlcoo~`x)jN zpnv*8e@_F+blfKB_U;TmkzKzq6cb+?qjHMLu+U^YNsRC%of@4tOsIVrGK}05jv|aV zZn~4(hW@zrzPP;9n>yRNL*$hyO@Ctk1|cE{A)-eCt#{2bzlhxxaPIWW8;KEhemp@! zj$iecYv}?s*VNoeTz;Jy=f%^I!w$tkGI&H>O3DN8;e*FxZtbF?OIbK3CcT#|!sPfWc{W#Y-~87AtpO0>G2OSIwA3&@o6Lr4ikX&(qOy)TNw zxpfwnAbg}?>O9jY<*YAtmgF5;~2G(L174A1V zJ1pn!XIm-nJ7LGBp=#}YuKI6jgGOEk8W&fHt-t*>?er&399j5JIr$o{4}DZuBhcV} z=u@K$G#{a7uI^16_yI(T=4c$0kPWUs6peSl4V3)4#`#Yif=Q>+3X>RZx zkzmwhDZ2r6D?F<{?o|)mCj@n9oQw9$MDXpFxiDY%GZ!w{@j%~1^jyFkvQ14XhC#sG z5L%`e`$F%Z^iv5>Px%3>i|#GX?mXihj-Dk`r%c{X;HJ!N+c8MYHz}XfA-7Zv-&tkD zaQHFVj%1*IGbo^_se%n8sP#q7jcO%yAs>>a=5b@mf)F8HF3S@k(%$sVJWf)6G9)-0 z*8qlaoZj6KwE`F|18in&_T101ZFPq1!GD|mKD;qdHX#P(n)2(KS5D`@CN?D<8^)IP zsGxB6l+16vxm_Qa1*lf6dh&l8t4X(uyZSxO`yqRxX8w7iWj_Yff9gDh;I`!!EW7r8 znMwxB7P*${H)=ufUhuIlhr4~LNmQ+O%EQ&@vGLPfo+?491$3U94|MJ3wtmM@1Ufl# zzyR6Ut48k2Yxi1G)P+dEnhD>iVwsVu9CFwfG>C`|&s%xNLmvPJ@Q7&>Dq<)P7nN9e z;%IUSs8PEC_11)2OZA8T-ga@H*1PV(aW^P* zCkXGXI-uRU8OLB%Qci0yRm}-E`#r~OIYJjvtL~Lc`0|Xn#*rwOQD`i0OKy{wq2n6G z&eB2bS?9P`h&pRtuM6Eb7jXmGA&t;4YpfRRxGPN%`ozC}$ACw`E_UIUYD%Ca%L*di zI`J<9M0`c%SEYC$6-}lOgfhk6@?yv%ngEv;AJZAV;y%{xUbJCxgt9)+Jch-H(t(6V z^;XfWxOPb>)=Z4fi;gBX75S9W?LUcZD?fhl+v270mFZxb8+xem&sGAZ`f3Rms#<5& z&aX0+Di3vU^}C9aOwcqd1r|r1)3AX3PC8q?t>2z@kFV?9vjByICow>@k$K@tn5B$o zGIA28XB;vX+Kq9~7RILv4tGC0W%Uy6J+x?WZGI(l2+%Up-)0RrLac|*SCrz4qpgfY z{4F;Jc((7%r5rm(;AouJ8nL2orBxxtRwlA(DFr5ki%b2%4$_yzXqzHx5F)&?3A1>B1s1$x6%a~3H z6fMMgupAXq;}h`0Rg20eBB;p8jG%(b3WA5!PG>qIA3nT{o1m?K>DbB0Z>RjEVfwHX za52-HQCG`x=sNx^bD4%W?opahx4bX%^P7L`G9e$z{3EY8QBl#Kg@BF)ltr%hVG<%X z0z85?=*bCon-O||b>?)b_houZC)UV){^X%GLd+|7iMiA+{U=A--03Zv-wn#oMq5wZ zJiqeb144zRA1m;R3|lCm?1gsZUs5!tR=al(DWddO4~IVOK)PDk>M7!87H_Q~T9MB1 z{)}U-aNT$wl=HsHs`Pw3AwuHNaX!{es<38{J%6|1!4-+n(1?Qi(xLWqk87um=&ogL zyx@Zb27rEY4Yw1$C-)WKx*GKKHe4i_I+3WPFniuQF`w!-4J?;BU?M&jqV>HAkR1XR zKKWw`*XoA@^qDL79%uBKos5JW&W~r_X3DU=TKjF)-hv-b`Mr*E?cNyQncpF+r%iaV z^GzjU?n;CQVM$Q#ZF55ynh>!J$E4|sxdj5b_-ov8W*|NZ4ot2SjJ zl5};R$&|2m!!-9AQ?Bc7A^h!dchbCNgNLh75kpr13>sr)c2;3Fm}OiHOBvwImJ%#S zwD!~}WcU){+G%E^#mROdC*KX&4iVBb=Gkt;JLJDrF! zrL9mTB?bdtSYmpIFR4^zo3X5Bv36sdV)O z*ic^}1qKTq#tBEk8b?4UOL$!-#VAznIhU=WBh(aSnP}pry|nVA)!$}Vp6z#^K$9;dU_7&YKH!urfHigyCKwEHzXK0*g8%d;d-jN;*$r_I9kkcFJppJ4QQFfMN`p>dMAv;RIEu+8q8D^^ zn`7kRu+5D=3Q`i%vp+*P!7>qOwMTH}?m4YCjRG5Eu?o62#aH9!81^vt{MgN0^dhxo z_R-g*Gg6p|06jP&vLC<@`$<`Wpnc#Jy-h-qXfwhd$9ZvTBOaV|ld%TyX}S4$r}lC< zxMtsncNtT3Hu3u&F9L=|{xWsoP4Z(WjdS9J3F$O@4RU= zO)by#8uMeHUgdh1w(5WARc*Kw3s*l`lJ+|necRP!AN@t?a;HFFASjZEEyGZ2-g6}qn}%cJ5Ho3H+x@oD`b*&J=3yoP^W)EITd7S zACsr7e4T|}1ZoS^jXpw%toVC5FpritPf;@_Jp&53Cww~w;Rq~j^dqS-o|ZSUdPLJd z))p#VbXd1f_va#-v@L&+xhjv1IR7O2ZP3QxKWJ?dL%IG1cd4n7@Uy%BHpWhzS zyN((6SpcC9h1N;^Idp*J_OXeYW8>!(KC2o0iTHI=X@T0Ty{PfB$7 zO;F{WYeo5Iob9Y`bAbD|+0OS3(~Wl$z>^o*33*fA?aO7I48$`3EANaE>!x2p$aS)1 zsP)fwBHw*~RWGeiNz`wN!610SIZrwC_W-IK4f4(gh|{(ADFDzsrR{R1-7OTF5O|qi zv=su)bG&bD)<(R@UMV}+E*0@?b0KlB_o2{$Aw zk5aVD=Il4U0qr`NTjrGhQ*Ikm>b_Ah_=RVqhb}jZ0RHvD57A94h+V;~h&0JBV4NHT z&C~bA>F%Rb(t^GC>~UEp65QCk3Rq+piiSH($1YjxPv=ho&q5dbXGj%m&v|nlDSKZg zg6!-Luppx!#-8P#{>Vpp6tG7{5S7($vxJ#)ly_yB&PXm<7wvBMx1$@PZ|%=|0v8>! zJL_V#OOr(glvQnq(qPKOPV=h9Wchq01O+4zqph8- z1vW>&dpc-s29j02ULtr+61vQ?r)%uSRAzbP-=kj}&0VztSy5V6w;=^_?`_WUg9luF z{b~g1Qf=tL@!$rH=k2!;EwK8<8 zKkBni-mh$F`JNVV>>QFDFu#FhoRqKaVGTs7y7A6p(xDS!acXKH-YzvIs2)H5Q2hN( zHroaQeK%=O#8=6v+o81?i(~x7|H`D!ov-=*eQqU7tMhGZK7qrP2@jVRdoEQCk~Io1 z`IG-6tHJDB%g0xLETs15tNR9yEc;q1stmeV*4IdaS@0D(8X-i-)!}$vk8YM5gSO7C zK%Z5bw1R6Lil$Bda^;OlZDKf{)?H!L97HFCo$d#+^1z}_pEL02xYWh;2@_xq)!WyZ z-<^b9D%!W-Zs?A?Vs`jKmN*|46a+M^vzwB*rgAQ@t6rM&whjaivFKAeBKjHH!N#`m zh_=@K1L3IlF6gZa3Z7mFT(8$r$7I?0@gjmO{iU$ScX>mGyVgC_TdO=Bl6ki#Z>t(F zSD{=ZWqAf4s>k~{n*ybGS$_%NqQB~M%J%jrJ2(b)pBa9Lntt59*9HTk2%R$DqC32U z7?0w8O$&AKflxFBU4GVhww&-S_WP)NzCI5{H3JYt@KX1(l>#ImZ+w~3_#UNzW>F8sD z6j{^W%nr={X5?siVM@kWcS+1npwo`Rs!f|%+5lFltO=>9RqSD(H#?z4Ury9e1lq}9 z$jAUk)n(+x!r3#~sD6?0hXqnVhL`Vhu(G6 zQrPm@N4A4n_VFWP;_pK#<9Jgt7_$3W^Im3Zw<74$P5BP9ZTapJW90_%el>tk;7Qyp z=>1HIiF^awwR5!}Y%>3ouRO6+mz&_@H@ape-W)^s07h-x9oA^2ad@XsI(%ej-7)$y z#Cqd>nn(p!MN{YR5UqO%I!dLsF*@!Iq^#C!ot3aG4d>T3CUlTv} z6iL>}xBCnYLS7?_U*HdpP+Asv+A};M@|fw7BkB?-mX|yzL#nT{H9M!4siijCN+rDs$a`6ha&^tJGSa`*3;ffc^s0R1)iUtIPJwQT4=sO;a$Ikbv zNo&GMQ5PKcm^fDfWdG0K2(2TR*hvI!U_{FBWpc!gC=e6sBF{af2I$~oB>sNPw41eWBvq*Qs~X^$ANZ7;~e(o z1e@F;!LD96*jSaZo5#-1wSjQ(>pcTxLgZBpgaslyKQvBK(2CmIB0dO^-v$m4pQ@7% z7SR!pf5tV3J25ajB1`Z)Y7HE`NAiG<#o%hsQBore#fSgoQZ%yp++wA8Wv*G*o=04s zB%&tKfMfce<+CcbJ1|fQ-aLjz73tlmGDDpzEt>68LjaG2b6};VaJNh0`)O;HN{J`( zc14w8U=TH}PX>O3+6T;_t>Q}t8KXNLngo6tCzBW;%4D@;g zRp_+Dr8G{Y?zZ7FBtM1C#Ze_>QNGw z0o3cpdqk|={5Ba05qHDANrT{g4aJyvu((tL2IX^<@5g5vxlY!QE^<(2)$6&F!KP1} zaO-aRRqRg7rAZ$eNe#?pBJyC15#3-pc;tnaR1(6-WXWVA0cG<$^SRYfG#R0Y{ zvu*zri8S=ty>@%M{zY=vpo`Rc^{x8cYAEWenC;c`Q;N@drB8fz(p4SL?x4;x9ue2h zGgGgH=WKp+8(IprABIbtLwjbHBU~7`+?)C)B48jh8kyoW-?Xqwp23jg+wP}z2gh42 zu+n`j))kosN%-2}@e~&&aoGA@!-Kgi?e79yofK*Wy=(AIj?^mioI3RdQO!{B6um^0k0f?r2RORd1H(-HXi{e2`MGE)T|3E4Qfx<5z|HbzeC=f5 zQcmc6AV`RsmG$OWQmX1K@^``Cpj6Xavyway_J_i0g||Nik%Ww&?3zhkBVBGk8J+8W z#fS6IC&s$wPdRa)LA6vQjNw}96S9g;BGv|ngK^Q_tk{`ndPw4W4hIjzM`Bck)EuMK z`MBC84ZFOBBa@d(bjJIX&KsDV8yEAF8GutUSO6Q17xZI2+ATaiwjbY#UDOZ4Vhebp zLF#weM`ZI35BOq^+kvQZ@7{qGy63e;F;XX3C_qndkYm;=2vR^Ue95q@co7D3Mb1SM z3e*CX<@M&!vuliCK#1(4_vtXxprfxgyal%;!5wge(^V;7tm!?QYuAGDIh#pY& zX~XV66F0@I&4%~I2(;(rTHksR&%Kbf%sH6aZEK_Mn8%yp!<@H&u(Z)73$p2m+^!hX z&gr6jP6F`?naXg2y^DzaF-$<`qj-EvuTn{CPU|q2L307C!lID)hWL#wjjN`_g;O#r zpXtoq$})b;@gu!wn%PCXHt5V5jMnLqfO8~8q)!}Ex?nLd`pWdlNm3$PIOxJj2G^~w zU#0N|mMrdS>;8p}?e&r5)D`-fTo2a<$sYY%d$@pR1#Nsl)DC-OA`7SfDz9l}ifhOnaW} z4F`uiWI;Sl0LA385fgrilD(eiVkcK}Fziy9<|it~;b0_vsq1zO>C>BlgdkYec&~OMe8<8^n=0UGRrSkSsa8bq2s--a?ALNP zzOjj%&BRA9(}PGhL`onWh*GvEN-`PH`r`P5c8(CN*=Hz_Tj!nX*o_qXcoP-Co47)E zZOlxgJA-&X#NX6TvDEt;{`G-Jy3OG!zP44kG-x=dm-n{?svZT>DM~YZxt`E0i%$%Kgl3T{CN1;JdbrwYtF zOaU+lsQOWPDPf|FU45_v!AGW!%FTp|_7Y2N>!Nqbo%*Y?ov$^!*DirqNDF8MovfL= zo}+g!2YOfc%rfuyK^2zXw=<*F8=kvjh7ptw+Ik0|Ci&N@0fL9|Q8gO$P%U(^Gb~tO zbj;nPIF4U_fk~@xa3Pq6UT$sQaSO;r|x=6?SbQUotj#h z?IvThzWxfmOWP790!-Tdk{DzH{PZvm1Df83{P?l>F%YRXvrh19n>}{J(+C-de4HqF z__KmK_6Q(#^&%}{ZJ)T~q1%09&b6VwTjWA(fq;HIC93jn+~5JGh9*|T8Q~&d?XkF0 zDl^if?tgau1kr}_AZJc7$hn?!ex$6AY)JNghBp6Rw@iIK1Vdy+(|?|Spacs^jpC^%6L?70!0EKY;LvB?e>|5! z=jA^aFVLg_$o?$#KaJRan_N>{QHg4EBUt~qwt(KPKbPMgSvK_T^<4_xzdT<5YE~l# z70t$i)>RJsW7YikMMnf|OsbqTs{a4BpIu-uLw-8Mp33xpOtb&Ksyre53+<{XYwlhb zrGj&=%hrvD7d$3MXOxUrj&ui;7EnFpMsI_U^mXrPEZg`yEdZAItxx_)nF2c|$9;mYrgt=dqov0O)6|R~xSzC_L<6P`QY!NgDE|Q{ zSxiqK#mvQ&X(z0--h>VuCMsXF&LSVl^WO}HWw61k9x!GF%PZulh}6gTd3Rse1i)Mda(=`o5c;EUu}gpdz>!pb869zdy5@E2z6k)AbZPOql)LiU-ALAUg4ZH`;t4tVq1~+L zeahMHKzOfABgwS=Kz<_U%}vF>HW9AgN2R@RcsxzG=v^K$=lP8{@!WsUc3^qeWNDhS zn!wfK;R~qt!&x*}f}9`Aj>=ST-vqC0N{_&HgkMLHFh z3Yr6bi;MV`P5mzkuzC8&(3p`S`9Fk7@(*9QFw}WB+`Xo26b|lvXHlpZho8$MFD((* zTR+)M%+D)(m)@HQ13NhFg`N9Q=u2AYY*3)Tc!XMp#02zo<3iF%|iN+rAOwL zEuc>GBDxVaS39Bm_|o z!}7{o{v{p%^_iFne0KiDSGK<=XOk3$T!HXVc<|jBVUqW*2Q$6BDdT0!#J9;Gl23M% z)LgbWY-#5-OHrj&l30F*ld(G%PO&@E^5Y61B$gnLV5tgIGMVSU)XQ1xsRC4=cCiw4 z6EniL(JXw5GQy6r=4io8sfauGeiB>(geP%LFJKbT|2uiW8$S$Upz9lwTS4aE!ug*s zn>fMp%X{qYDN8AVKdD87Za1CJJ+7@)k@%MYSEfMrVAEDcv`G4ka)6RZ1o+x1OcLK9 zRDN_Vw@WvZz7?4KOO^wL6X{gKF(MAxUraCW3>GS=+8%6_1syL*%%GeWAM!83-^Gpk zZI!#vrMok3^gTmOcPVQ4i29V_vy%aXKBPsx0bQGXrPAnixnK^rsw50>Ov1#n;9TfP zV_n;_ZFGFH`u%9imW;LHj3Kgk-%kBwgJlC#RYL9q(ez%2J=(W0qET<=p#@BDwXPQ;28Y!R9s)Eod;maV}n z4$f1BgUSf>X*5a$ie6*2utvCTgO9nC*I_xUYa+1e}rePUfhe# zacjCS=hFSf8lWges6e$_`50jdcvkO!7if8RU~~%YhA6ZAU#yoc34~O&n8PUS|FQTf zxCdO{D?G7=1^?>Rj|K%8N|3)EP5=L05`!fG121$e;4j)nKuxGXG6t`H+?1)Zo@h_zi^rJ!ANYnZK3#1 zB~;pkicg1?5PL-N2JUUUB{$l~B13wenpZghY*mSX|Rz0Caf(<4M}r-zw+-RuUmlzk%&hF#PjMRTU76xt;^U zO8nR>w?WU~iIZyk2KCI6vJ zOmSb)RP_#1V?diIM20gA^@?$P6ux38PxqR zbIL!L>Y720R_8Jiu5zj)p6Kt*JQ(VUq|_%ls~y^R-&T2y&d41j5Jtn*#3jou1V=6! zHyGU4y3hmo@l+W)H^sHp0C0CLH??eIDXLY2ADz1EFY$lN zdEh+=kRWhB{moNS(p`{k+JYYI5h*X|k?tIV&wn%5e4MXihW1UAry2X*&~OGLaNU1; zEUx=|l2TFRAw0&pwZ{Ka6O>8x$fkB!IIOr;Av}~@)Xx4M#^)^Gw4pEO5=pw_$zn2F z*#9FMfuN_05$Bx5*2Eg|@lsCQ9w_Y7Z=aLE0+#3@M;Xyw(|{+;Ep|3=C&7G4G7S51 z?24gGfwHXY*s!wAS06i59ZZuxJ)wlBm@3%+TwUiPG2M?0O(0<9wGmc*|62c};o+4n zQwF@2C0=0Gk-c<6H*`riHp71W!-{`2@)Z`)<68LUE-)e>Q02fD@BC)q0ZdcpH*|#c z_!`!#0ShT^G5i2LAjiNze1-R>@^d+~6H0c&-+&b$e{L#QB3E@z!ar0#Q?LIIm(EA} zI-ZKf3?-OUMc=nbY8&L^yx^%eD|5~_v%G5OY5hk;_{~D6-x8^;1nYbUXtz_L6*^B# z0#_A}{1&B%|5tPU+|kq=FL2zFD8bs7A7X9@4`=?f=9J2GwK(mp`^)gC+V-b*7TrhQ zIGHD@eHl_p&4s>2=Uo1;FsPD-NT=dBmta%^2ea1J18C1NpqRZlgpK;?oWIB2{XdaZ zeFi%@;tO!;v6(Ir>svBJa4`XHkAfrPTyf=*QyxhM-^*xr$BL8;=1AmKD!=eylgydZ z(doM+8XL_mYpS%4%%F{Q8uU_OgKPREQUBTrE^8d=z(Y=`^Jc1bG{45`_nRAijsZ2k z&Uqew?g+fX^l)P?Wu(i((Yo@N6rHFf`N+QI^sBdf z!~S1m*BR8*7H$#I(4<(XDkunuT#z261O-u)BA`?$8kz_MP+9^BK_W<3lwL0)Djmeo ziwH>Xy@Xy2A@meN^5QMZoi}ghdi4rb`S09;-()sh^tIJza)Gl0q#99bBDnsR&gY;b34ZQ zdFdpyJI^`}1lBvUXI14OrG8umR|%#|j;|K-Z<1L*lAw*KwqIQA@0o4?iB3ug8LPhu zKJqq~`ynvzXnwHy)63@*pX+}-iR@!8*DO|uG!p$c?c7yD#tvf#cQa2RrD04u=?BBd z?GyCykzM6O67<=O@+OVHg(w-+?JlZ&gF3y~)PkhXv%RC%Ju#1-kKnZv`~ zQbx7E#~vDZj_b!w6d9R&+uRsvQszka+}{xvS;6%|i;1l2BVYM6sl7bL=-=;@jGhnei{?y*be0$p}$1AacjbH*k$E~@b+_@MH)O*f%kw*nvdEbpAZ zM~!g!Xm*9^whHJCSigwJ0i+^S>2KL{%7@gz(_vD7Um+P7FxI_lM`qiU4{Nw}X=R=b zs-17!vUlsxlsAg?i8HkZMm=jxtaO;b8|Xv_CkJ@ddv1wQq&K{>sxiTYw1H!`d(g15 zId;FZH(%Y+V(dM0ueaGDqph3fHKpy*#*fCN)jE>+zJ2}UII|U6#a+$R`@xc-iIrH1iz{>9I4On%URqs$+ptV9>DIZ~ zbTCP3+1~%D=?$jcullu&(xHJcc!?bM5E~=<^I*??AkW~RCEC{JP@bl1JHJrvy>vPr zN&VQqfRu_a~REf>vuc zvF2X3$I6Z#@QU+AcNytI``RrYXWVhqE?-H;ns2F+{r{|#?KP&h%}Z^4rNZoNxhCAp zs7%wQ-xem!*nW$i4CHo`T>c~L@*#s!Yl%1fw1*x?nbq1D=X=jrjbC90%kxvPI$JxR z_Q~ySk)zOHoYP3ZPj_!7WDO>wEWx01Tc z^i^MkBkF|21DV;P_Ne}xVCFnV`oLJZh3I-#T)JD*X@;D0tLu(#cZ49c1$iUE(#NLzCNj5W>UsppJLb)GOA5*%cyz zzDv3bOYr~WdZ%BUeZh=(X|cyeJ>&T~`Vng=CI!C4fxHvZ!55WOD?}oN`h}9Z?~PVE z48LdIUdy;tV3zA-ATY42@>%t{8QBx57ZP$ai=>z{$xu zeP2_h+L;o=2ko6d0Xt;!Mg|(k{ETFBE9y2;=!c*$SLEJfJL({C5j^|Z6HYbd=y_|%o2Q>!NvMH#XB!rO*pm0L!8H|VOu zh{&eos4tJDS=F=JBz^f#d6g%*aRbU0dcLj1h8t&}am-ny@j3bdEe~7Fy2R{$L8J3? zT^$2Nb%L1SG0E1w5wP|jg`Rndy-qLxg8(^^CRyguGa@$);~KB6`M_W*RWfO6h#jKN zA7RSlg5SIQU7mnT6BUZEec5*+k1l#8xGPGSLj*aV4TfH1v?@FMN{wz_4t>e-jNuG^ zsJHaVEaGBZ1=_dlT#r!I%+4Jvkk4S2?8C1$vbZU}HpxBG&1XNuPuSbXjVoIxWA4~} z1!*%DY#AWPv2L6kWqvDMAZGd1Yf2Gpr5=11uTC#}e7b!wM7+_?xZhs*V5-s!J0u9H zE-ovxpLEjg@FhWMQ#CLUl{n`v_H69dYb>Oe)Y>9MqtYmc))XhqJ4ORJaqsow9q#Ic z(N!-dBjYKgOK z-))~dIsyvhgNEo7_poc~icWqw=C+x{-o5^2@MQm+d206FW~i>?6>HPJA3}qk)g0OR zPb?yHlHg*rEq3gwHvUM3sV`_d zi|wCklb7o^RjccnD=yi8gak@;=_gCLq%qB~@_1JSp(r<8hReQ<6A_<78v5Ln zWeoF#E*DEI6X5xOmV-3Gx8G)r}ydLDY?Cvkg+} zI2ukOq~me8{eBuzE_^)STLqUrsgHP4nkrh>HDgV1_j$_JuaN|)_6Puj!Am<0Q)%Yw z1;bb$K+Tikf`-8_Y0C6%4>av%0qb%W2mzXKA!t$a=u4pf7)~3}qt~w?` z@{3us)H3tzX3lZASR*Q%yXse^%k3n?WB)5@hSvA;sxdv{VN6Is4j+;|bl66_A$V~d z4H5a^KR20dgV^6eBdWqL%d4l9)FFV&j_w(Co#6R;o&CYWT3^DXkKyk0G(&P^rQdU| zE@{tMg7d?f4p`bhLTE30PhRq#t(QYHAbkc8!yRgAmj2tUS%t2>Fuy z{aS}08c_zlXE2+e@9H!@IT@MtZdm1~O@YY78hjm%iUrY5y`4RFlepmRspK58r1l`= zDhd_(0&{;z@^mRXOY1+RAdoqa_YiNA*UKGegzu9XpIHFtR2Q~0Ss8<%1$4k_VM|o{ zP!Ra3=S$t*yM)SAO@E~eg~Ng|-@?3v0=BvRU4VrKX{rsZJ?=>x&qc!9ilG31b4td1 zX}9|1#DqF<*>-GO8RNgz$O+O|Y>re0?XAjJMFHYAod#mB6(Wkz(F#M`BTi-994&2U zztjJ&S|+`&UO@Q!w@j+I}4_LBc-n~{peBC$IFz+XVvS46m{1;{67dA`ogt=%Cm9o{%O+)3f3JJgr zy}5Qur1X9?`=M%cO=aq6>gNaW@U2cv?cN%7YpMGor>M{rl(>~nqlC-Ai+$Dwkap#D z60B+oo5-4V7$x>so()eT7G@&#ytuc!m_G7eEN=_PV=JvHQtUHHAB<`pg!w zNAsbm#>5wwKq-4O>7G&m=q~cy-_?VkK(kEP)*t=wUB`tsSEVf$gHQ#5S^3{SA|#@~Ax;5@p7V{sK&Vtl(?V-h^xkrBw0jw3vKDEy z9yqi3^~j5@FkTM2Ex&T16v4M^@NnOa*(2@^EvV<=3o6shggq<&H`(JW8B^nEw;I9} z>@G@j+5kaXh)283BrDJGXL$oL5<7F821#amG6u!0J7t$lh4zU=cN)XPRU66=$0DH< zA7N_pegYUF^ifV&tL8n9&ww^5^W`-2Z=%2I8bw-6DJ`mL61{LDr+BJ^>G0~Q)O(NG z@LHG=W zD6f#QQKzjpa-zQ|Uv?EZLh#--WOBQd;u4%GV>Nc2!k7D06}HQ0oF$vj51Y9Pej6oj za8}LNT__W~zdwn_zy*7q?y-vVc@1JA!bSVZ7z(Sjf>_A2s{r3u8#pxg$vS8eW^_Cx ze&TVNY+Y7u>DH&gF*0Q<9lqHrUIb`Z-46h3lpEPbQVqXtk-uin^8+4f3b$1KgYO7zBCyqP0=*W%j_sktWj!nX6ZNtfkt1 zN$X}dj88pNTaZ_kp{h?u`==ciL0S%(?Co5=cEE-jIN(H7p>-8ytqLMiNm+GD%YunuMfMcM&xoGMdIWjjPi=9we z(|dcC{9I@a1fw#AFT2`|eaj8m1JqE3Nwe{FO<~zSYgAlQyQ6z8w-=l;KO-}nK3N;E zm0PIGE*Pb*>cv#I@#6eqr0qwp^1Y&oF2abAbSxm;ELy4yscxO#H|0B8r+KOMt8mNa zGX1|DH`v40y;%9fS4e!UTt$dieDc1~imaM~Y27~XsZI$YXM+!v2!|BOX5eMm?bzs! z4gbxS?S8KygHIC5A(2bHL>Y9geZLZfLl?9X-{YYInQ&9JY}8&X_VSmfUELqh|B7q_ ztnu-yN|`z7XUIjD8!wM8wM&l7o%s0dY-zd&^IOMAM{=TSZhUL0e@=^l`zL<=DamJ> z(hZVHvUM3WyNk>j(tCvYDdVRJsd?672>Tp-+87Ft{n&V0xe8;i8H(?q{I-B5AT->|*&d{jc8Rq~&_~i7zOctE{{Hi$t zYK5ri$8NvIDr2S)+jFI+g-<%)-ESWg?3{5~9g7r1ce+HHcoNpEv_8%_Ha=+yJ`vnE z(;}j#OWZd?tVI-wiv5TF3kMQq#kfSKDC;wtc>zcXnbS^E&dA}-F`OO+w&T3CF_&W& z$)U_ae4~Redd|xxV^iTtdYxF;izQ$cbvwRKhk{Fc>$R51*kKi7Piqyv7UL`it19%d zuA9J&lV%$BRWQ_@g>XlYHx0%uV}c(}X1m8SgaCD>S(Ig8zMFJ$asAwX4S|60zRZo6 z!Qs{x%_s2L?j0NWrY%#P!9Q_bvF;wcA!Oy?&%lYZq zw?m7eOZv~RLo44NFn6>C3X&TDbhq?BCM2)3j(h_g#i-S*1TBt>d6bMg&t_|O^!j4+ zEg>~KJl8ke3vKXgwhr>w>xNE6WszgWyU&8)o{#t196|f@L!HV>$^zzL<=fQ3OO`^_ zwB54Y@r0@fUnPna*1M6xx4vk;p=NL2jX=Mho2x>a8cEjRRV{s=9SE^g(vN?rQ=?rq zcxJn9k4oJaIy)f1!ZT2M$u~pFIplb7dd_Em4`bkazHF<1Xx?3=L$H89EOWO{v2?&b zX%wrwnm%VZy=+>ThN<*pyJIcv9^RcgL5ZfZgoCTMh&%MC8h#}6p_r;wXhZRfSc_=v zbM{v&xVckVRR(Dw*oJdXMX}<^K*EzaS8ZS7R1LoHLB(z_0B=?C8AUqBd>kAxTKHL& zywO0wfa-IO3F@#Jy82U?WLK}f;6xJA)!~~BFg#+NT4he=>`qhD)S~R-EQw*PpC3jY zt#t|w1*&%ik%s8#=5WwZ9hH+7*eY1W1y{A!{1)HgdMmm2?cvVN-;7kAKRI&L0pkgYWidC;VSBA&N}^dQz78<%9P2V>(% zVYh=$sdj8HMe5D*fDpT6IxbrfswG|#3!bR-ViTBN{MxD=vqz^R{ZzP+{*_v{P`&cu z^-RHICtggAsuA0jF>ska)+5Zs~@HqUo+2I8Kg7R3O zE4;DpFMf8GNYfs50BhpvH;w|xZG(3TtrKJz-9C@gcIv3{c_8B2RH_O_El@mcn98+@ z0*A>kJ~~l`*qpgZ9BIo(L+ZUPoA)*x#6W?lV>S(!jfJt4k%}eMF+*x;f4g`dG&K^XqGcgt@GnK5KBo z=1wmA9h(GYLPv;chBJ}TCIZ6yNR6GzfA6qVU>wMgyxG)lMXU-5nS58gF?ACHLLfVx z5cqYEZN4;*FqRHI|JBQ`5G&jcr#B}ImMZR^$NlG&69la+?W0ooBhn6UE3}o*?Wyfm z%o0DGrsl^Oefo-1h}XtLT#+FhP&qYGt$r!lb0I4JKI0LIFNp4YnmXKXz_2d6##y%1 z!_%{oAxAQI0qr-D_;zU*)tHUy%dj?qv+t}jfABUpKxpf$r^a4{$Dje2@jW)NBKBnCoTEJ)QDTfd6bm-PGI(ZYza$r$COGmGe(fi#Cp` zF`otRgn3u(j2a-IOI_$_xPNH;#xB8!>u~ch)5Zpff~pJa-s5jkECBa4+0>Ei_Ohq)$<&SI$-Nt z{r>lkZV`{I5v)#Y77cR_Nmi{4A`A0|E!}ZoI_6A7w* zy%_Nu$7&o^V)>HSley!x#dc?vQ)X46T`wtqEfMWUw9hzw{$|V}z=NS4!YG43gupY%W-zy3CeiEfy)$V%UQPrwbDa!K3nPrc2*#~2WRv@=~ zraiNn4L{4bw0!WfB|eaf-(i;35@R5}#|CHfzB-6RZ}b_tD5vvVevNw>7%5>HZlxf= zp-`J1q(>=gzdGc7qAKpApSX+O2svs_xcN@6B~u9l+#LHdYW#AVgLhPWhe#*T_DeMS zYVFxirmCv319yg3*x$&6%P%e3d{E_Mdcte?jf@Jt`AoHix|3-ROQhZmm6xO(^6t1=x9CdXq4hTtW5)8550h>eRJjZfL2H?ze$EuO8yRN^j@@F|lkx{h6 zSW({oK>d;qsqQ_ew+K3KKhHZ@mN7d{Z|{kJmu2_5`Kp#K3-3i|l~ literal 0 HcmV?d00001 diff --git a/src/arr/ansible/unraid/vars.yml b/src/arr/ansible/unraid/vars.yml new file mode 100644 index 0000000..ee9829d --- /dev/null +++ b/src/arr/ansible/unraid/vars.yml @@ -0,0 +1,59 @@ +--- +# Unraid-specific overrides, merged over ../../common/vars.yml (these win). +env_defaults: + # SQLite lives here — all four production databases (primary, cache, queue, + # cable) plus Active Storage. Shelfarr has no external database, so unlike + # forgejo this directory *is* the application state, and it is also where + # everything configured in Admin → Settings is kept. Back it up accordingly. + SHELFARR_DATA_PATH: /mnt/user/appdata/arr/data + + # CHECK THESE AGAINST YOUR OWN SHARES before the first deploy — they are + # conventional Unraid layouts, not something read off nas1. + # + # `/mnt/user/...`, never `/mnt/cache/...` or `/mnt/disk1/...`: the latter + # bypass the user-share layer and strand files where the mover won't find + # them. + # + # See the note in docker-compose.yml about keeping downloads and the library + # on one mount — as three separate binds, imports are copies rather than + # instant moves. + SHELFARR_AUDIOBOOKS_PATH: /mnt/user/Media/Books/audio + SHELFARR_EBOOKS_PATH: /mnt/user/Media/Books/ebooks + SHELFARR_DOWNLOADS_PATH: /mnt/user/downloads + + # 99:100 is nobody:users — what Unraid owns share content as. Upstream + # defaults to 1000:1000, which is a normal Linux desktop user and wrong + # here; files written as 1000 are invisible to everything else on the box. + PUID: "99" + PGID: "100" + + # `auto` chowns only when it has to. `always` would fight Unraid's own + # permissions handling on every restart; `never` risks a container that + # can't write to its own appdata after a share rebuild. + CHOWN_ON_START: auto + + # Prowlarr's config.xml, indexer definitions and SQLite database. Sibling of + # SHELFARR_DATA_PATH under the same stack directory — one appdata folder per + # stack, one subdirectory per container. + PROWLARR_CONFIG_PATH: /mnt/user/appdata/arr/prowlarr + + # Container clock. Prowlarr timestamps its search history and enforces + # per-indexer rate limits against it; Shelfarr takes its time from the Rails + # default and doesn't read this. + TZ: Europe/London + + # The stack's Docker network — both containers join it, which is how + # Shelfarr resolves `prowlarr` by name. + SHELFARR_NETWORK: caddy-net + + # --- Unraid Docker page presentation (net.unraid.docker.* labels) -------- + # + # STACK_ICON is normally left to the compose_stack role, which points it at + # /mnt/user/appdata/icons/.png when this directory contains an + # icon.png. Uncomment to use a hosted icon instead of committing one: + # + # STACK_ICON: https://example.org/arr.png + # + # PNG only — Unraid renders its placeholder for SVG and nothing at all for + # WebP, and a URL it can't reach when the Docker page renders fails the same + # way, which is the argument for committing the file. diff --git a/src/arr/common/vars.yml b/src/arr/common/vars.yml new file mode 100644 index 0000000..69ffa1d --- /dev/null +++ b/src/arr/common/vars.yml @@ -0,0 +1,64 @@ +--- +# Portable config — values that hold regardless of where Shelfarr runs. +# Platform-specific values (filesystem paths, the PUID/PGID the container +# writes as) live in ../ansible//vars.yml and are merged over these +# at deploy time. +env_defaults: + # Pinned rather than `latest`. Upstream tags releases YYYY.MM.DD.N and the + # image tag carries no leading `v`, unlike the git tag — `v2026.08.05.1` is + # the release, `2026.08.05.1` is the image. One tag pins both the app image + # and the (currently unused) Libation companion. + SHELFARR_VERSION: "2026.08.09.1" + + # Published port. 5056 is upstream's convention; the container listens on 80 + # internally, which is left at its default rather than being made + # configurable — there is nothing else in the container to collide with. + SHELFARR_HTTP_PORT: "5056" + + # SECRET_KEY_BASE and the three ACTIVE_RECORD_ENCRYPTION_* values are + # secrets — sourced from Vault (homelab/arr), never set here. The + # encryption trio is not just a session key: it encrypts every credential + # Shelfarr stores (indexer API keys, download client passwords), so losing + # it means re-entering all of them. + # + # RAILS_MASTER_KEY is deliberately absent. It belongs to the + # config/credentials.yml.enc compiled into the image, so any value you + # generate yourself fails to decrypt it and the container won't boot. See + # the note in ../ansible/unraid/docker-compose.yml. + + SHELFARR_SETTING_OIDC_ENABLED: "true" + SHELFARR_SETTING_OIDC_PROVIDER_NAME: Authentik + SHELFARR_SETTING_OIDC_ISSUER: https://auth.turtlesystems.uk/application/o/shelfarr/ + SHELFARR_SETTING_OIDC_CLIENT_ID: jf1IKMxxYjgZxZikvONdFfOTauFrFiC4AYmJEbmN + SHELFARR_SETTING_OIDC_SCOPES: openid profile email + SHELFARR_SETTING_OIDC_AUTO_CREATE_USERS: "true" + SHELFARR_SETTING_OIDC_DEFAULT_ROLE: user + SHELFARR_SETTING_OIDC_LINK_EXISTING_USERS: "true" + SHELFARR_SETTING_OIDC_AUTO_REDIRECT: "false" + + # --- Prowlarr -------------------------------------------------------------- + # + # LinuxServer's tags are the upstream version plus their own build suffix + # (`1.37.0.5076-ls117`), and the suffix can't be derived from the release + # number — so a pin has to be read off the registry rather than guessed. + # `latest` until then; check + # https://github.com/linuxserver/docker-prowlarr/pkgs/container/prowlarr and + # replace this with the full tag, which is what every other version in this + # repo does. + PROWLARR_VERSION: latest + + # Published port. 9696 is upstream's default and what the container listens + # on internally; kept the same on the host so the WebUI label, the LAN URL + # and the docs all read alike. + PROWLARR_HTTP_PORT: "9696" + + # PROWLARR_API_KEY is a secret — Vault (homelab/arr), never set here. + # Generate a 32-character hex string: openssl rand -hex 16 + # Prowlarr generates its own into /config/config.xml if this is left empty, + # which works fine but means the value is only discoverable from the UI. + + # STACK_ICON_PROWLARR is supplied by the compose_stack role, from the + # committed ../ansible/unraid/icon-prowlarr.png — not set here. Uncomment to + # use a hosted icon instead, exactly as with STACK_ICON: + # + # STACK_ICON_PROWLARR: https://example.org/prowlarr.png diff --git a/src/arr/terraform/README.md b/src/arr/terraform/README.md new file mode 100644 index 0000000..92da798 --- /dev/null +++ b/src/arr/terraform/README.md @@ -0,0 +1,11 @@ +# terraform/ — arr + +Not built, and not expected to be. Terraform's only job in this repo is +creating Proxmox LXCs; Shelfarr runs as a Compose stack on nas1, where the +host already exists. + +This folder is a placeholder so the app keeps the same +`common/` + `ansible/` + `terraform/` shape as every other one. If Shelfarr +ever moves to Proxmox it would need an LXC defined here and wired into +`build/config/terraform/main.tf` — see `../ansible/proxmox/README.md` first +for why that move is less straightforward than it looks. diff --git a/src/authentik/ansible/kubernetes/helmchart.yaml.j2 b/src/authentik/ansible/kubernetes/helmchart.yaml.j2 new file mode 100644 index 0000000..ec35b7b --- /dev/null +++ b/src/authentik/ansible/kubernetes/helmchart.yaml.j2 @@ -0,0 +1,130 @@ +{# + Managed by Ansible (roles/k3s_app) — do not edit on the node. + + A HelmChart CR for k3s's bundled helm-controller, the same mechanism every + cluster service here already uses (roles/k3s_metallb, k3s_monitoring, + k3s_postgres, k3s_cert_manager). The difference is only in who owns the + file: those roles each carry their own template, whereas this one belongs + to the app and roles/k3s_app renders whatever *.yaml.j2 it finds here. + + This is the k3s analogue of src//ansible/unraid/docker-compose.yml. + It is not static the way the compose files are — Helm values have no + ${VAR} interpolation to defer to a rendered .env, so config is substituted + at render time from `app_config` (common/vars.yml + this platform's + vars.yml). What survives from the Unraid convention is the part that + matters: no secret is ever interpolated into this file. Secrets are + referenced by secretKeyRef against the Secret roles/k3s_app renders from + Vault, so this manifest stays 0644 on the node and safe to read. + + No Redis anywhere below, deliberately: recent Authentik no longer requires + it, and the chart carries no Redis dependency (its only conditional + subchart is Bitnami Postgres, which is disabled here in favour of the + cluster's shared CNPG instance). +-#} +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: authentik + namespace: kube-system +spec: + chart: authentik + repo: https://charts.goauthentik.io + # Chart version and app version are the same string upstream, so + # AUTHENTIK_VERSION pins both. See common/vars.yml — this must match the + # version the database was dumped from at cutover time. + version: "{{ app_config.AUTHENTIK_VERSION }}" + targetNamespace: {{ app_config.K8S_NAMESPACE }} + createNamespace: true + valuesContent: |- + global: + # Explicit `env` entries rather than an `envFrom: [secretRef]`, for a + # precedence reason rather than a stylistic one: the chart builds its + # own env for the values under `authentik:` below, and container `env` + # deterministically wins over anything arriving via `envFrom`, whereas + # two envFrom sources resolve by list order the chart controls. This + # way the non-secret half stays readable as ordinary Helm values and + # the secret half is unambiguously authoritative. + env: + # Signs session cookies. Carried over from the Unraid deployment, not + # generated — a new value logs every user out the moment DNS flips. + - name: AUTHENTIK_SECRET_KEY + valueFrom: + secretKeyRef: + name: authentik-secrets + key: AUTHENTIK_SECRET_KEY + - name: AUTHENTIK_POSTGRESQL__PASSWORD + valueFrom: + secretKeyRef: + name: authentik-secrets + key: AUTHENTIK_POSTGRESQL__PASSWORD + + authentik: + log_level: info + # Off: this is a homelab, and the default ships crash reports to + # Sentry. + error_reporting: + enabled: false + postgresql: + host: {{ app_config.DB_HOST }} + port: {{ app_config.DB_PORT }} + name: {{ app_config.DB_NAME }} + user: {{ app_config.DB_USER }} + # password comes from global.env above, never from here. + + # The chart's bundled Bitnami Postgres. Off — this cluster has a shared + # CNPG instance (roles/k3s_postgres) and an app getting its own database + # server would defeat the point of having one. + postgresql: + enabled: false + + server: + replicas: 1 + resources: + requests: + cpu: 100m + memory: {{ app_config.SERVER_MEMORY_REQUEST }} + limits: + memory: {{ app_config.SERVER_MEMORY_LIMIT }} + + ingress: + enabled: true + ingressClassName: {{ app_config.INGRESS_CLASS }} + annotations: + # cert-manager watches Ingresses for this annotation and creates + # the Certificate (and hence the Secret named under tls: below) on + # its own — no Certificate resource of ours to keep in sync. + cert-manager.io/cluster-issuer: {{ app_config.CERT_ISSUER }} + hosts: + - {{ app_config.AUTHENTIK_HOST }} + tls: + - secretName: authentik-tls + hosts: + - {{ app_config.AUTHENTIK_HOST }} + + # /media — uploaded icons and flow backgrounds, the only Authentik + # state that isn't in Postgres. The same PVC is mounted by the worker + # below; see vars.yml for why one ReadWriteOnce volume across two pods + # is fine here and what it costs. + volumes: + - name: media + persistentVolumeClaim: + claimName: authentik-media + volumeMounts: + - name: media + mountPath: /media + + worker: + replicas: 1 + resources: + requests: + cpu: 100m + memory: {{ app_config.WORKER_MEMORY_REQUEST }} + limits: + memory: {{ app_config.WORKER_MEMORY_LIMIT }} + volumes: + - name: media + persistentVolumeClaim: + claimName: authentik-media + volumeMounts: + - name: media + mountPath: /media diff --git a/src/authentik/ansible/kubernetes/media-pvc.yaml.j2 b/src/authentik/ansible/kubernetes/media-pvc.yaml.j2 new file mode 100644 index 0000000..ff319ef --- /dev/null +++ b/src/authentik/ansible/kubernetes/media-pvc.yaml.j2 @@ -0,0 +1,33 @@ +{# + Managed by Ansible (roles/k3s_app) — do not edit on the node. + + Authentik's /media volume, claimed here rather than left to the chart: the + chart has no key that creates a PVC, only `volumes:`/`volumeMounts:` that + reference one, so something has to declare it. + + Owning it separately turns out to be the safer arrangement anyway. A PVC + created by the chart would carry Helm's ownership metadata and be a + candidate for deletion whenever the HelmChart CR is uninstalled or the + chart's templates change shape across an upgrade. This one is a plain + manifest whose lifecycle is its own, so `state: absent` on the app — which + deletes the manifests and lets k3s garbage-collect them — is the only thing + that touches it, and roles/k3s_app/tasks/remove.yml documents PVCs as + deliberately surviving a default teardown. + + WaitForFirstConsumer on the local-path StorageClass means this stays + Pending until the first Authentik pod is scheduled; that is the provisioner + working as intended, not a failure, and it's what pins the volume — and + therefore both pods — to a single worker Pi. +-#} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: authentik-media + namespace: {{ app_config.K8S_NAMESPACE }} +spec: + accessModes: + - ReadWriteOnce + storageClassName: {{ app_config.MEDIA_STORAGE_CLASS }} + resources: + requests: + storage: {{ app_config.MEDIA_STORAGE_SIZE }} diff --git a/src/authentik/ansible/kubernetes/vars.yml b/src/authentik/ansible/kubernetes/vars.yml new file mode 100644 index 0000000..7c61b2d --- /dev/null +++ b/src/authentik/ansible/kubernetes/vars.yml @@ -0,0 +1,63 @@ +--- +# Kubernetes-specific overrides, merged over ../../common/vars.yml (these +# win). Consumed by roles/k3s_app — the same "portable config + platform +# overrides" split compose_stack and lxc_app apply, on the third platform. +env_defaults: + K8S_NAMESPACE: authentik + + # --- Database --------------------------------------------------------- + # + # CNPG's read-write Service, by in-cluster DNS. This is the *primary*, and + # it follows a failover — CNPG re-points the selector when it promotes the + # replica, so the name stays correct without Authentik being restarted. + # + # Deliberately not the LoadBalancer address (192.168.50.81): that exists so + # the Ansible controller can provision databases from off-cluster, and + # sending in-cluster traffic out to the LAN and back would put MetalLB's + # L2 speaker in the path of every query for no benefit. See + # roles/k3s_app/tasks/deploy.yml, which uses the LB address for exactly the + # one thing that needs it. + DB_HOST: shared-postgres-rw.shared-postgres.svc.cluster.local + DB_PORT: "5432" + + # --- Ingress ---------------------------------------------------------- + # + # k3s's bundled Traefik, already running on 192.168.50.80 (MetalLB's first + # pool address). Nothing here pins that IP — the Ingress attaches to the + # class and Traefik's own Service owns the address. + INGRESS_CLASS: traefik + + # The ClusterIssuer roles/k3s_cert_manager creates. Must match + # k3s_cert_manager_issuer_name in inventory/group_vars/k3s_cluster.yml. + CERT_ISSUER: letsencrypt + + # --- Storage ---------------------------------------------------------- + # + # /media holds uploaded application icons and flow backgrounds — the things + # that make the login page look like yours rather than stock. Small, but + # the one piece of Authentik's state that isn't in Postgres, so it has to + # be copied across at cutover (docs/authentik-migration.md) or the migration + # is visibly incomplete. + # + # ReadWriteOnce on k3s's local-path provisioner, and both the server and + # worker pods mount it. That works without any affinity rules of our own: + # local-path PVs carry node affinity, so once the volume binds to whichever + # worker the first pod lands on, the scheduler is obliged to place the + # second pod on that same node — and RWO permits multiple pods per node. + # The cost is that both pods are pinned to one Pi and neither can be + # rescheduled while it's down. Acceptable for a homelab SSO that is already + # a single logical instance; the fix, if it ever matters, is + # ReadWriteMany-capable storage, not a second PVC. + MEDIA_STORAGE_SIZE: 2Gi + MEDIA_STORAGE_CLASS: local-path + + # --- Sizing ----------------------------------------------------------- + # + # Pi 4, 3.8Gi usable, already running CNPG + Prometheus + MetalLB. The + # worker gets more headroom than the server because it's the half that runs + # migrations on startup and processes outposts/policies in the background; + # it is the one that gets OOM-killed if these are set naively equal. + SERVER_MEMORY_REQUEST: 384Mi + SERVER_MEMORY_LIMIT: 768Mi + WORKER_MEMORY_REQUEST: 512Mi + WORKER_MEMORY_LIMIT: 1Gi diff --git a/src/authentik/ansible/proxmox/README.md b/src/authentik/ansible/proxmox/README.md new file mode 100644 index 0000000..e3e3773 --- /dev/null +++ b/src/authentik/ansible/proxmox/README.md @@ -0,0 +1,15 @@ +# Authentik on Proxmox — placeholder + +Not deployed on Proxmox. Authentik runs on the k3s cluster +(`../kubernetes/`), having moved off a hand-made Unraid container — see +`docs/authentik-migration.md`. + +To add it here later: `vars.yml` of Proxmox-specific overrides, `install.yml` +of install steps, and templates for the app's config and systemd unit, plus +`src/authentik/terraform/` defining its LXC. See `src/forgejo/ansible/proxmox/` +for a worked example, and CLAUDE.md → "Adding a new app" — in particular that +an app must only ever be live on one platform at a time. + +This folder is kept empty-but-present on purpose: every app carries the same +`common/` + `ansible/` + `terraform/` shape whether or not each platform is +in use, so adding one later doesn't mean restructuring. diff --git a/src/authentik/ansible/unraid/README.md b/src/authentik/ansible/unraid/README.md new file mode 100644 index 0000000..921d8ce --- /dev/null +++ b/src/authentik/ansible/unraid/README.md @@ -0,0 +1,23 @@ +# Authentik on Unraid — deliberately not managed here + +Authentik **does** currently run on Unraid, but it was created by hand +through the Unraid UI, not by this repo. There is no `docker-compose.yml` +here and there should not be one: adding it would put Authentik in `nas01`'s +`stacks:` list, and CLAUDE.md → "Adding a new app" is explicit that an app +must never be declared on two platforms at once — they would share a Vault +path and a database, and two live deployments would corrupt each other's +state. + +The k3s deployment under `../kubernetes/` is where Authentik is going. See +`docs/authentik-migration.md` for the cutover: dump the database off the +Unraid Postgres, restore onto the cluster's CNPG instance, bring Authentik up +on k3s on the *same* version, verify, then move DNS. + +Once DNS has moved and the k3s instance has been trusted for long enough, +stop and delete the hand-made Unraid container. That teardown is a UI action, +not a `state: absent` run — this repo never deployed it, so it has nothing to +tear down. + +This folder stays as a placeholder rather than being deleted, per CLAUDE.md: +the shape is the point, and Authentik moving back to Unraid one day should +mean writing a compose file here, not restructuring the app. diff --git a/src/authentik/common/vars.yml b/src/authentik/common/vars.yml new file mode 100644 index 0000000..6437ce2 --- /dev/null +++ b/src/authentik/common/vars.yml @@ -0,0 +1,38 @@ +--- +# Portable config — values that hold regardless of where Authentik runs. +# Platform-specific values (namespace, storage class, ingress class, how the +# database is reached) live in ../ansible//vars.yml and are merged +# over these at deploy time. +env_defaults: + # >>> CONFIRM BEFORE THE CUTOVER <<< + # + # This must match the version currently running on Unraid at the moment + # the database is dumped. Authentik runs its Django migrations on startup + # against whatever schema it finds, and those migrations are one-way: a + # newer Authentik pointed at an older dump will silently upgrade the + # schema, and there is no downgrade path if the cutover then has to be + # rolled back. Deploy on the *same* version, confirm it works, flip DNS, + # and only then bump this in a separate commit. + # + # The chart version and the app version are the same string upstream (see + # ../ansible/kubernetes/helmchart.yaml.j2, which uses this for both), so + # this is the only place a version is written down. + AUTHENTIK_VERSION: "2026.5.4" + + AUTHENTIK_HOST: auth.turtlesystems.uk + AUTHENTIK_URL: "https://auth.turtlesystems.uk" + + # Authentik's own HTTP port inside the container. Not the published port — + # on k3s the Ingress terminates and forwards here; there is no host port. + AUTHENTIK_HTTP_PORT: "9000" + + DB_NAME: authentik + DB_USER: authentik + # AUTHENTIK_SECRET_KEY and AUTHENTIK_POSTGRESQL__PASSWORD are secrets — + # sourced from Vault (homelab/authentik), never set here. + # + # AUTHENTIK_SECRET_KEY specifically must be carried over from the existing + # Unraid deployment rather than generated fresh: it signs session cookies, + # so a new value logs every user out at the moment DNS flips, and the + # cutover stops looking like the no-op it's meant to be. See + # docs/authentik-migration.md. diff --git a/src/authentik/terraform/README.md b/src/authentik/terraform/README.md new file mode 100644 index 0000000..23773f6 --- /dev/null +++ b/src/authentik/terraform/README.md @@ -0,0 +1,11 @@ +# Authentik Terraform — placeholder + +No Terraform for Authentik. It runs on the k3s cluster, and the k3s cluster +is 4 Raspberry Pis that already exist — there is no guest for Terraform to +provision, the same reason `homelab-utils` has no `src//terraform/` +entries of its own (CLAUDE.md → "K3s cluster"). + +Terraform would only come into this if Authentik moved to Proxmox, where it +would need a module defining its LXC (template, cores, memory, disk, IP), +called from `build/config/terraform/main.tf`. See `src/forgejo/terraform/` +for the shape. diff --git a/src/forgejo/ansible/proxmox/README.md b/src/forgejo/ansible/proxmox/README.md new file mode 100644 index 0000000..491fd72 --- /dev/null +++ b/src/forgejo/ansible/proxmox/README.md @@ -0,0 +1,94 @@ +# ansible/proxmox/ — forgejo + +Native install into the Terraform-provisioned LXC (`../../terraform/`). Run by +the shared `lxc_app` role, which merges `../../common/vars.yml` with `vars.yml` +here, fetches `homelab/forgejo` from Vault, provisions the database, then +includes `install.yml`. + +| File | What it is | +|---|---| +| `vars.yml` | Proxmox-specific overrides — paths, service user, exact release, LAN address of the shared Postgres | +| `install.yml` | Packages, `git` user, directory layout, binary download, config/unit rendering, dump schedule | +| `templates/app.ini.j2` | Forgejo's config — the equivalent of the `FORGEJO__section__KEY` env vars in `../unraid/docker-compose.yml` | +| `templates/forgejo.service.j2` | systemd unit, replacing Compose's `restart: unless-stopped` | +| `templates/forgejo-dbdump.*` | Nightly `pg_dump`, script + service + timer | +| `templates/pgpass.j2` | Credentials for that dump, mode 0600 | + +## How this is backed up + +Nothing is bind-mounted in from the NAS. Every path Forgejo writes to — +repositories, LFS, attachments, indexers, queues, sessions — is left at its +default under `APP_DATA_PATH` on the container's own disk, so a vzdump of the +guest is a complete copy of its filesystem state. Proxmox excludes bind mounts +from vzdump, so an NFS-mounted repository directory would have been the one +thing missing from the backup that was made to protect it. + +The off-box copy is the vzdump job in +`build/config/ansible/roles/pve_backup`, writing to an NFS storage on the NAS. + +**The database is the other half.** Postgres runs elsewhere — the shared +Postgres LXC at 192.168.50.54, a different guest and therefore a different +archive — so a restored container would come back with every repository and no +issues, pull requests, users or permissions. `forgejo-dbdump.timer` runs +`pg_dump` into `FORGEJO_BACKUP_PATH` on the container's own disk, which puts +the dump inside the same vzdump archive as the repositories it belongs to. One +artifact, one restore. + +That only holds if the dump finishes before the backup window, so the two +schedules are a pair: + +| | Set in | Default | +|---|---|---| +| Database dump | `vars.yml` → `FORGEJO_DB_DUMP_ONCALENDAR` | 01:30 | +| vzdump job | `group_vars/proxmox_nodes.yml` → `pve_backup_schedule` | 02:00 | + +Move one and move the other. Retention is independent: `pg_dump` output is +pruned inside the container by `FORGEJO_BACKUP_KEEP_DAYS`, while the archives +themselves are pruned by the storage's `prune-backups` settings — Proxmox +prunes archives, not their contents. + +### Restoring + +1. Restore the container from the archive (`pct restore`, or the UI). +2. Recreate the database and role if they're gone — + `ansible-playbook playbooks/proxmox.yml -e only_stacks=forgejo` does it + idempotently. +3. Load the newest dump from `/var/lib/forgejo/backups`: + ```sh + systemctl stop forgejo + zcat /var/lib/forgejo/backups/forgejo-.sql.gz \ + | psql -h -U forgejo -d forgejo + systemctl start forgejo + ``` + +## Upgrading Forgejo + +`FORGEJO_VERSION` in `../../common/vars.yml` is `10` — a Docker tag, a rolling +pointer at the latest 10.x, which is all the Compose stack needs. A native +install downloads one specific artifact, so `vars.yml` here carries +`FORGEJO_RELEASE` with the exact version. Bump it and redeploy: the new binary +lands alongside the running one, the `/usr/local/bin/forgejo` symlink flips, +and the service restarts. The old binary stays on disk, so rolling back is +editing `FORGEJO_RELEASE` and redeploying. + +## Differences from the Unraid stack + +- **SSH port 22, not 2222.** On Unraid, 2222 was a published Docker port + dodging the host's own sshd. Here the container has its own IP and its own + sshd, which Forgejo uses directly (`START_SSH_SERVER = false`) — it manages + the `git` user's `authorized_keys` rather than running its own server. This + changes the SSH clone URLs Forgejo advertises, so existing remotes need + updating after a migration. +- **`INSTALL_LOCK = true`.** `app.ini` is rendered in full from Vault, so the + web installer is skipped entirely. Nothing is left for Forgejo to generate + on first boot — which is why `LFS_JWT_SECRET` has to exist in Vault here + even though the Compose stack lets Forgejo invent one. +- **`DB_HOST` is a LAN address**, not the `shared-postgres` Docker network + alias — there's no `unraid_shared` network to resolve it through. + +## One platform at a time + +Both platforms use the same Vault path and the same database, so they must not +run at once. This was why `host_vars/forgejo.yml` carried an empty `apps:` +list for so long; nas2 has since been retired, the list is live, and the +bring-up runbook is `docs/forgejo-proxmox.md`. diff --git a/src/forgejo/ansible/proxmox/install.yml b/src/forgejo/ansible/proxmox/install.yml new file mode 100644 index 0000000..27cc872 --- /dev/null +++ b/src/forgejo/ansible/proxmox/install.yml @@ -0,0 +1,190 @@ +--- +# App-specific install steps, included by the `lxc_app` role after it has +# merged ../../common/vars.yml with ./vars.yml into `app_config` and fetched +# `vault_secrets` from the app's Vault path. The role handles enabling and +# starting the service once this file has put the unit in place. +# +# Template paths are absolute rather than relative: this file is included from +# a role, so a bare `src:` would be looked up against that role's templates/ +# directory, not this app's. + +# A native install renders app.ini in full and sets INSTALL_LOCK, so nothing +# is left for Forgejo to generate on first boot. A missing key would otherwise +# surface as an undefined-variable error three tasks later. +- name: Check that every required Vault key is present + ansible.builtin.assert: + that: + - vault_secrets.DB_PASSWORD is defined + - vault_secrets.SECRET_KEY is defined + - vault_secrets.INTERNAL_TOKEN is defined + - vault_secrets.JWT_SECRET is defined + - vault_secrets.LFS_JWT_SECRET is defined + fail_msg: >- + Vault path {{ app.vault_path }} is missing one or more of DB_PASSWORD, + SECRET_KEY, INTERNAL_TOKEN, JWT_SECRET, LFS_JWT_SECRET. See + docs/vault-secrets.md — LFS_JWT_SECRET is needed by the native install + even though the Unraid Compose stack lets Forgejo generate it. + quiet: true + +- name: Install packages Forgejo needs + ansible.builtin.apt: + name: + - git + - gzip + - ca-certificates + # pg_dump for the nightly database dump at the end of this file. The + # distro client and the shared Postgres LXC (17, from the PGDG archive) + # are close enough that either direction works, but the supported one is + # a client at least as new as the server — check this if the dump ever + # starts failing after a Postgres major upgrade over there. + - postgresql-client + state: present + update_cache: true + cache_valid_time: 3600 + +- name: Create the Forgejo service group + ansible.builtin.group: + name: "{{ app_config.FORGEJO_USER }}" + system: true + state: present + +- name: Create the Forgejo service user + ansible.builtin.user: + name: "{{ app_config.FORGEJO_USER }}" + group: "{{ app_config.FORGEJO_USER }}" + system: true + # A real shell, not nologin: this account is also the SSH login clients + # use for git@host:owner/repo.git. + shell: /bin/bash + home: "{{ app_config.FORGEJO_HOME }}" + create_home: true + comment: Forgejo + +- name: Create Forgejo directories + ansible.builtin.file: + path: "{{ item.path }}" + state: directory + owner: "{{ item.owner | default(app_config.FORGEJO_USER) }}" + group: "{{ item.group | default(app_config.FORGEJO_USER) }}" + mode: "{{ item.mode }}" + loop: + - { path: "{{ app_config.FORGEJO_HOME }}", mode: "0750" } + - { path: "{{ app_config.FORGEJO_DATA_PATH }}", mode: "0750" } + - { path: "{{ app_config.FORGEJO_LOG_PATH }}", mode: "0750" } + # Forgejo writes the git user's authorized_keys here itself. + - { path: "{{ app_config.FORGEJO_HOME }}/.ssh", mode: "0700" } + # Nightly database dumps. On the container's own disk on purpose — that is + # what gets them into the vzdump archive. + - { path: "{{ app_config.FORGEJO_BACKUP_PATH }}", mode: "0700" } + # Config is root-owned and group-readable by Forgejo: it holds the + # database password and the three signing secrets, and Forgejo has no + # business rewriting it — everything in it is rendered from Vault below. + - { path: "{{ app_config.FORGEJO_CONFIG_DIR }}", owner: root, mode: "0750" } + - { path: "{{ app_config.FORGEJO_INSTALL_DIR }}", owner: root, group: root, mode: "0755" } + loop_control: + label: "{{ item.path }}" + +- name: Set Forgejo release facts + ansible.builtin.set_fact: + forgejo_artifact: >- + forgejo-{{ app_config.FORGEJO_RELEASE }}-linux-{{ app_config.FORGEJO_ARCH }} + forgejo_release_url: >- + https://codeberg.org/forgejo/forgejo/releases/download/v{{ app_config.FORGEJO_RELEASE }} + +- name: Download the Forgejo binary + ansible.builtin.get_url: + url: "{{ forgejo_release_url }}/{{ forgejo_artifact }}" + dest: "{{ app_config.FORGEJO_INSTALL_DIR }}/{{ forgejo_artifact }}" + # Checksum from the release's own .sha256 file. Ansible fetches it and + # matches the line for this artifact — which is why `dest` keeps the + # upstream filename rather than being renamed to plain `forgejo`. + checksum: "sha256:{{ forgejo_release_url }}/{{ forgejo_artifact }}.sha256" + owner: root + group: root + mode: "0755" + register: forgejo_binary + +# Versioned file plus a symlink, rather than overwriting one path: an upgrade +# lands the new binary alongside the running one and flips the link, so a +# failed download can't leave a half-written executable in place. +- name: Link the current Forgejo binary onto PATH + ansible.builtin.file: + src: "{{ app_config.FORGEJO_INSTALL_DIR }}/{{ forgejo_artifact }}" + dest: /usr/local/bin/forgejo + state: link + register: forgejo_link + +- name: Render Forgejo configuration + ansible.builtin.template: + src: "{{ app_local_dir }}/ansible/proxmox/templates/app.ini.j2" + dest: "{{ app_config.FORGEJO_CONFIG_PATH }}" + owner: root + group: "{{ app_config.FORGEJO_USER }}" + mode: "0640" + register: forgejo_config + no_log: true + +- name: Install the Forgejo systemd unit + ansible.builtin.template: + src: "{{ app_local_dir }}/ansible/proxmox/templates/forgejo.service.j2" + dest: /etc/systemd/system/forgejo.service + owner: root + group: root + mode: "0644" + register: forgejo_unit + +# `lxc_app` starts the service after this file, but starting an already-running +# service is a no-op — it would not pick up a changed app.ini or a new binary. +- name: Restart Forgejo if its binary or configuration changed + ansible.builtin.systemd_service: + name: forgejo + state: restarted + daemon_reload: true + when: forgejo_binary.changed or forgejo_link.changed + or forgejo_config.changed or forgejo_unit.changed + +# --- Database dump ------------------------------------------------------- +# +# The vzdump job on the Proxmox node captures this container's filesystem, but +# Forgejo's database is not in it — Postgres runs elsewhere, in the shared +# Postgres LXC at 192.168.50.54, which is a *different guest* and so a +# different archive. Restoring the container alone would give back every repository with +# no issues, pull requests, users or permissions. So dump the database into the +# container's own filesystem on a schedule that finishes before the vzdump +# window, and the one archive holds both halves. + +- name: Render the Postgres password file for the dump job + ansible.builtin.template: + src: "{{ app_local_dir }}/ansible/proxmox/templates/pgpass.j2" + dest: "{{ app_config.FORGEJO_HOME }}/.pgpass" + owner: "{{ app_config.FORGEJO_USER }}" + group: "{{ app_config.FORGEJO_USER }}" + mode: "0600" + no_log: true + +- name: Install the database dump script + ansible.builtin.template: + src: "{{ app_local_dir }}/ansible/proxmox/templates/forgejo-dbdump.sh.j2" + dest: /usr/local/bin/forgejo-dbdump + owner: root + group: root + mode: "0755" + +- name: Install the database dump systemd units + ansible.builtin.template: + src: "{{ app_local_dir }}/ansible/proxmox/templates/{{ item }}.j2" + dest: "/etc/systemd/system/{{ item }}" + owner: root + group: root + mode: "0644" + loop: + - forgejo-dbdump.service + - forgejo-dbdump.timer + register: forgejo_dbdump_units + +- name: Enable the nightly database dump timer + ansible.builtin.systemd_service: + name: forgejo-dbdump.timer + enabled: true + state: started + daemon_reload: "{{ forgejo_dbdump_units.changed }}" diff --git a/src/forgejo/ansible/proxmox/templates/app.ini.j2 b/src/forgejo/ansible/proxmox/templates/app.ini.j2 new file mode 100644 index 0000000..b9e98d7 --- /dev/null +++ b/src/forgejo/ansible/proxmox/templates/app.ini.j2 @@ -0,0 +1,116 @@ +{# + Rendered by Ansible from src/forgejo/common/vars.yml merged with + ../vars.yml (as `app_config`), plus Vault path homelab/forgejo (as + `vault_secrets`). The native-install equivalent of the + FORGEJO__section__KEY environment variables in ../../unraid/docker-compose.yml. + + Do not edit this file on the host — the next deploy overwrites it. It is + root-owned and group-readable by Forgejo precisely so Forgejo cannot rewrite + it behind Ansible's back. + + Note what is NOT configured here: every path Forgejo stores state under is + left at its default beneath APP_DATA_PATH — indexers, queues, sessions, + avatars, packages. That is deliberate. All of it sitting on the container's + own disk is what makes a vzdump of this guest a complete copy of Forgejo's + filesystem state. +#} +APP_NAME = Forgejo +RUN_USER = {{ app_config.FORGEJO_USER }} +RUN_MODE = prod +WORK_PATH = {{ app_config.FORGEJO_WORK_PATH }} + +[server] +PROTOCOL = http +DOMAIN = {{ app_config.FORGEJO_DOMAIN }} +ROOT_URL = {{ app_config.FORGEJO_ROOT_URL }} +HTTP_ADDR = 0.0.0.0 +HTTP_PORT = {{ app_config.FORGEJO_HTTP_PORT }} +APP_DATA_PATH = {{ app_config.FORGEJO_DATA_PATH }} +{# The container's own sshd serves git over SSH, so Forgejo only advertises + the address and manages the git user's authorized_keys. On Unraid this was + Forgejo's built-in server behind a published Docker port. #} +SSH_DOMAIN = {{ app_config.FORGEJO_DOMAIN }} +SSH_PORT = {{ app_config.FORGEJO_SSH_PORT }} +START_SSH_SERVER = false +DISABLE_SSH = false +LFS_START_SERVER = true +LFS_JWT_SECRET = {{ vault_secrets.LFS_JWT_SECRET }} + +[database] +DB_TYPE = postgres +HOST = {{ app_config.DB_HOST }}:{{ app_config.DB_PORT }} +NAME = {{ app_config.DB_NAME }} +USER = {{ app_config.DB_USER }} +PASSWD = {{ vault_secrets.DB_PASSWORD }} +{# The shared Postgres publishes a plain port on the LAN and has no TLS + configured — see src/shared/postgres/ansible/unraid/docker-compose.yml. + Change both together if that ever gains a certificate. #} +SSL_MODE = disable + +[repository] +ROOT = {{ app_config.FORGEJO_DATA_PATH }}/forgejo-repositories + +[security] +{# Fully configured from Vault, so skip the web installer entirely — without + this, a fresh container serves the setup wizard on first boot and would + write its own app.ini over this one. #} +INSTALL_LOCK = true +SECRET_KEY = {{ vault_secrets.SECRET_KEY }} +INTERNAL_TOKEN = {{ vault_secrets.INTERNAL_TOKEN }} + +[oauth2] +JWT_SECRET = {{ vault_secrets.JWT_SECRET }} + +[oauth2_client] +{# Policy for signing in through an external OIDC provider (Authentik). The + provider itself is NOT configured here and cannot be: Forgejo keeps + authentication sources in its database, added once with + `forgejo admin auth add-oauth` — see step 8 of docs/forgejo-proxmox.md. + This section only decides what happens to a user who has just + authenticated there. #} + +{# Create a Forgejo account on first successful OIDC login. Without this, + Authentik authenticates the user and Forgejo then has nobody to log in as. + Gated by the registration settings below, not independent of them. #} +ENABLE_AUTO_REGISTRATION = true + +{# Attach an OIDC identity to an existing local account when the email + matches, rather than creating a second one alongside it. This is what lets + the break-glass admin from step 4b become the same account you arrive as + through Authentik, instead of demoting you to a fresh unprivileged user on + first SSO login. `login` would ask the user to confirm by entering their + local password; `auto` links silently, which is only safe because + Authentik is the sole source of verified addresses here. #} +ACCOUNT_LINKING = auto + +{# Take the Forgejo username from the provider's preferred_username claim. + The alternative, `userid`, would name accounts after Authentik's opaque + subject UUID. #} +USERNAME = nickname + +[service] +{# INSTALL_LOCK skips the wizard, which is also where these would have been + chosen. + + These two are a pair and the combination is deliberate. DISABLE_REGISTRATION + cannot stay `true` once SSO is wanted: it blocks OIDC auto-registration as + well as the local signup form, so Authentik logins authenticate correctly + and are then refused an account — which reads like a broken provider rather + than a policy setting. ALLOW_ONLY_EXTERNAL_REGISTRATION restores exactly + the property that was wanted: no self-service signup, accounts only through + a configured provider. + + Safe to have in place before the Authentik source exists. With no external + provider configured, "only external registration" permits nothing, so this + is not a window during which the forge is open — flipping the first value + alone would have been. #} +DISABLE_REGISTRATION = false +ALLOW_ONLY_EXTERNAL_REGISTRATION = true +REQUIRE_SIGNIN_VIEW = false + +[log] +{# journald, via systemd capturing stdout — `journalctl -u forgejo`. ROOT_PATH + still matters: some subsystems write their own files regardless. #} +MODE = console +LEVEL = info +ROOT_PATH = {{ app_config.FORGEJO_LOG_PATH }} diff --git a/src/forgejo/ansible/proxmox/templates/forgejo-dbdump.service.j2 b/src/forgejo/ansible/proxmox/templates/forgejo-dbdump.service.j2 new file mode 100644 index 0000000..b8710ce --- /dev/null +++ b/src/forgejo/ansible/proxmox/templates/forgejo-dbdump.service.j2 @@ -0,0 +1,16 @@ +{# Rendered by Ansible — see ../install.yml. #} +[Unit] +Description=Dump the Forgejo database into this container's filesystem +Documentation=file://{{ app_config.FORGEJO_CONFIG_PATH }} +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User={{ app_config.FORGEJO_USER }} +Group={{ app_config.FORGEJO_USER }} +ExecStart=/usr/local/bin/forgejo-dbdump +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=full +ReadWritePaths={{ app_config.FORGEJO_BACKUP_PATH }} diff --git a/src/forgejo/ansible/proxmox/templates/forgejo-dbdump.sh.j2 b/src/forgejo/ansible/proxmox/templates/forgejo-dbdump.sh.j2 new file mode 100644 index 0000000..49a8d02 --- /dev/null +++ b/src/forgejo/ansible/proxmox/templates/forgejo-dbdump.sh.j2 @@ -0,0 +1,50 @@ +#!/bin/bash +{# Rendered by Ansible — see ../install.yml. Do not edit on the host. #} +# +# Dumps the Forgejo database into this container's filesystem, where the +# Proxmox vzdump job will pick it up along with everything else. Postgres runs +# elsewhere (shared, on Unraid), so without this a restored container would +# have every repository and no issues, pull requests, users or permissions. +# +# Run by forgejo-dbdump.timer, not by hand — though running it by hand is +# harmless and is the quickest way to check the credentials work. + +# pipefail matters more than usual here: pg_dump feeds gzip, and without it a +# failed dump still exits 0 through gzip and gets published as a valid-looking +# but truncated archive. +set -euo pipefail + +readonly BACKUP_DIR="{{ app_config.FORGEJO_BACKUP_PATH }}" +readonly KEEP_DAYS="{{ app_config.FORGEJO_BACKUP_KEEP_DAYS }}" + +export PGPASSFILE="{{ app_config.FORGEJO_HOME }}/.pgpass" + +umask 077 + +dest="${BACKUP_DIR}/{{ app_config.DB_NAME }}-$(date -u +%Y%m%dT%H%M%SZ).sql.gz" + +# Write to .part and rename only on success, so a dump interrupted by the +# vzdump window (or a reboot) never leaves a partial file that looks complete +# to the retention sweep below. +pg_dump \ + --host="{{ app_config.DB_HOST }}" \ + --port="{{ app_config.DB_PORT }}" \ + --username="{{ app_config.DB_USER }}" \ + --dbname="{{ app_config.DB_NAME }}" \ + --format=plain \ + --no-owner \ + --no-privileges \ + | gzip -9 > "${dest}.part" + +mv "${dest}.part" "${dest}" + +# Retention is here rather than in the storage's prune settings because these +# live inside the container: Proxmox prunes backup archives, not their +# contents. Only complete dumps are counted, so a stale .part never displaces +# a good one. +find "${BACKUP_DIR}" -maxdepth 1 -type f -name '*.sql.gz' \ + -mtime "+${KEEP_DAYS}" -delete +find "${BACKUP_DIR}" -maxdepth 1 -type f -name '*.sql.gz.part' \ + -mtime +1 -delete + +echo "wrote ${dest}" diff --git a/src/forgejo/ansible/proxmox/templates/forgejo-dbdump.timer.j2 b/src/forgejo/ansible/proxmox/templates/forgejo-dbdump.timer.j2 new file mode 100644 index 0000000..725b66e --- /dev/null +++ b/src/forgejo/ansible/proxmox/templates/forgejo-dbdump.timer.j2 @@ -0,0 +1,16 @@ +{# Rendered by Ansible — see ../install.yml. #} +[Unit] +Description=Nightly Forgejo database dump + +[Timer] +{# Must land before the vzdump window on the Proxmox node + (build/config/ansible/inventory/host_vars/pve.yml). The archive is only a + complete restore point if the dump inside it is from the same night. #} +OnCalendar={{ app_config.FORGEJO_DB_DUMP_ONCALENDAR }} +{# Catch up after a reboot rather than skipping a night silently. #} +Persistent=true +{# No randomised delay: the gap to the backup window is the whole point. #} +AccuracySec=1min + +[Install] +WantedBy=timers.target diff --git a/src/forgejo/ansible/proxmox/templates/forgejo.service.j2 b/src/forgejo/ansible/proxmox/templates/forgejo.service.j2 new file mode 100644 index 0000000..5086ded --- /dev/null +++ b/src/forgejo/ansible/proxmox/templates/forgejo.service.j2 @@ -0,0 +1,33 @@ +{# Rendered by Ansible — see ../install.yml. The native-install replacement for + Compose's `restart: unless-stopped`. #} +[Unit] +Description=Forgejo +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User={{ app_config.FORGEJO_USER }} +Group={{ app_config.FORGEJO_USER }} +WorkingDirectory={{ app_config.FORGEJO_WORK_PATH }} +ExecStart=/usr/local/bin/forgejo web --config {{ app_config.FORGEJO_CONFIG_PATH }} +Restart=always +RestartSec=2s +Environment=USER={{ app_config.FORGEJO_USER }} +Environment=HOME={{ app_config.FORGEJO_HOME }} +Environment=GITEA_WORK_DIR={{ app_config.FORGEJO_WORK_PATH }} + +# HTTP_PORT is 3000, so no CAP_NET_BIND_SERVICE is needed. Forgejo does need +# to write the git user's authorized_keys and its own data, so the filesystem +# is not made read-only wholesale. +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=full +ProtectHome=false +ReadWritePaths={{ app_config.FORGEJO_HOME }} +{# Restarting mid-push should not orphan git subprocesses. #} +KillMode=control-group +TimeoutStopSec=30s + +[Install] +WantedBy=multi-user.target diff --git a/src/forgejo/ansible/proxmox/templates/pgpass.j2 b/src/forgejo/ansible/proxmox/templates/pgpass.j2 new file mode 100644 index 0000000..e4557a8 --- /dev/null +++ b/src/forgejo/ansible/proxmox/templates/pgpass.j2 @@ -0,0 +1,8 @@ +{# Rendered by Ansible — see ../install.yml. Lets the nightly dump job + authenticate without the password appearing in a command line or a systemd + unit. Mode 0600, owned by the Forgejo service user; libpq refuses to read it + otherwise. + + Format is host:port:database:user:password, with literal backslashes and + colons in the password escaped. #} +{{ app_config.DB_HOST }}:{{ app_config.DB_PORT }}:{{ app_config.DB_NAME }}:{{ app_config.DB_USER }}:{{ vault_secrets.DB_PASSWORD | replace('\\', '\\\\') | replace(':', '\\:') }} diff --git a/src/forgejo/ansible/proxmox/vars.yml b/src/forgejo/ansible/proxmox/vars.yml new file mode 100644 index 0000000..4cefc97 --- /dev/null +++ b/src/forgejo/ansible/proxmox/vars.yml @@ -0,0 +1,50 @@ +--- +# Proxmox-specific overrides, merged over ../../common/vars.yml (these win). +# Reachable in install.yml and the templates as `app_config`. +env_defaults: + # ../../common/vars.yml carries FORGEJO_VERSION: "10" — that's a Docker tag, + # a rolling pointer at the latest 10.x. A native install downloads one + # specific release artifact, so it needs the exact version. Bump this to + # upgrade; the next deploy fetches the new binary and restarts the service. + FORGEJO_RELEASE: "10.0.3" + FORGEJO_ARCH: amd64 + + # Service account and filesystem layout — the Proxmox counterpart to the + # Unraid appdata path. `git` is the conventional user name for a forge: it's + # also the SSH login clients use (git@host:owner/repo.git). + FORGEJO_USER: git + FORGEJO_HOME: /var/lib/forgejo + FORGEJO_WORK_PATH: /var/lib/forgejo + FORGEJO_DATA_PATH: /var/lib/forgejo/data + FORGEJO_LOG_PATH: /var/lib/forgejo/log + FORGEJO_CONFIG_DIR: /etc/forgejo + FORGEJO_CONFIG_PATH: /etc/forgejo/app.ini + FORGEJO_INSTALL_DIR: /usr/local/lib/forgejo + + # 22, not the 2222 in common/vars.yml. On Unraid that was a published Docker + # port dodging the host's own sshd; here Forgejo uses the container's sshd + # directly, and the container has a whole IP to itself. Note this changes the + # SSH clone URLs Forgejo advertises — see README.md in this directory. + FORGEJO_SSH_PORT: "22" + + # No `unraid_shared` Docker network here to resolve `shared-postgres` + # through, so the shared Postgres is reached over the LAN, at the address of + # the LXC running it. Keep in step with `ip_address` on the `postgres` + # module in build/config/terraform/main.tf, which is the authoritative copy + # (it is also published as the `postgres_address` output). + # + # Was 192.168.50.2 — the Unraid host nas2, now retired. Note that address + # has since been reassigned to turtle-proxmox-01, so a stale copy of this + # value fails by connecting to the wrong machine rather than by timing out. + DB_HOST: 192.168.50.54 + DB_PORT: "5432" + + # Nightly pg_dump of the Forgejo database, written to the container's own + # disk so that the vzdump archive of this guest carries it off-box along with + # everything else. Must finish before the vzdump window in + # build/config/ansible/inventory/group_vars/proxmox_nodes.yml (02:00, half an + # hour after this) — see the README in this directory for why the ordering + # matters. + FORGEJO_BACKUP_PATH: /var/lib/forgejo/backups + FORGEJO_BACKUP_KEEP_DAYS: "14" + FORGEJO_DB_DUMP_ONCALENDAR: "*-*-* 01:30:00" diff --git a/src/forgejo/ansible/unraid/.env.example b/src/forgejo/ansible/unraid/.env.example new file mode 100644 index 0000000..14bc60d --- /dev/null +++ b/src/forgejo/ansible/unraid/.env.example @@ -0,0 +1,28 @@ +# Reference only — real values are rendered by Ansible from vars.yml (non- +# secret) and Vault path homelab/forgejo (secret). Do not fill this in and +# deploy it directly. + +# Non-secret (see vars.yml) +FORGEJO_VERSION=10 +FORGEJO_DATA_PATH=/mnt/user/appdata/forgejo/data +FORGEJO_HTTP_PORT=3000 +FORGEJO_SSH_PORT=2222 +FORGEJO_DOMAIN=git.example.internal +FORGEJO_ROOT_URL=http://git.example.internal:3000/ +DB_HOST=shared-postgres +DB_NAME=forgejo +DB_USER=forgejo + +# Non-secret, supplied by the compose_stack role rather than either vars.yml — +# the path it copies ansible/unraid/icon.png to on the host, for the +# net.unraid.docker.icon label. Empty when the stack ships no icon.png, which +# just leaves Unraid's question-mark placeholder. Override it in vars.yml to +# point at a hosted PNG instead. +STACK_ICON=/mnt/user/appdata/icons/forgejo.png + +# Secrets — Vault keys: DB_PASSWORD, SECRET_KEY, INTERNAL_TOKEN, JWT_SECRET +# Generate the latter three with: docker run --rm codeberg.org/forgejo/forgejo:10 forgejo generate secret +DB_PASSWORD= +SECRET_KEY= +INTERNAL_TOKEN= +JWT_SECRET= diff --git a/src/forgejo/ansible/unraid/docker-compose.yml b/src/forgejo/ansible/unraid/docker-compose.yml new file mode 100644 index 0000000..04b0118 --- /dev/null +++ b/src/forgejo/ansible/unraid/docker-compose.yml @@ -0,0 +1,48 @@ +services: + forgejo: + image: codeberg.org/forgejo/forgejo:${FORGEJO_VERSION} + container_name: forgejo + restart: unless-stopped + # Unraid's Docker page normally reads icon, WebUI link and console shell + # from the dockerMan template that created the container. A Compose stack + # has no template, so 6.10+ falls back to these labels; without them the + # container renders as a question mark with no WebUI or Console entry. + # + # STACK_ICON comes from the compose_stack role (a path under + # /mnt/user/appdata/icons, or a URL if vars.yml overrides it). `[IP]` is + # substituted by the webgui with the host's address; the port has to be + # the published one, hence FORGEJO_HTTP_PORT rather than the container's + # 3000. Alpine image, so Console gets sh, not bash. + labels: + net.unraid.docker.icon: ${STACK_ICON} + net.unraid.docker.webui: "http://[IP]:${FORGEJO_HTTP_PORT}/" + net.unraid.docker.shell: sh + environment: + USER_UID: "1000" + USER_GID: "1000" + FORGEJO__database__DB_TYPE: postgres + FORGEJO__database__HOST: ${DB_HOST} + FORGEJO__database__NAME: ${DB_NAME} + FORGEJO__database__USER: ${DB_USER} + FORGEJO__database__PASSWD: ${DB_PASSWORD} + FORGEJO__server__DOMAIN: ${FORGEJO_DOMAIN} + FORGEJO__server__ROOT_URL: ${FORGEJO_ROOT_URL} + FORGEJO__server__SSH_DOMAIN: ${FORGEJO_DOMAIN} + FORGEJO__server__SSH_PORT: ${FORGEJO_SSH_PORT} + # Fixed rather than left to be auto-generated, so recreating the + # container from a fresh volume doesn't invalidate sessions/tokens. + FORGEJO__security__SECRET_KEY: ${SECRET_KEY} + FORGEJO__security__INTERNAL_TOKEN: ${INTERNAL_TOKEN} + FORGEJO__oauth2__JWT_SECRET: ${JWT_SECRET} + volumes: + - ${FORGEJO_DATA_PATH}:/data + ports: + - "${FORGEJO_HTTP_PORT}:3000" + - "${FORGEJO_SSH_PORT}:22" + networks: + - shared + +networks: + shared: + name: unraid_shared + external: true diff --git a/src/forgejo/ansible/unraid/vars.yml b/src/forgejo/ansible/unraid/vars.yml new file mode 100644 index 0000000..9b68c40 --- /dev/null +++ b/src/forgejo/ansible/unraid/vars.yml @@ -0,0 +1,21 @@ +--- +# Unraid-specific overrides, merged over ../../common/vars.yml (these win). +env_defaults: + FORGEJO_DATA_PATH: /mnt/user/appdata/forgejo/data + # Docker network alias of the shared Postgres container — reachable because + # both stacks join the `unraid_shared` network. Not the same as + # host_vars/*.yml stack.db.provision_host, which Ansible uses from the + # controller, outside Docker. + DB_HOST: shared-postgres + + # --- Unraid Docker page presentation (net.unraid.docker.* labels) -------- + # + # STACK_ICON is normally left to the compose_stack role, which points it at + # /mnt/user/appdata/icons/.png when this directory contains an + # icon.png. Uncomment to use a hosted icon instead of committing one: + # + # STACK_ICON: https://example.org/forgejo.png + # + # PNG only — Unraid renders its placeholder for SVG and nothing at all for + # WebP, and a URL it can't reach when the Docker page renders fails the same + # way, which is the argument for committing the file. diff --git a/src/forgejo/common/vars.yml b/src/forgejo/common/vars.yml new file mode 100644 index 0000000..17c31b9 --- /dev/null +++ b/src/forgejo/common/vars.yml @@ -0,0 +1,15 @@ +--- +# Portable config — values that hold regardless of whether Forgejo runs as a +# Compose stack on Unraid or natively in a Proxmox LXC. Platform-specific +# values (filesystem paths, how the database is reached) live in +# ../ansible//vars.yml and are merged over these at deploy time. +env_defaults: + FORGEJO_VERSION: "10" + FORGEJO_HTTP_PORT: "3000" + FORGEJO_SSH_PORT: "2222" + FORGEJO_DOMAIN: git.turtlesystems.uk + FORGEJO_ROOT_URL: "https://git.turtlesystems.uk" + DB_NAME: forgejo + DB_USER: forgejo + # DB_PASSWORD, SECRET_KEY, INTERNAL_TOKEN, JWT_SECRET are secrets — sourced + # from Vault (homelab/forgejo), never set here. diff --git a/src/forgejo/terraform/README.md b/src/forgejo/terraform/README.md new file mode 100644 index 0000000..7692490 --- /dev/null +++ b/src/forgejo/terraform/README.md @@ -0,0 +1,52 @@ +# terraform/ — forgejo + +The LXC Forgejo runs in on Proxmox: container template, cores, memory, disk, +static IP. Called from `build/config/terraform/main.tf`, which holds the +provider and the `backend "pg"` state config. + +Terraform's scope stops at the guest — installing Forgejo into it is +`../ansible/proxmox/`. + +## Required inputs + +Everything else has a default (see `variables.tf`): + +| Variable | Example | +|---|---| +| `node_name` | `pve` | +| `template_file_id` | `local:vztmpl/debian-12-standard_12.7-1_amd64.tar.zst` | +| `ip_address` | `192.168.50.52/24` | +| `gateway` | `192.168.50.254` | +| `ssh_public_keys` | the public half of the key in Vault at `homelab/ci/ssh` | + +## No bind mounts, on purpose + +There is no `mount_point` block. All of Forgejo's state — repositories, LFS, +attachments, Bleve indexers, queues — sits on the container's own rootfs, so +one vzdump archive is a complete copy of it. A bind-mounted NFS share would +break that guarantee: vzdump deliberately excludes bind mounts, so the thing +you most wanted backed up would be the one thing missing from the backup. + +Getting a copy off the box is the backup job's job, not the container's: +`build/config/ansible/roles/pve_backup` defines an NFS storage pointing at the +NAS and a vzdump schedule that writes there. The Postgres database lives +off-container and is handled separately — see `../ansible/proxmox/README.md`. + +This is why `disk_size` defaults to 64 GiB rather than something minimal: the +repositories are on it. + +## `prevent_destroy` + +`main.tf` sets `lifecycle { prevent_destroy = true }`. Several innocuous-looking +changes (shrinking `disk_size`, changing `template_file_id`) are replacements +rather than in-place updates, and replacing this container destroys every +repository on it. The flag turns those into a failed plan instead of a +successful one. To genuinely retire the guest, remove the block first — it +can't be overridden from the command line. + +## Sizing + +`cores = 2`, `memory = 2048`, `disk_size = 64`. Memory is sized for Forgejo +plus its indexers; disk for repositories plus headroom for push staging and +the nightly database dumps kept under `FORGEJO_BACKUP_PATH`. Growing the disk +later is safe. diff --git a/src/forgejo/terraform/main.tf b/src/forgejo/terraform/main.tf new file mode 100644 index 0000000..26e7b02 --- /dev/null +++ b/src/forgejo/terraform/main.tf @@ -0,0 +1,116 @@ +# The LXC Forgejo runs in on Proxmox. Terraform's scope stops here: it creates +# the guest and nothing else, because the Proxmox provider has no way to +# install an application into one. Ansible (`lxc_app` + +# ../ansible/proxmox/install.yml) picks up from the point this container +# exists. +# +# Deliberately no `mount_point` block. Everything Forgejo owns — repositories, +# LFS, attachments, indexers, queues — lives on this container's own rootfs, so +# a vzdump of the guest is a complete copy of its filesystem state. Bind mounts +# would break that: vzdump skips them by design. The off-box copy is the +# backup job in build/config/ansible/roles/pve_backup, which writes vzdump +# archives to the NAS over NFS. See ../ansible/proxmox/README.md for why the +# database is handled separately. + +resource "proxmox_virtual_environment_container" "forgejo" { + node_name = var.node_name + vm_id = var.vm_id + tags = var.tags + + # Unprivileged: nothing here needs host privileges. Forgejo binds 3000, not + # a low port, and git operations are ordinary userspace file I/O. + unprivileged = true + start_on_boot = true + + # systemd >= 252 in an unprivileged container needs a cgroup namespace of + # its own to set up delegation, which LXC only grants with nesting on. + # Without it systemd still boots, but units using the sandboxing directives + # (PrivateTmp, ProtectSystem, ProtectHome) fail to start — and the unit + # ../ansible/proxmox/ installs is exactly that shape. This is what the + # provider's "WARN: Systemd NNN detected. You may need to enable nesting." + # is pointing at, and it is the Proxmox UI's own default for unprivileged + # containers. + # + # Safe here precisely *because* the container is unprivileged: nesting on a + # privileged container widens a real escape surface, on an unprivileged one + # it grants a namespace the kernel already refuses to let root out of. + features { + nesting = true + } + + # Setting this on an *already existing* container takes root@pam. PVE + # hard-codes that for feature flags — it is not an ACL and no role grants + # it — and while its own error message exempts nesting, the provider sends + # the other flags alongside, so the exemption doesn't apply. A 403 reading + # "changing feature flags (except nesting) is only allowed for root@pam" + # means this container predates the block; fix it on the node with + # + # pct set --features nesting=1 && pct reboot + # + # after which the plan reads clean, because the provider only sends + # `features` when it differs. Whether a *fresh* create hits the same wall is + # untested — CT 161 was already up when this block was added. + + initialization { + hostname = var.hostname + + ip_config { + ipv4 { + address = var.ip_address + gateway = var.gateway + } + } + + # Omitted entirely when unset, so the container inherits the node's + # resolver rather than being handed an empty server list. + dynamic "dns" { + for_each = var.dns_servers == null ? [] : [1] + + content { + servers = var.dns_servers + } + } + + # Ansible connects as root over SSH, same as the Unraid hosts — see + # "SSH access" in the top-level README.md. Without a key here the guest + # comes up unreachable and playbooks/proxmox.yml can't do anything with it. + user_account { + keys = var.ssh_public_keys + } + } + + cpu { + cores = var.cores + } + + memory { + dedicated = var.memory + swap = var.swap + } + + disk { + datastore_id = var.datastore_id + size = var.disk_size + } + + network_interface { + name = "eth0" + bridge = var.network_bridge + vlan_id = var.network_vlan_id + firewall = var.network_firewall + enabled = true + } + + operating_system { + template_file_id = var.template_file_id + type = var.operating_system_type + } + + # `disk.size` can be grown in place but not shrunk, and the provider replaces + # the container rather than failing if you try. Replacing this guest means + # losing every repository on it, so refuse the plan instead and let a resize + # be a deliberate act. + lifecycle { + prevent_destroy = true + } +} diff --git a/src/forgejo/terraform/outputs.tf b/src/forgejo/terraform/outputs.tf new file mode 100644 index 0000000..a2a89e4 --- /dev/null +++ b/src/forgejo/terraform/outputs.tf @@ -0,0 +1,25 @@ +# Ansible finds the guest through the Proxmox API (inventory/proxmox.yml), not +# by having these wired into inventory — see the note at the top of +# build/config/terraform/main.tf. They exist so the backup job and anything +# else in the root module can refer to the guest without restating its +# identity. + +output "hostname" { + description = "Container hostname — the name this guest appears under in Ansible inventory." + value = proxmox_virtual_environment_container.forgejo.initialization[0].hostname +} + +output "vm_id" { + description = "Proxmox VMID. Pass to the vzdump job if it is pinned to an explicit VMID list." + value = proxmox_virtual_environment_container.forgejo.vm_id +} + +output "ip_address" { + description = "IPv4 address without the prefix length." + value = split("/", var.ip_address)[0] +} + +output "node_name" { + description = "Proxmox node the container was created on." + value = proxmox_virtual_environment_container.forgejo.node_name +} diff --git a/src/forgejo/terraform/providers.tf b/src/forgejo/terraform/providers.tf new file mode 100644 index 0000000..6714e16 --- /dev/null +++ b/src/forgejo/terraform/providers.tf @@ -0,0 +1,13 @@ +terraform { + required_version = ">= 1.6" + + required_providers { + proxmox = { + source = "bpg/proxmox" + # Deliberately unpinned until the first real `terraform init` — pin to + # whatever it resolves, and commit .terraform.lock.hcl (which is not + # gitignored) so everyone and CI get the same provider. + # version = "~> 0.0" + } + } +} \ No newline at end of file diff --git a/src/forgejo/terraform/variables.tf b/src/forgejo/terraform/variables.tf new file mode 100644 index 0000000..be07067 --- /dev/null +++ b/src/forgejo/terraform/variables.tf @@ -0,0 +1,149 @@ +# Nothing environment-specific is hardcoded in the module — every value the +# caller could reasonably differ on is a variable, set from +# build/config/terraform/main.tf. + +variable "node_name" { + description = "Proxmox node to create the container on, e.g. `pve`." + type = string +} + +variable "vm_id" { + description = <<-EOT + Fixed VMID rather than letting Proxmox pick the next free one. The backup + job in build/config/ansible/roles/pve_backup can be pinned to an explicit + VMID list, and a guest whose ID moves between rebuilds silently drops out + of it. + EOT + type = number + default = 160 +} + +variable "hostname" { + description = <<-EOT + Container hostname. This is also the inventory hostname Ansible sees, via + the community.proxmox dynamic inventory — so it has to match the + build/config/ansible/inventory/host_vars/.yml that declares the + `apps:` list. + EOT + type = string + default = "forgejo" +} + +variable "cores" { + description = "CPU cores. Forgejo is not CPU-bound at homelab scale; 2 is comfortable." + type = number + default = 2 +} + +variable "memory" { + description = "RAM in MiB. 2048 covers Forgejo plus its Bleve indexers." + type = number + default = 2048 +} + +variable "swap" { + description = "Swap in MiB." + type = number + default = 512 +} + +variable "disk_size" { + description = <<-EOT + Root disk in GiB. This holds everything — the OS, the Forgejo binary, and + all repository/LFS/attachment data — because nothing is offloaded to an + NFS mount; that is what makes a vzdump of this guest a complete filesystem + backup. Size it for the repos you expect, plus headroom for push staging + and the local database dumps the app keeps under + `FORGEJO_BACKUP_PATH`. Growing it later is safe; shrinking is not (see the + `lifecycle` block in main.tf). + EOT + type = number + default = 64 +} + +variable "datastore_id" { + description = "Proxmox storage the root disk is created on, e.g. `local-lvm`." + type = string + default = "local-lvm" +} + +variable "template_file_id" { + description = <<-EOT + Container template volume ID, e.g. + `local:vztmpl/debian-12-standard_12.7-1_amd64.tar.zst`. Must already be + downloaded on the node — see "Prerequisites for Proxmox" in the top-level + README.md. List what you have with `pveam list local`. + EOT + type = string +} + +variable "operating_system_type" { + description = "Template OS family, as Proxmox labels it. Match the template above." + type = string + default = "debian" +} + +variable "ip_address" { + description = <<-EOT + Static IPv4 in CIDR form, e.g. `192.168.50.60/24`. Static rather than DHCP + because this is the address the `FORGEJO_ROOT_URL` domain resolves to and + the one Ansible connects to. + EOT + type = string + + validation { + condition = can(cidrnetmask(var.ip_address)) + error_message = "ip_address must include a prefix length, e.g. 192.168.50.60/24." + } +} + +variable "gateway" { + description = "IPv4 default gateway." + type = string +} + +variable "dns_servers" { + description = "Resolvers for the container. Leave null to inherit the Proxmox node's." + type = list(string) + default = null +} + +variable "network_bridge" { + description = "Proxmox bridge to attach eth0 to." + type = string + default = "vmbr0" +} + +variable "network_vlan_id" { + description = "VLAN tag for eth0, or null for an untagged bridge." + type = number + default = null +} + +variable "network_firewall" { + description = "Enable the Proxmox firewall on this interface." + type = bool + default = false +} + +variable "ssh_public_keys" { + description = <<-EOT + Public keys installed for root in the container. Ansible authenticates + with the matching private half (`homelab/ci/ssh` in Vault); without at + least one key here the guest is unreachable and playbooks/proxmox.yml has + nothing it can do with it. + EOT + type = list(string) +} + +variable "tags" { + description = <<-EOT + Proxmox tags, shown in the UI and read by the Ansible dynamic inventory. + `terraform` is load-bearing: build/config/ansible/inventory/proxmox.yml + filters on it to tell guests this repo creates apart from ones made by + hand on the node. Drop it and this container is simply never deployed + to — silently, with no error anywhere. + EOT + type = list(string) + default = ["forgejo", "terraform"] +} diff --git a/src/hashicorp-vault/config.hcl b/src/hashicorp-vault/config.hcl new file mode 100644 index 0000000..5c0cf7f --- /dev/null +++ b/src/hashicorp-vault/config.hcl @@ -0,0 +1,18 @@ +storage "raft" { + path = "/vault/file" + node_id = "nas-vault-01" +} + +listener "tcp" { + address = "0.0.0.0:8200" + tls_disable = "true" + x_forwarded_for_authorized_addrs = "192.168.50.51/32" + x_forwarded_for_reject_not_present = false +} + +api_addr = "https://safe.turtlesystems.uk" +cluster_addr = "https://vault:8201" + +ui = true + +disable_mlock = true diff --git a/src/hashicorp-vault/policies/unraid-deploy.hcl b/src/hashicorp-vault/policies/unraid-deploy.hcl new file mode 100644 index 0000000..910463d --- /dev/null +++ b/src/hashicorp-vault/policies/unraid-deploy.hcl @@ -0,0 +1,3 @@ +path "kv/data/homelab/*" { + capabilities = ["read"] +} \ No newline at end of file diff --git a/src/hashicorp-vault/policies/vault-readonly.hcl b/src/hashicorp-vault/policies/vault-readonly.hcl new file mode 100644 index 0000000..a5d6adc --- /dev/null +++ b/src/hashicorp-vault/policies/vault-readonly.hcl @@ -0,0 +1,6 @@ +path "secret/data/*" { + capabilities = ["read", "list"] +} +path "secret/metadata/*" { + capabilities = ["read", "list"] +} diff --git a/src/hashicorp-vault/policies/vault-write.hcl b/src/hashicorp-vault/policies/vault-write.hcl new file mode 100644 index 0000000..7423640 --- /dev/null +++ b/src/hashicorp-vault/policies/vault-write.hcl @@ -0,0 +1,3 @@ +path "secret/data/*" { + capabilities = ["create", "update", "delete"] +} diff --git a/src/scripts/configure_k3s_node.zsh b/src/scripts/configure_k3s_node.zsh new file mode 100644 index 0000000..e9f0ddf --- /dev/null +++ b/src/scripts/configure_k3s_node.zsh @@ -0,0 +1,15 @@ +#!/usr/bin/env zsh + +set -euo pipefail + +# Get the hostname that will be accessed +TARGET_HOST=$1 + +# Build up the command to add the ansible user and add to the right +# group to allow passwordless sudo +ssh $TARGET_HOST "sudo useradd -m -s /bin/bash ansible && sudo usermod -aG sudo ansible && echo 'ansible ALL=(ALL) NOPASSWD:ALL' | sudo tee /etc/sudoers.d/ansible" + +# Add the SSH public key for the ansible user +ssh $TARGET_HOST "sudo mkdir -p /home/ansible/.ssh && sudo chmod 700 /home/ansible/.ssh && sudo touch /home/ansible/.ssh/authorized_keys && sudo chmod 600 /home/ansible/.ssh/authorized_keys && sudo chown -R ansible:ansible /home/ansible/.ssh" +ssh $TARGET_HOST "echo '$(cat ~/.ssh/id_ed_ansible.pub)' | sudo tee -a /home/ansible/.ssh/authorized_keys" + diff --git a/src/shared/postgres/ansible/kubernetes/vars.yml b/src/shared/postgres/ansible/kubernetes/vars.yml new file mode 100644 index 0000000..4567576 --- /dev/null +++ b/src/shared/postgres/ansible/kubernetes/vars.yml @@ -0,0 +1,20 @@ +--- +# Kubernetes-specific overrides, merged over ../../common/vars.yml (these +# win) — the third platform for shared/postgres, alongside ansible/unraid/ +# and ansible/proxmox/. Consumed by roles/k3s_postgres, not compose_stack or +# lxc_app: there's no Compose file or systemd unit here, this cluster is +# deployed as a CloudNativePG Cluster CR (see that role's templates/), but +# the same "portable config + platform overrides" split still applies. +env_defaults: + # 1 primary + 1 replica, workers only — see roles/k3s_postgres/tasks/main.yml + # for the nodeAffinity that keeps both off the control-plane Pi. + POSTGRES_INSTANCES: "2" + + # k3s's built-in default StorageClass (local-path-provisioner) — each + # instance's PV is pinned to whichever worker it first schedules on. + # That's fine here: CNPG's resilience comes from Postgres-level streaming + # replication between instances, not from the storage layer, the same way + # losing one instance's local disk doesn't lose the data as long as the + # other instance is still up. + POSTGRES_STORAGE_SIZE: 10Gi + POSTGRES_STORAGE_CLASS: local-path diff --git a/src/shared/postgres/ansible/proxmox/README.md b/src/shared/postgres/ansible/proxmox/README.md new file mode 100644 index 0000000..677c82e --- /dev/null +++ b/src/shared/postgres/ansible/proxmox/README.md @@ -0,0 +1,76 @@ +# ansible/proxmox/ — postgres + +Native install into the Terraform-provisioned LXC (`../../terraform/`, VMID +161, 192.168.50.54). Deployed by the `lxc_app` role from the `apps:` list in +`build/config/ansible/inventory/host_vars/postgres.yml`. + +This is the shared Postgres for everything outside the k3s cluster. It is the +one app with no `db:` block of its own — the per-app databases are created +*against* it, by whichever `compose_stack` / `lxc_app` / `k3s_app` run +declares them, using `homelab/shared/postgres` as `admin_vault_path`. + +## Files + +| File | What it does | +|---|---| +| `vars.yml` | Proxmox-specific overrides merged over `../../common/vars.yml` — version, cluster name, listen address, allowed CIDRs, tuning | +| `install.yml` | PGDG repository, package install, config, and the superuser password from Vault | +| `templates/homelab.conf.j2` | A `conf.d` drop-in, so a package upgrade rewriting `postgresql.conf` can't revert it | +| `templates/pg_hba.conf.j2` | Rendered whole — there is no drop-in mechanism here and first-match-wins ordering has to be owned outright | + +## Postgres 17, not 13 + +The Unraid Compose stack was pinned to 13 because its data directory predated +this repo. This is a fresh cluster loaded by dump/restore, so there is nothing +to stay compatible with, and Postgres comes from the PGDG archive rather than +Debian's own so that the major version is a decision rather than whatever the +distro happens to ship. + +Bumping `POSTGRES_VERSION` on an existing host does **not** upgrade it. The +new major's package installs alongside the old one and `pg_createcluster` +makes a second, empty cluster on the next free port. Moving data across a +major is `pg_dumpall`/`psql` or `pg_upgradecluster` — a run-once operation, so +it belongs in a runbook rather than in this role. + +## Everything is on the rootfs, and here that is a correctness requirement + +`../../terraform/` puts the container on the `AppData` ZFS pool and creates a +replication job to the second node. Proxmox replicates by sending ZFS +snapshots, and it snapshots each of a guest's volumes independently. + +Splitting `PGDATA` and the WAL across two volumes would therefore replicate +them as two snapshots taken at different instants, and the far node could end +up holding a data directory and a WAL that disagree — which is not a state WAL +replay can recover from. One volume means one atomic snapshot, which is +exactly the crash-consistent image Postgres *is* designed to recover from. + +Same conclusion as Forgejo's "no bind mounts", arrived at from a different +direction: there it was about what `vzdump` includes, here it is about what a +snapshot is atomic across. + +## `full_page_writes = off` + +Set in `vars.yml`, and the one setting here that would be a data-loss bug if +copied to another platform. + +`full_page_writes` exists to survive a torn page — half of an 8 KiB write +reaching disk before a crash. ZFS is copy-on-write and never overwrites a live +block in place, so that failure mode does not arise, and the full-page images +are pure write amplification that on this host also becomes replication +traffic. It is safe **only** on copy-on-write storage. On ext4 or xfs the same +setting risks unrecoverable corruption, which is why it lives in the Proxmox +vars file and not in `../../common/vars.yml`. + +## Replication is not a backup + +The second copy protects against losing a node. It does nothing about a +dropped table, which replicates within five minutes. Two things cover that, +and both already exist: + +- `roles/pve_backup` — the node's vzdump schedule, writing to the NAS. +- An app-side dump, the way Forgejo writes a nightly `pg_dump` onto its own + disk so the vzdump archive carries a consistent copy of its database. + +Note also that replication is asynchronous. An unplanned failover loses every +transaction committed since the last successful run — up to +`replication_schedule`, five minutes by default. diff --git a/src/shared/postgres/ansible/proxmox/install.yml b/src/shared/postgres/ansible/proxmox/install.yml new file mode 100644 index 0000000..bfdd357 --- /dev/null +++ b/src/shared/postgres/ansible/proxmox/install.yml @@ -0,0 +1,157 @@ +--- +# App-specific install steps, included by the `lxc_app` role after it has +# merged ../../common/vars.yml with ./vars.yml into `app_config` and fetched +# `vault_secrets` from homelab/shared/postgres. The role enables and starts +# `app.service_name` once this file has finished. +# +# Template paths are absolute rather than relative: this file is included from +# a role, so a bare `src:` would resolve against that role's templates/ +# directory, not this app's. +# +# This is the shared database other things provision into, so it is the one +# app whose install has no `db:` block of its own — the per-app databases are +# created against it later, by whichever `compose_stack` / `lxc_app` / +# `k3s_app` run declares them. + +- name: Check that every required Vault key is present + ansible.builtin.assert: + that: + - vault_secrets.POSTGRES_SUPERUSER is defined + - vault_secrets.POSTGRES_SUPERUSER_PASSWORD is defined + fail_msg: >- + Vault path {{ app.vault_path }} is missing POSTGRES_SUPERUSER or + POSTGRES_SUPERUSER_PASSWORD. See docs/vault-secrets.md — the same path + seeds the Unraid, Proxmox and k3s instances. + quiet: true + +# Derived once here rather than restated in vars.yml, so a POSTGRES_VERSION +# bump can't leave one of them pointing at the previous major's cluster. +- name: Set cluster paths + ansible.builtin.set_fact: + pg_conf_dir: "/etc/postgresql/{{ app_config.POSTGRES_VERSION }}/{{ app_config.POSTGRES_CLUSTER }}" + pg_data_dir: "/var/lib/postgresql/{{ app_config.POSTGRES_VERSION }}/{{ app_config.POSTGRES_CLUSTER }}" + +- name: Install prerequisites for the PGDG repository + ansible.builtin.apt: + name: + - ca-certificates + - curl + - gnupg + # psycopg2, for the community.postgresql modules used below. They run on + # this host over the Unix socket, unlike the per-app provisioning in the + # `lxc_app` role, which is delegated to the controller and connects over + # TCP. + - python3-psycopg2 + state: present + update_cache: true + cache_valid_time: 3600 + +# Debian 12 ships Postgres 15 in its own archive. PGDG carries every supported +# major, which is what makes POSTGRES_VERSION a real choice — see vars.yml. +- name: Install the PGDG repository signing key + ansible.builtin.get_url: + url: https://www.postgresql.org/media/keys/ACCC4CF8.asc + dest: /usr/share/keyrings/postgresql.asc + mode: "0644" + +- name: Add the PGDG apt repository + ansible.builtin.apt_repository: + repo: >- + deb [signed-by=/usr/share/keyrings/postgresql.asc] + https://apt.postgresql.org/pub/repos/apt + {{ ansible_facts['distribution_release'] }}-pgdg main + filename: pgdg + state: present + update_cache: true + +- name: Install PostgreSQL + ansible.builtin.apt: + name: + - "postgresql-{{ app_config.POSTGRES_VERSION }}" + - "postgresql-client-{{ app_config.POSTGRES_VERSION }}" + state: present + # The package runs pg_createcluster itself, so by the end of this task a + # cluster exists and is running with stock settings on 5432. Everything + # below is reconfiguration of that cluster, not creation of one. + register: pg_install + +# Both templates are rendered as drop-ins and full files respectively, never +# by editing what the packaging wrote: postgresql.conf is Debian's and carries +# an `include_dir = 'conf.d'` at the bottom for exactly this purpose, so a +# drop-in survives a package upgrade rewriting the main file. +- name: Configure the cluster + ansible.builtin.template: + src: "{{ app_local_dir }}/ansible/proxmox/templates/homelab.conf.j2" + dest: "{{ pg_conf_dir }}/conf.d/homelab.conf" + owner: postgres + group: postgres + mode: "0644" + register: pg_conf + +# pg_hba.conf has no include mechanism worth relying on across versions, so +# this one is rendered whole. 0640 rather than 0644: it enumerates which +# networks can reach the database, which is not something to publish to every +# account on the host. +- name: Configure host-based authentication + ansible.builtin.template: + src: "{{ app_local_dir }}/ansible/proxmox/templates/pg_hba.conf.j2" + dest: "{{ pg_conf_dir }}/pg_hba.conf" + owner: postgres + group: postgres + mode: "0640" + register: pg_hba + +# Restarted rather than reloaded, and done here rather than left to the role's +# start task: listen_addresses and shared_buffers both need a restart, and the +# superuser password below has to be set against a cluster already listening +# with the pg_hba above. The role's own "enable and start" then finds it +# running and does nothing. +- name: Restart PostgreSQL to apply configuration + ansible.builtin.systemd_service: + name: "{{ app_config.POSTGRES_SERVICE_NAME }}" + state: restarted + enabled: true + daemon_reload: true + when: pg_install.changed or pg_conf.changed or pg_hba.changed + +# Every task below runs as the `postgres` account, which means Ansible needs a +# temp directory under that account's home. Left to itself it creates one and +# warns, because a directory root made 0700 for another user is a plausible +# permissions trap; created here it is unambiguously postgres's own. +- name: Ensure the postgres account has an Ansible temp directory + ansible.builtin.file: + path: /var/lib/postgresql/.ansible/tmp + state: directory + owner: postgres + group: postgres + mode: "0700" + +- name: Wait for PostgreSQL to accept connections + ansible.builtin.command: >- + pg_isready --quiet + --host /var/run/postgresql + --port {{ app_config.POSTGRES_PORT }} + become: true + become_user: postgres + changed_when: false + register: pg_ready + until: pg_ready.rc == 0 + retries: 12 + delay: 5 + +# Over the Unix socket as the `postgres` OS account, which peer-authenticates +# — this is the one connection that cannot use the password, because setting +# the password is what it is here to do. Vault is authoritative: if the two +# ever disagree, this task makes the database match Vault rather than the +# other way round. +- name: Set the superuser password from Vault + community.postgresql.postgresql_user: + name: "{{ vault_secrets.POSTGRES_SUPERUSER }}" + password: "{{ vault_secrets.POSTGRES_SUPERUSER_PASSWORD }}" + role_attr_flags: SUPERUSER,LOGIN + login_unix_socket: /var/run/postgresql + login_port: "{{ app_config.POSTGRES_PORT }}" + state: present + become: true + become_user: postgres + no_log: true diff --git a/src/shared/postgres/ansible/proxmox/templates/homelab.conf.j2 b/src/shared/postgres/ansible/proxmox/templates/homelab.conf.j2 new file mode 100644 index 0000000..37cd57e --- /dev/null +++ b/src/shared/postgres/ansible/proxmox/templates/homelab.conf.j2 @@ -0,0 +1,50 @@ +{{ ansible_managed | comment }} +# +# Rendered by src/shared/postgres/ansible/proxmox/install.yml into +# {{ pg_conf_dir }}/conf.d/, which Debian's postgresql.conf includes from its +# last line. A drop-in rather than an edit to postgresql.conf itself: the +# packaging owns that file and rewrites it on a package upgrade, which would +# silently revert anything set there. +# +# Settings here win over postgresql.conf because the include comes last. + +# --- Connections -------------------------------------------------------- +listen_addresses = '{{ app_config.POSTGRES_LISTEN_ADDRESSES }}' +port = {{ app_config.POSTGRES_PORT }} +max_connections = {{ app_config.POSTGRES_MAX_CONNECTIONS }} + +# Passwords for new and changed roles are hashed with this. It has to agree +# with the method in pg_hba.conf — a role whose password was stored as md5 +# cannot authenticate against a scram-sha-256 line, which presents as a +# password failure with correct credentials. +password_encryption = '{{ app_config.POSTGRES_AUTH_METHOD }}' + +# --- Memory ------------------------------------------------------------- +shared_buffers = {{ app_config.POSTGRES_SHARED_BUFFERS }} +effective_cache_size = {{ app_config.POSTGRES_EFFECTIVE_CACHE_SIZE }} +maintenance_work_mem = {{ app_config.POSTGRES_MAINTENANCE_WORK_MEM }} +work_mem = {{ app_config.POSTGRES_WORK_MEM }} + +# --- Write-ahead log ---------------------------------------------------- +# +# See vars.yml for why full_page_writes is off here and must not be copied to +# a deployment on ext4 or xfs: it is safe because ZFS is copy-on-write and +# never overwrites a live block in place, so a page cannot be torn by a crash. +full_page_writes = {{ app_config.POSTGRES_FULL_PAGE_WRITES }} + +# --- Logging ------------------------------------------------------------ +# +# To stderr, which the packaging's systemd unit routes into the journal, so +# `journalctl -u postgresql@{{ app_config.POSTGRES_VERSION }}-{{ app_config.POSTGRES_CLUSTER }}` +# is the whole story and there is no second log directory to rotate or to +# remember to include in a backup. +log_destination = 'stderr' +logging_collector = off +log_line_prefix = '%m [%p] %q%u@%d ' + +# Enough to answer "who connected, and did anything fail", without logging +# every statement — this cluster serves app traffic, not an audit trail. +log_connections = on +log_disconnections = on +log_checkpoints = on +log_min_duration_statement = 1000 diff --git a/src/shared/postgres/ansible/proxmox/templates/pg_hba.conf.j2 b/src/shared/postgres/ansible/proxmox/templates/pg_hba.conf.j2 new file mode 100644 index 0000000..da775b3 --- /dev/null +++ b/src/shared/postgres/ansible/proxmox/templates/pg_hba.conf.j2 @@ -0,0 +1,32 @@ +{{ ansible_managed | comment }} +# +# Rendered whole by src/shared/postgres/ansible/proxmox/install.yml, replacing +# the file the packaging wrote. Unlike postgresql.conf there is no drop-in +# directory to use here, and order matters in this file — the first matching +# line wins and no later line can widen it — so it has to be owned outright +# rather than appended to. +# +# TYPE DATABASE USER ADDRESS METHOD + +# The postgres superuser over the Unix socket, peer-authenticated. This is the +# path install.yml uses to set the superuser password from Vault, and the one +# that still works if the password in Vault and the password in the database +# ever drift apart — remove it and a wrong password locks everyone out with no +# way back in short of single-user mode. +local all postgres peer + +# Local connections for everything else still need a password: `psql` from a +# shell on this container is not a stronger claim of identity than a +# connection from the LAN, and treating it as one would mean an app's database +# is readable by anyone who can open a shell here. +local all all {{ app_config.POSTGRES_AUTH_METHOD }} + +{% for cidr in app_config.POSTGRES_ALLOWED_CIDRS %} +# {{ cidr }} +host all all {{ cidr }} {{ app_config.POSTGRES_AUTH_METHOD }} +{% endfor %} + +# No `host all all 0.0.0.0/0` fallback, deliberately. A network not listed +# above is refused at connection time with a clear "no pg_hba.conf entry" +# error, which is a better failure than a database quietly reachable from +# somewhere nobody meant to allow. diff --git a/src/shared/postgres/ansible/proxmox/vars.yml b/src/shared/postgres/ansible/proxmox/vars.yml new file mode 100644 index 0000000..43ed020 --- /dev/null +++ b/src/shared/postgres/ansible/proxmox/vars.yml @@ -0,0 +1,75 @@ +--- +# Proxmox-specific overrides, merged over ../../common/vars.yml (these win). +# Reachable in install.yml and the templates as `app_config`. +env_defaults: + # 17, not the 13 the Unraid Compose stack was pinned to. That pin existed + # because the data directory predated this repo and an in-place major + # upgrade of a running container was not worth the risk; this is a fresh + # cluster loaded by dump/restore, so there is nothing to stay compatible + # with. Postgres comes from the PGDG archive rather than Debian's own + # precisely so this is a choice rather than whatever the distro ships. + # + # Changing this on an existing host does NOT upgrade it — the new major's + # package installs alongside the old one and pg_createcluster makes a + # second, empty cluster. Moving data between majors is pg_dumpall/psql or + # pg_upgradecluster, a run-once operation that belongs in a runbook. + POSTGRES_VERSION: "17" + + # Debian's postgresql-common lays a cluster out as /etc/postgresql// + # and /var/lib/postgresql//. `main` is the one the + # package creates on install; install.yml builds both paths from this and + # POSTGRES_VERSION rather than restating them here, so a version bump can't + # leave a stale path behind. + POSTGRES_CLUSTER: main + + # Listen on every interface. Unlike the Unraid deployment — where the + # container published 5432 and Docker did the forwarding — this LXC has an + # address of its own, and Postgres itself is what has to answer on it. What + # may connect is constrained by pg_hba below, not by the bind address. + POSTGRES_LISTEN_ADDRESSES: "*" + + # Networks allowed to authenticate, rendered into pg_hba.conf in order. + # A list of CIDRs rather than one LAN-wide entry, so that what can reach the + # database reads as a decision: + # - the LAN, covering the Ansible controller that provisions per-app + # databases and every Proxmox guest that connects to one + # - the k3s pod CIDR, for cluster apps pointed at this instance rather + # than the in-cluster CloudNativePG one + POSTGRES_ALLOWED_CIDRS: + - 192.168.50.0/24 + - 10.42.0.0/16 + + # scram-sha-256, not md5: the default since Postgres 14, md5 is deprecated + # upstream, and every client here is a current libpq or psycopg2 — there is + # no legacy driver to accommodate. + POSTGRES_AUTH_METHOD: scram-sha-256 + + # --- Tuning ------------------------------------------------------------ + # + # Sized against the LXC's 2048 MiB in ../../terraform/variables.tf. These do + # not track that value — raise the container's memory and these stay put + # until edited. + POSTGRES_MAX_CONNECTIONS: "100" + POSTGRES_SHARED_BUFFERS: 512MB + POSTGRES_EFFECTIVE_CACHE_SIZE: 1536MB + POSTGRES_MAINTENANCE_WORK_MEM: 128MB + POSTGRES_WORK_MEM: 8MB + + # ZFS-specific, and the reason this is not a copy of a generic tuning guide. + # + # full_page_writes exists to survive a torn page — half of an 8 KiB write + # landing before a crash. ZFS is copy-on-write and never overwrites a live + # block in place, so that failure mode does not arise and the full-page + # images are pure write amplification, which on this host is also + # replication traffic. Safe *only* on copy-on-write storage; the same + # setting on ext4 or xfs is a data-loss bug, which is why it lives in the + # Proxmox vars file and not in common/vars.yml. + POSTGRES_FULL_PAGE_WRITES: "off" + + # --- Service ----------------------------------------------------------- + # + # `lxc_app` enables and starts `app.service_name` once install.yml finishes. + # `postgresql` is the packaging's umbrella unit, which pulls in the + # per-cluster postgresql@-.service — targeting the umbrella + # means this needs no edit on a major bump. + POSTGRES_SERVICE_NAME: postgresql diff --git a/src/shared/postgres/ansible/unraid/.env.example b/src/shared/postgres/ansible/unraid/.env.example new file mode 100644 index 0000000..5be7262 --- /dev/null +++ b/src/shared/postgres/ansible/unraid/.env.example @@ -0,0 +1,18 @@ +# Reference only — real values are rendered by Ansible from vars.yml (non- +# secret) and Vault path homelab/shared/postgres (secret). Do not fill this in +# and deploy it directly. + +# Non-secret (see vars.yml) +POSTGRES_DATA_PATH=/mnt/user/appdata/shared-postgres/data +POSTGRES_PORT=5432 +POSTGRES_SUPERUSER=postgres + +# Non-secret, supplied by the compose_stack role rather than either vars.yml — +# the path it copies ansible/unraid/icon.png to on the host, for the +# net.unraid.docker.icon label. Empty when the stack ships no icon.png, which +# just leaves Unraid's question-mark placeholder. Override it in vars.yml to +# point at a hosted PNG instead. +STACK_ICON=/mnt/user/appdata/icons/postgres.png + +# Secret — Vault key: POSTGRES_SUPERUSER_PASSWORD +POSTGRES_SUPERUSER_PASSWORD= diff --git a/src/shared/postgres/ansible/unraid/docker-compose.yml b/src/shared/postgres/ansible/unraid/docker-compose.yml new file mode 100644 index 0000000..88e16e3 --- /dev/null +++ b/src/shared/postgres/ansible/unraid/docker-compose.yml @@ -0,0 +1,32 @@ +services: + postgres: + image: postgres:13 + container_name: shared-postgres + restart: unless-stopped + # Unraid's Docker page normally reads icon, WebUI link and console shell + # from the dockerMan template that created the container. A Compose stack + # has no template, so 6.10+ falls back to these labels; without them the + # container renders as a question mark on the Docker page. + # + # No `net.unraid.docker.webui` — Postgres has no web interface, and the + # label's only effect would be a WebUI entry that goes nowhere. Console + # still needs a shell, and this image is Debian-based, so bash. + labels: + net.unraid.docker.icon: ${STACK_ICON} + net.unraid.docker.shell: bash + environment: + POSTGRES_USER: ${POSTGRES_SUPERUSER} + POSTGRES_PASSWORD: ${POSTGRES_SUPERUSER_PASSWORD} + PGDATA: /var/lib/postgresql/data/pgdata + volumes: + - ${POSTGRES_DATA_PATH}:/var/lib/postgresql/data + ports: + # Exposed on the host so the Ansible controller can provision + # per-app databases/roles directly (see ansible/roles/compose_stack). + - "${POSTGRES_PORT}:5432" + networks: + - shared + +networks: + shared: + name: unraid_shared diff --git a/src/shared/postgres/ansible/unraid/vars.yml b/src/shared/postgres/ansible/unraid/vars.yml new file mode 100644 index 0000000..e1fd4a6 --- /dev/null +++ b/src/shared/postgres/ansible/unraid/vars.yml @@ -0,0 +1,16 @@ +--- +# Unraid-specific overrides, merged over ../../common/vars.yml (these win). +env_defaults: + POSTGRES_DATA_PATH: /mnt/user/appdata/shared-postgres/data + + # --- Unraid Docker page presentation (net.unraid.docker.icon label) ------ + # + # STACK_ICON is normally left to the compose_stack role, which points it at + # /mnt/user/appdata/icons/.png when this directory contains an + # icon.png. Uncomment to use a hosted icon instead of committing one: + # + # STACK_ICON: https://example.org/postgres.png + # + # PNG only — Unraid renders its placeholder for SVG and nothing at all for + # WebP, and a URL it can't reach when the Docker page renders fails the same + # way, which is the argument for committing the file. diff --git a/src/shared/postgres/common/vars.yml b/src/shared/postgres/common/vars.yml new file mode 100644 index 0000000..c0f6b16 --- /dev/null +++ b/src/shared/postgres/common/vars.yml @@ -0,0 +1,8 @@ +--- +# Portable config — see ../ansible//vars.yml for the values that +# depend on where this runs. +env_defaults: + POSTGRES_PORT: "5432" + POSTGRES_SUPERUSER: postgres + # POSTGRES_SUPERUSER_PASSWORD is a secret — sourced from Vault + # (homelab/shared/postgres), never set here. diff --git a/src/shared/postgres/terraform/README.md b/src/shared/postgres/terraform/README.md new file mode 100644 index 0000000..1a00dac --- /dev/null +++ b/src/shared/postgres/terraform/README.md @@ -0,0 +1,103 @@ +# terraform/ — postgres + +The LXC the shared Postgres runs in on Proxmox, plus the ZFS replication job +that copies it to the second node. Called from +`build/config/terraform/main.tf`, which holds the provider and the +`backend "pg"` state config. + +Terraform's scope stops at the guest and its replication job — installing and +configuring Postgres inside it is `../ansible/proxmox/`. + +## Required inputs + +Everything else has a default (see `variables.tf`): + +| Variable | Example | +|---|---| +| `node_name` | `turtle-proxmox-01` | +| `template_file_id` | `local:vztmpl/debian-12-standard_12.7-1_amd64.tar.zst` | +| `ip_address` | `192.168.50.54/24` | +| `gateway` | `192.168.50.254` | +| `ssh_public_keys` | the public half of the key in Vault at `homelab/ci/ssh` | + +Two defaults are worth setting deliberately rather than inheriting: +`datastore_id` (`AppData`) and `replication_target_node` +(`turtle-proxmox-02`). See below. + +## The state circularity, and how it is avoided + +Terraform keeps its state in Postgres — so provisioning *the Postgres LXC* +with Terraform looks circular: the database has to exist before `terraform +init` works, and Terraform is what creates the container the database runs in. + +It is avoided by putting state somewhere this configuration has no hand in +building: the CloudNativePG cluster on the k3s Pis, at +`k3s_postgres_loadbalancer_ip` (192.168.50.81). That cluster is deployed by +Ansible against hardware Terraform never touches, so nothing here is a +prerequisite for it. + +The cost is a dependency the rest of the repo avoids — `terraform apply` now +needs the Pis up — and it is worth paying because the alternative is a +bootstrap dance (local state, apply, migrate) that has to be got right exactly +once and is unrecoverable if it is not. Full ordering in +`docs/postgres-proxmox.md`. + +## On the ZFS pool, on purpose + +`datastore_id` defaults to `AppData`, a ZFS pool, rather than `local-lvm`. +Proxmox replicates by sending ZFS snapshots, so a guest on LVM-thin has no +replication available to it at all — the option simply is not offered. + +The storage has to exist under this exact name on the target node too. `pvesr` +resolves the volume by storage ID on the far side, so a pool named differently +there fails at replication time rather than at apply time, which is a much +later and quieter place to find out. + +## Replication is a per-guest job, not a pool property + +This is the part that is easy to get wrong from the UI. A ZFS pool is not +"replicated"; each *guest* on it has a replication job. So a container created +on `AppData` with no job is a single point of failure that looks identical to +a replicated one in the storage view. + +Hence `proxmox_replication` in `main.tf` rather than a manual `pvesr +create-local-job` — the job is part of what defines this guest, and leaving it +to a hand-run command means a rebuild silently produces an unreplicated +database. + +`replication_target_node = null` disables it. That is correct on a single-node +cluster, but set it explicitly, because null is also what "someone forgot" +looks like. + +## Everything on one volume + +There is no `mount_point` block splitting `PGDATA` or the WAL onto a second +volume, and unlike Forgejo's version of this decision it is a correctness +requirement rather than a tidiness preference: replication snapshots each +volume independently, so two volumes replicate as two snapshots taken at +different instants. See `../ansible/proxmox/README.md`. + +The practical consequence is that running low on space has exactly one fix: +raise `disk_size`. There is no second disk to add, and no way to move `PGDATA` +off the rootfs without giving up the atomic snapshot. That is a smaller +imposition than it sounds — an increase applies as a `pct resize` of the +rootfs, and on ZFS that is a refquota change rather than a partition +operation, so it takes effect while the cluster is running and needs nothing +done inside the guest afterwards. Size generously up front regardless: +shrinking is a replacement, which `prevent_destroy` refuses. + +## `prevent_destroy` + +`main.tf` sets `lifecycle { prevent_destroy = true }`. Shrinking `disk_size` +or changing `template_file_id` are replacements rather than in-place updates, +and replacing this container destroys every database on it — including any +whose only other copy is a replication target that would be recreated along +with it. To genuinely retire the guest, remove the block first; it cannot be +overridden from the command line. + +## Sizing + +`cores = 2`, `memory = 2048`, `disk_size = 256`. The tuning in +`../ansible/proxmox/vars.yml` is sized against that 2048 MiB and does **not** +track it — raise the container's memory and `shared_buffers` / +`effective_cache_size` stay where they are until edited. diff --git a/src/shared/postgres/terraform/main.tf b/src/shared/postgres/terraform/main.tf new file mode 100644 index 0000000..3f50c71 --- /dev/null +++ b/src/shared/postgres/terraform/main.tf @@ -0,0 +1,147 @@ +# The LXC the shared Postgres runs in on Proxmox. Terraform's scope stops at +# the guest and its replication job — installing and configuring Postgres +# inside it is ../ansible/proxmox/, run by the `lxc_app` role. +# +# Two things make this module different from src/forgejo/terraform, and both +# come from the same decision: the database lives on the `AppData` ZFS pool so +# that Proxmox can replicate it to the second node. +# +# 1. `datastore_id` defaults to a ZFS pool, not `local-lvm`. Only ZFS-backed +# volumes can be replicated — pvesr works by sending ZFS snapshots, so a +# guest on LVM-thin simply has no replication available to it. +# +# 2. Everything is on the rootfs. There is no `mount_point` block splitting +# PGDATA or the WAL onto a second volume, and that is a correctness +# requirement here rather than the tidiness preference it is for Forgejo. +# Replication snapshots each of a guest's volumes independently, so a +# cluster split across two volumes replicates as two snapshots taken at +# different instants — the far node can end up with a data directory and +# a WAL that disagree, which is not a state Postgres knows how to recover +# from. One volume means one atomic snapshot, which is exactly the +# crash-consistent image WAL replay is designed to handle. + +resource "proxmox_virtual_environment_container" "postgres" { + node_name = var.node_name + vm_id = var.vm_id + tags = var.tags + + # Unprivileged: Postgres binds 5432 and runs as its own service account. + # Nothing it does needs host privileges. + unprivileged = true + start_on_boot = true + + # systemd >= 252 in an unprivileged container needs a cgroup namespace of + # its own to set up delegation, which LXC only grants with nesting on. + # Without it systemd still boots, but units using the sandboxing directives + # (PrivateTmp, ProtectSystem, ProtectHome) fail to start — and the unit + # ../ansible/proxmox/ installs is exactly that shape. This is what the + # provider's "WARN: Systemd NNN detected. You may need to enable nesting." + # is pointing at, and it is the Proxmox UI's own default for unprivileged + # containers. + # + # Safe here precisely *because* the container is unprivileged: nesting on a + # privileged container widens a real escape surface, on an unprivileged one + # it grants a namespace the kernel already refuses to let root out of. + features { + nesting = true + } + + # Setting this on an *already existing* container takes root@pam. PVE + # hard-codes that for feature flags — it is not an ACL and no role grants + # it — and while its own error message exempts nesting, the provider sends + # the other flags alongside, so the exemption doesn't apply. A 403 reading + # "changing feature flags (except nesting) is only allowed for root@pam" + # means this container predates the block; fix it on the node with + # + # pct set --features nesting=1 && pct reboot + # + # after which the plan reads clean, because the provider only sends + # `features` when it differs. Whether a *fresh* create hits the same wall is + # untested — CT 161 was already up when this block was added. + + initialization { + hostname = var.hostname + + ip_config { + ipv4 { + address = var.ip_address + gateway = var.gateway + } + } + + dynamic "dns" { + for_each = var.dns_servers == null ? [] : [1] + + content { + servers = var.dns_servers + } + } + + user_account { + keys = var.ssh_public_keys + } + } + + cpu { + cores = var.cores + } + + memory { + dedicated = var.memory + swap = var.swap + } + + disk { + datastore_id = var.datastore_id + size = var.disk_size + + # Defaults to true, stated explicitly because the replication job below + # is silently pointless without it — a job on a guest whose only volume + # is excluded runs successfully and sends nothing. + replicate = true + } + + network_interface { + name = "eth0" + bridge = var.network_bridge + vlan_id = var.network_vlan_id + firewall = var.network_firewall + enabled = true + } + + operating_system { + template_file_id = var.template_file_id + type = var.operating_system_type + } + + # This container is the database. Several innocuous-looking edits (shrinking + # `disk_size`, changing `template_file_id`) are replacements rather than + # in-place updates, and a replacement here loses every database on the host. + lifecycle { + prevent_destroy = true + } +} + +# ZFS replication to the second node. Terraform-managed rather than created by +# hand in the UI, because replication is configured *per guest*, not per pool: +# the `AppData` pool being "replicated" is really every guest on it having a +# job like this one, and a container created without one is quietly a single +# point of failure that looks identical in the storage view. +# +# Note what this does and does not buy. The far copy is an inactive volume, +# not a running guest — recovery is migrating the container to the other node, +# which is a deliberate act unless a HA resource is added on top. And because +# replication is asynchronous, a failover loses every commit since the last +# successful run, up to one `schedule` interval of them. +resource "proxmox_replication" "postgres" { + count = var.replication_target_node == null ? 0 : 1 + + # -. Job 0 is this module's; a second target would be job 1. + id = "${var.vm_id}-0" + target = var.replication_target_node + type = "local" + schedule = var.replication_schedule + comment = "managed-by-homelab-iac — shared Postgres to ${var.replication_target_node}" + + depends_on = [proxmox_virtual_environment_container.postgres] +} diff --git a/src/shared/postgres/terraform/outputs.tf b/src/shared/postgres/terraform/outputs.tf new file mode 100644 index 0000000..aeee0e8 --- /dev/null +++ b/src/shared/postgres/terraform/outputs.tf @@ -0,0 +1,34 @@ +# Ansible finds the guest through the Proxmox API (inventory/proxmox.yml), not +# by having these wired into inventory — see the note at the top of +# build/config/terraform/main.tf. They exist so the backup job and anything +# else in the root module can refer to the guest without restating its +# identity. + +output "hostname" { + description = "Container hostname — the name this guest appears under in Ansible inventory." + value = proxmox_virtual_environment_container.postgres.initialization[0].hostname +} + +output "vm_id" { + description = "Proxmox VMID. Pass to the vzdump job if it is pinned to an explicit VMID list." + value = proxmox_virtual_environment_container.postgres.vm_id +} + +output "ip_address" { + description = <<-EOT + IPv4 address without the prefix length. This is the address every app's + `db.provision_host` and `DB_HOST` has to agree with — see the note on + `var.ip_address` for why keeping it in one place matters. + EOT + value = split("/", var.ip_address)[0] +} + +output "node_name" { + description = "Proxmox node the container was created on." + value = proxmox_virtual_environment_container.postgres.node_name +} + +output "replication_target_node" { + description = "Node this guest replicates to, or null if replication is disabled." + value = var.replication_target_node +} diff --git a/src/shared/postgres/terraform/providers.tf b/src/shared/postgres/terraform/providers.tf new file mode 100644 index 0000000..6714e16 --- /dev/null +++ b/src/shared/postgres/terraform/providers.tf @@ -0,0 +1,13 @@ +terraform { + required_version = ">= 1.6" + + required_providers { + proxmox = { + source = "bpg/proxmox" + # Deliberately unpinned until the first real `terraform init` — pin to + # whatever it resolves, and commit .terraform.lock.hcl (which is not + # gitignored) so everyone and CI get the same provider. + # version = "~> 0.0" + } + } +} \ No newline at end of file diff --git a/src/shared/postgres/terraform/variables.tf b/src/shared/postgres/terraform/variables.tf new file mode 100644 index 0000000..9c900fa --- /dev/null +++ b/src/shared/postgres/terraform/variables.tf @@ -0,0 +1,207 @@ +# Nothing environment-specific is hardcoded in the module — every value the +# caller could reasonably differ on is a variable, set from +# build/config/terraform/main.tf. + +variable "node_name" { + description = "Proxmox node to create the container on, e.g. `turtle-proxmox-01`." + type = string +} + +variable "vm_id" { + description = <<-EOT + Fixed VMID rather than letting Proxmox pick the next free one. Two things + key off it: the vzdump job in build/config/ansible/roles/pve_backup can be + pinned to an explicit VMID list, and the replication job ID in main.tf is + derived from it — a guest whose ID moves between rebuilds drops out of the + first and orphans the second. + EOT + type = number + default = 161 +} + +variable "hostname" { + description = <<-EOT + Container hostname, which is also the inventory hostname Ansible sees via + the community.proxmox dynamic inventory — so it has to match + build/config/ansible/inventory/host_vars/.yml, the file that declares + the `apps:` list installing Postgres into it. + EOT + type = string + default = "postgres" +} + +variable "cores" { + description = "CPU cores. Postgres at homelab query volumes is not CPU-bound; 2 is comfortable." + type = number + default = 2 +} + +variable "memory" { + description = <<-EOT + RAM in MiB. 2048 leaves room for `shared_buffers` (set to a quarter of this + in ../ansible/proxmox/vars.yml) plus per-connection work memory and the + kernel page cache Postgres leans on for everything not in shared_buffers. + Raise both together — the Postgres setting does not track this one. + EOT + type = number + default = 2048 +} + +variable "swap" { + description = <<-EOT + Swap in MiB. Deliberately small: a swapping database is a slow database, + and the useful behaviour when memory really runs out is a fast failure + rather than a cluster that stays up and misses every latency target. + EOT + type = number + default = 512 +} + +variable "disk_size" { + description = <<-EOT + Root disk in GiB, holding the OS *and* every database - nothing is split + onto a second volume, on purpose (see the header comment in main.tf: one + volume is what makes a replication snapshot atomic). Size it for the data + plus WAL plus headroom for any dump written locally. + + Because of that, this is the only place a "the database is running out of + space" problem gets solved: adding a second disk is not an option here, so + the number just goes up. An increase applies as a `pct resize` of the + rootfs, which on a ZFS-backed guest is a refquota change - it takes effect + online, with no data move and no filesystem grow step, so Postgres does + not need stopping. Going the other way is a replacement rather than an + in-place update, and `prevent_destroy` in main.tf blocks it, so pick a + size with room in it rather than raising this every few months. + + The real ceiling is free space on `datastore_id`, which Terraform will not + warn about - an apply that overcommits the pool succeeds, and the guest + hits ENOSPC later, when a write fails. + EOT + type = number + default = 256 +} + +variable "datastore_id" { + description = <<-EOT + Proxmox storage the root disk is created on. Defaults to the `AppData` ZFS + pool rather than `local-lvm`, because replication sends ZFS snapshots and + is therefore unavailable to a guest on any other storage type. + + The storage has to exist under this exact name on the replication target + node as well — pvesr resolves the volume by storage ID on the far side, so + a pool named differently there fails at replication time rather than at + apply time. + EOT + type = string + default = "AppData" +} + +variable "replication_target_node" { + description = <<-EOT + Proxmox node to replicate this guest to, e.g. `turtle-proxmox-02`. Null + disables the replication job entirely, which is the right setting for a + single-node cluster — but note it is also indistinguishable from "someone + forgot", so set it explicitly rather than relying on the default. + EOT + type = string + default = null +} + +variable "replication_schedule" { + description = <<-EOT + How often the ZFS snapshot is sent, as a systemd calendar event. This is + the data-loss window: an unplanned failover loses every transaction + committed since the last successful run, so the interval trades write + amplification against how much you are prepared to lose. + + `*/5` rather than Proxmox's own `*/15` default — sends are incremental, so + a shorter interval on a quiet database costs little beyond snapshot churn, + and 5 minutes of lost commits is a materially different conversation from + 15. + EOT + type = string + default = "*/5" +} + +variable "template_file_id" { + description = <<-EOT + Container template volume ID, e.g. + `local:vztmpl/debian-12-standard_12.7-1_amd64.tar.zst`. Must already be + downloaded on the node — see "Prerequisites → Proxmox" in the top-level + README.md. List what you have with `pveam list local`. + EOT + type = string +} + +variable "operating_system_type" { + description = "Template OS family, as Proxmox labels it. Match the template above." + type = string + default = "debian" +} + +variable "ip_address" { + description = <<-EOT + Static IPv4 in CIDR form, e.g. `192.168.50.54/24`. Static and not DHCP for + a stronger reason than usual: this address is written down in several + places that cannot discover it (the `provision_host` on every app that + declares a `db:`, and each app's own DB_HOST), so it moving is a + multi-file edit rather than a lease renewal. + EOT + type = string + + validation { + condition = can(cidrnetmask(var.ip_address)) + error_message = "ip_address must include a prefix length, e.g. 192.168.50.54/24." + } +} + +variable "gateway" { + description = "IPv4 default gateway." + type = string +} + +variable "dns_servers" { + description = "Resolvers for the container. Leave null to inherit the Proxmox node's." + type = list(string) + default = null +} + +variable "network_bridge" { + description = "Proxmox bridge to attach eth0 to." + type = string + default = "vmbr0" +} + +variable "network_vlan_id" { + description = "VLAN tag for eth0, or null for an untagged bridge." + type = number + default = null +} + +variable "network_firewall" { + description = "Enable the Proxmox firewall on this interface." + type = bool + default = false +} + +variable "ssh_public_keys" { + description = <<-EOT + Public keys installed for root in the container. Ansible authenticates + with the matching private half (`homelab/ci/ssh` in Vault); without at + least one key here the guest is unreachable and playbooks/proxmox.yml has + nothing it can do with it. + EOT + type = list(string) +} + +variable "tags" { + description = <<-EOT + Proxmox tags, shown in the UI and read by the Ansible dynamic inventory. + `terraform` is load-bearing: build/config/ansible/inventory/proxmox.yml + filters on it to tell guests this repo creates apart from ones made by + hand on the node. Drop it and this container is simply never deployed + to — silently, with no error anywhere. + EOT + type = list(string) + default = ["postgres", "shared", "terraform"] +}