--- # Bootstraps the homelab-utils k3s cluster from bare Pis: no Terraform, the # hosts already exist (inventory/hosts.yml → k3s_cluster). Manual-only for # now — not wired into .forgejo/workflows/deploy.yml, since converging 4 # physical nodes on every push is a bigger blast radius than restarting a # Compose stack (same reasoning the repo already applies to `terraform # apply`). Run by hand: # # ansible-playbook playbooks/k3s.yml # # Every play below the SSH-key import is tagged so a single piece can be # converged without touching the rest — deploying one cluster service # shouldn't mean re-running the node install across 4 Pis: # # --tags nodes k3s install/join + exec-line drift (both node plays) # --tags metallb MetalLB chart + IPAddressPool # --tags monitoring kube-prometheus-stack chart # --tags postgres CloudNativePG operator + Cluster # --tags cert-manager cert-manager chart + ClusterIssuer # --tags traefik dashboard route on k3s's bundled Traefik # --tags services all five of the above, no node install # --tags apps the k3s_apps: list (roles/k3s_app) — apps, not # cluster services; narrow further with -e only_apps=… # --tags upgrades unattended-upgrades config # --tags kubeconfig re-fetch the admin kubeconfig # # e.g. `ansible-playbook playbooks/k3s.yml --tags postgres`. The SSH-key # import is tagged `always` rather than given a name of its own: it's not a # thing you'd select, it's the prerequisite for any of these connecting at # all, so it has to survive every --tags filter. # # `services` covers the cluster's own infrastructure and `apps` covers what # runs on top of it; neither implies the other, so a run meaning "everything # except reinstalling k3s" is `--tags services,apps`. # # Prerequisites (see README.md "K3s (Raspberry Pi)"): the `ansible` user # exists on every Pi with NOPASSWD sudo and this repo's SSH key installed, # and homelab/k3s-homelab-utils/K3S_TOKEN is already set in Vault — see # docs/vault-secrets.md. Both plays fetch that same fixed token independently # rather than one generating it and handing it to the other, which is what # makes a full rebuild (wipe both SD cards, reinstall) reproduce the same # cluster identity. # First, before anything tries to connect: pull the SSH key the plays below # authenticate with out of Vault and onto the controller. See # k3s_ssh_key.yml — it's a separate file because k3s_maintenance.yml imports # it too. - ansible.builtin.import_playbook: k3s_ssh_key.yml tags: always - name: Bootstrap the k3s control plane hosts: k3s_control_plane gather_facts: true tags: nodes roles: - k3s_node # serial: 1 not for joining itself (idempotent, safe in parallel) but for # roles/k3s_node's exec-line drift detection: if k3s_extra_args changes and # every worker's k3s-agent restarts to pick it up, one at a time keeps more # than one node's kubelet from bouncing simultaneously. See tasks/agent.yml. - name: Join k3s worker nodes hosts: k3s_workers gather_facts: true serial: 1 tags: nodes roles: - k3s_node # Against the control plane only — it just drops manifests for k3s's own # helm-controller and deploy controller to reconcile, so it doesn't need # every node like the plays above. Runs after workers have joined so the # resulting speaker DaemonSet schedules across the whole cluster on first # reconcile, though this doesn't strictly matter: the controller picks up # new nodes on its own regardless of ordering. Before monitoring so a # LoadBalancer Service (e.g. exposing Prometheus/Grafana later) has # somewhere to get an IP from as soon as it's requested. - name: Deploy MetalLB (LoadBalancer IPs) hosts: k3s_control_plane gather_facts: false tags: - services - metallb roles: - k3s_metallb # Against the control plane only — it just drops a HelmChart manifest for # k3s's own helm-controller to reconcile, so it doesn't need every node like # the plays above. Runs after workers have joined so the resulting # node-exporter DaemonSet schedules across the whole cluster on first # reconcile, though this doesn't strictly matter: the controller picks up # new nodes on its own regardless of ordering. - name: Deploy cluster monitoring (Prometheus) hosts: k3s_control_plane gather_facts: false tags: - services - monitoring roles: - k3s_monitoring # Against the control plane only, same reasoning as monitoring/MetalLB above # — it just drops manifests for k3s's own controllers to reconcile. Needs # repo_root (unlike the other k3s roles) because it reads config from # src/shared/postgres/, the same shared-service config Unraid/Proxmox # already deploy from — see roles/k3s_postgres/tasks/main.yml. - name: Deploy shared Postgres (CloudNativePG) hosts: k3s_control_plane gather_facts: false tags: - services - postgres vars: repo_root: "{{ playbook_dir }}/../../../.." roles: - k3s_postgres # Against the control plane only, same reasoning as the services above. # Before the apps play because an app's Ingress annotates itself against the # ClusterIssuer this creates — not that ordering is load-bearing (k3s's # deploy controller retries, and cert-manager picks up an Ingress whenever it # appears), but an app deployed first would sit without a certificate until # this ran, which reads as a broken deploy rather than a pending one. - name: Deploy cert-manager (TLS certificates) hosts: k3s_control_plane gather_facts: false tags: - services - cert-manager roles: - k3s_cert_manager # Against the control plane only, same reasoning as the services above. The # odd one out among them: it installs nothing, because k3s installs Traefik # itself — it only adjusts what k3s already put there, via a HelmChartConfig. # After cert-manager because publishing the dashboard on a hostname asks for a # Certificate from the ClusterIssuer that play creates; ordering isn't # load-bearing (the deploy controller retries), it just avoids a route sitting # without a certificate in between. - name: Configure Traefik (ingress + dashboard) hosts: k3s_control_plane gather_facts: false tags: - services - traefik roles: - k3s_traefik # Apps, as opposed to the cluster services above — see roles/k3s_app for the # distinction and inventory/group_vars/k3s_cluster.yml for the list. Against # the control plane because that's where the manifests directory is; nothing # about an app belongs to that Pi in particular. # # Note what a green run here does and doesn't mean, the same caveat every # service play above carries: the role renders manifests for k3s's # controllers to reconcile, so success means the files landed and any # database was provisioned — not that the workload came up. Check with # `kubectl -n get pods`. - name: Converge apps on the cluster hosts: k3s_control_plane gather_facts: false tags: apps vars: repo_root: "{{ playbook_dir }}/../../../.." # Comma-separated app names to restrict this run to, e.g. # `-e only_apps=authentik`. Empty (the default) converges every app in # k3s_apps. Same knob as only_stacks in playbooks/unraid.yml — --tags # can select the apps play as a whole, but not one app within it. only_apps: "" tasks: # "Converge", not "deploy": an entry carrying `state: absent` is torn # down rather than brought up. Removals have to stay in the list to be # acted on — see roles/k3s_app/tasks/main.yml. - name: Converge each declared app ansible.builtin.include_role: name: k3s_app loop: >- {{ k3s_apps | default([]) if only_apps == '' else k3s_apps | default([]) | selectattr('name', 'in', only_apps.split(',')) | list }} loop_control: loop_var: app label: "{{ app.name }}" # Every node, both roles — hands-off patching so the Pis don't need logging # into just to stay updated. Reboots are deliberately not part of this: see # roles/unattended_upgrades and playbooks/k3s_maintenance.yml. - name: Configure unattended upgrades hosts: k3s_cluster gather_facts: true tags: upgrades roles: - unattended_upgrades # Last, and against the control plane specifically (there's only one) — pulls # the admin kubeconfig k3s wrote for itself back to the controller so # `kubectl` works from your workstation. - name: Fetch the cluster kubeconfig hosts: k3s_control_plane gather_facts: false tags: kubeconfig vars: repo_root: "{{ playbook_dir }}/../../../.." tasks: - name: Read the cluster's kubeconfig ansible.builtin.slurp: src: /etc/rancher/k3s/k3s.yaml register: k3s_kubeconfig_raw - name: Ensure the local kubeconfig directory exists ansible.builtin.file: path: "{{ k3s_kubeconfig_local_path | dirname }}" state: directory mode: "0700" delegate_to: localhost become: false # k3s.yaml points at 127.0.0.1 and names everything "default" — both # correct only on the node itself. Rewritten so the file is usable # straight off the controller: the server address becomes reachable from # off-box, and the cluster/context/user names become this cluster's own # rather than colliding with every other "default" in ~/.kube/config. - name: Write the rewritten kubeconfig to the controller ansible.builtin.copy: dest: "{{ k3s_kubeconfig_local_path }}" content: >- {{ (k3s_kubeconfig_raw.content | b64decode) | replace('127.0.0.1', ansible_host) | regex_replace('\\bdefault\\b', k3s_cluster_name) }} mode: "0600" delegate_to: localhost become: false - name: Show how to use the fetched kubeconfig ansible.builtin.debug: msg: >- Kubeconfig written to {{ k3s_kubeconfig_local_path }}. Use it with `export KUBECONFIG={{ k3s_kubeconfig_local_path }}`, or merge it into ~/.kube/config by hand.