homelab/build/config/ansible/playbooks/k3s.yml

---
# Bootstraps the homelab-utils k3s cluster from bare Pis: no Terraform, the
# hosts already exist (inventory/hosts.yml → k3s_cluster). Manual-only for
# now — not wired into .forgejo/workflows/deploy.yml, since converging 4
# physical nodes on every push is a bigger blast radius than restarting a
# Compose stack (same reasoning the repo already applies to `terraform
# apply`). Run by hand:
#
#   ansible-playbook playbooks/k3s.yml
#
# Every play below the SSH-key import is tagged so a single piece can be
# converged without touching the rest — deploying one cluster service
# shouldn't mean re-running the node install across 4 Pis:
#
#   --tags nodes        k3s install/join + exec-line drift (both node plays)
#   --tags metallb      MetalLB chart + IPAddressPool
#   --tags monitoring   kube-prometheus-stack chart
#   --tags postgres     CloudNativePG operator + Cluster
#   --tags cert-manager cert-manager chart + ClusterIssuer
#   --tags traefik      dashboard route on k3s's bundled Traefik
#   --tags services     all five of the above, no node install
#   --tags apps         the k3s_apps: list (roles/k3s_app) — apps, not
#                       cluster services; narrow further with -e only_apps=…
#   --tags upgrades     unattended-upgrades config
#   --tags kubeconfig   re-fetch the admin kubeconfig
#
# e.g. `ansible-playbook playbooks/k3s.yml --tags postgres`. The SSH-key
# import is tagged `always` rather than given a name of its own: it's not a
# thing you'd select, it's the prerequisite for any of these connecting at
# all, so it has to survive every --tags filter.
#
# `services` covers the cluster's own infrastructure and `apps` covers what
# runs on top of it; neither implies the other, so a run meaning "everything
# except reinstalling k3s" is `--tags services,apps`.
#
# Prerequisites (see README.md "K3s (Raspberry Pi)"): the `ansible` user
# exists on every Pi with NOPASSWD sudo and this repo's SSH key installed,
# and homelab/k3s-homelab-utils/K3S_TOKEN is already set in Vault — see
# docs/vault-secrets.md. Both plays fetch that same fixed token independently
# rather than one generating it and handing it to the other, which is what
# makes a full rebuild (wipe both SD cards, reinstall) reproduce the same
# cluster identity.

# First, before anything tries to connect: pull the SSH key the plays below
# authenticate with out of Vault and onto the controller. See
# k3s_ssh_key.yml — it's a separate file because k3s_maintenance.yml imports
# it too.
- ansible.builtin.import_playbook: k3s_ssh_key.yml
  tags: always

- name: Bootstrap the k3s control plane
  hosts: k3s_control_plane
  gather_facts: true
  tags: nodes
  roles:
    - k3s_node

# serial: 1 not for joining itself (idempotent, safe in parallel) but for
# roles/k3s_node's exec-line drift detection: if k3s_extra_args changes and
# every worker's k3s-agent restarts to pick it up, one at a time keeps more
# than one node's kubelet from bouncing simultaneously. See tasks/agent.yml.
- name: Join k3s worker nodes
  hosts: k3s_workers
  gather_facts: true
  serial: 1
  tags: nodes
  roles:
    - k3s_node

# Against the control plane only — it just drops manifests for k3s's own
# helm-controller and deploy controller to reconcile, so it doesn't need
# every node like the plays above. Runs after workers have joined so the
# resulting speaker DaemonSet schedules across the whole cluster on first
# reconcile, though this doesn't strictly matter: the controller picks up
# new nodes on its own regardless of ordering. Before monitoring so a
# LoadBalancer Service (e.g. exposing Prometheus/Grafana later) has
# somewhere to get an IP from as soon as it's requested.
- name: Deploy MetalLB (LoadBalancer IPs)
  hosts: k3s_control_plane
  gather_facts: false
  tags:
    - services
    - metallb
  roles:
    - k3s_metallb

# Against the control plane only — it just drops a HelmChart manifest for
# k3s's own helm-controller to reconcile, so it doesn't need every node like
# the plays above. Runs after workers have joined so the resulting
# node-exporter DaemonSet schedules across the whole cluster on first
# reconcile, though this doesn't strictly matter: the controller picks up
# new nodes on its own regardless of ordering.
- name: Deploy cluster monitoring (Prometheus)
  hosts: k3s_control_plane
  gather_facts: false
  tags:
    - services
    - monitoring
  roles:
    - k3s_monitoring

# Against the control plane only, same reasoning as monitoring/MetalLB above
# — it just drops manifests for k3s's own controllers to reconcile. Needs
# repo_root (unlike the other k3s roles) because it reads config from
# src/shared/postgres/, the same shared-service config Unraid/Proxmox
# already deploy from — see roles/k3s_postgres/tasks/main.yml.
- name: Deploy shared Postgres (CloudNativePG)
  hosts: k3s_control_plane
  gather_facts: false
  tags:
    - services
    - postgres
  vars:
    repo_root: "{{ playbook_dir }}/../../../.."
  roles:
    - k3s_postgres

# Against the control plane only, same reasoning as the services above.
# Before the apps play because an app's Ingress annotates itself against the
# ClusterIssuer this creates — not that ordering is load-bearing (k3s's
# deploy controller retries, and cert-manager picks up an Ingress whenever it
# appears), but an app deployed first would sit without a certificate until
# this ran, which reads as a broken deploy rather than a pending one.
- name: Deploy cert-manager (TLS certificates)
  hosts: k3s_control_plane
  gather_facts: false
  tags:
    - services
    - cert-manager
  roles:
    - k3s_cert_manager

# Against the control plane only, same reasoning as the services above. The
# odd one out among them: it installs nothing, because k3s installs Traefik
# itself — it only adjusts what k3s already put there, via a HelmChartConfig.
# After cert-manager because publishing the dashboard on a hostname asks for a
# Certificate from the ClusterIssuer that play creates; ordering isn't
# load-bearing (the deploy controller retries), it just avoids a route sitting
# without a certificate in between.
- name: Configure Traefik (ingress + dashboard)
  hosts: k3s_control_plane
  gather_facts: false
  tags:
    - services
    - traefik
  roles:
    - k3s_traefik

# Apps, as opposed to the cluster services above — see roles/k3s_app for the
# distinction and inventory/group_vars/k3s_cluster.yml for the list. Against
# the control plane because that's where the manifests directory is; nothing
# about an app belongs to that Pi in particular.
#
# Note what a green run here does and doesn't mean, the same caveat every
# service play above carries: the role renders manifests for k3s's
# controllers to reconcile, so success means the files landed and any
# database was provisioned — not that the workload came up. Check with
# `kubectl -n <namespace> get pods`.
- name: Converge apps on the cluster
  hosts: k3s_control_plane
  gather_facts: false
  tags: apps
  vars:
    repo_root: "{{ playbook_dir }}/../../../.."
    # Comma-separated app names to restrict this run to, e.g.
    # `-e only_apps=authentik`. Empty (the default) converges every app in
    # k3s_apps. Same knob as only_stacks in playbooks/unraid.yml — --tags
    # can select the apps play as a whole, but not one app within it.
    only_apps: ""

  tasks:
    # "Converge", not "deploy": an entry carrying `state: absent` is torn
    # down rather than brought up. Removals have to stay in the list to be
    # acted on — see roles/k3s_app/tasks/main.yml.
    - name: Converge each declared app
      ansible.builtin.include_role:
        name: k3s_app
      loop: >-
        {{ k3s_apps | default([]) if only_apps == ''
           else k3s_apps | default([])
                | selectattr('name', 'in', only_apps.split(',')) | list }}
      loop_control:
        loop_var: app
        label: "{{ app.name }}"

# Every node, both roles — hands-off patching so the Pis don't need logging
# into just to stay updated. Reboots are deliberately not part of this: see
# roles/unattended_upgrades and playbooks/k3s_maintenance.yml.
- name: Configure unattended upgrades
  hosts: k3s_cluster
  gather_facts: true
  tags: upgrades
  roles:
    - unattended_upgrades

# Last, and against the control plane specifically (there's only one) — pulls
# the admin kubeconfig k3s wrote for itself back to the controller so
# `kubectl` works from your workstation.
- name: Fetch the cluster kubeconfig
  hosts: k3s_control_plane
  gather_facts: false
  tags: kubeconfig
  vars:
    repo_root: "{{ playbook_dir }}/../../../.."

  tasks:
    - name: Read the cluster's kubeconfig
      ansible.builtin.slurp:
        src: /etc/rancher/k3s/k3s.yaml
      register: k3s_kubeconfig_raw

    - name: Ensure the local kubeconfig directory exists
      ansible.builtin.file:
        path: "{{ k3s_kubeconfig_local_path | dirname }}"
        state: directory
        mode: "0700"
      delegate_to: localhost
      become: false

    # k3s.yaml points at 127.0.0.1 and names everything "default" — both
    # correct only on the node itself. Rewritten so the file is usable
    # straight off the controller: the server address becomes reachable from
    # off-box, and the cluster/context/user names become this cluster's own
    # rather than colliding with every other "default" in ~/.kube/config.
    - name: Write the rewritten kubeconfig to the controller
      ansible.builtin.copy:
        dest: "{{ k3s_kubeconfig_local_path }}"
        content: >-
          {{ (k3s_kubeconfig_raw.content | b64decode)
             | replace('127.0.0.1', ansible_host)
             | regex_replace('\\bdefault\\b', k3s_cluster_name) }}
        mode: "0600"
      delegate_to: localhost
      become: false

    - name: Show how to use the fetched kubeconfig
      ansible.builtin.debug:
        msg: >-
          Kubeconfig written to {{ k3s_kubeconfig_local_path }}. Use it with
          `export KUBECONFIG={{ k3s_kubeconfig_local_path }}`, or merge it
          into ~/.kube/config by hand.