100 lines
3.7 KiB
YAML
100 lines
3.7 KiB
YAML
---
|
|
# Rolling "reboot this node if unattended-upgrades left one pending" for one
|
|
# k3s node — one iteration of the serial: 1 loop in
|
|
# playbooks/k3s_maintenance.yml. roles/unattended_upgrades installs updates on
|
|
# its own schedule with Automatic-Reboot disabled, so a kernel/library update
|
|
# can sit applied-but-inactive on a node indefinitely; this is what actually
|
|
# reboots it — cordoned and drained first, so workloads move off before the
|
|
# node disappears rather than during.
|
|
#
|
|
# kubectl commands are delegated to the control-plane node and run as
|
|
# `k3s kubectl`, k3s's own bundled client — no separate kubectl install or
|
|
# local kubeconfig needed. Works the same whether the node having its turn
|
|
# right now *is* the control plane: delegating to itself, over SSH, before it
|
|
# reboots itself.
|
|
|
|
- name: Sanity-check there is exactly one control-plane node
|
|
ansible.builtin.assert:
|
|
that: groups['k3s_control_plane'] | length == 1
|
|
fail_msg: >-
|
|
k3s_maintenance delegates kubectl to groups['k3s_control_plane'][0] —
|
|
it assumes a single control-plane node. Update this role before adding
|
|
a second one for HA.
|
|
quiet: true
|
|
|
|
- name: Check whether a reboot is required
|
|
ansible.builtin.stat:
|
|
path: /var/run/reboot-required
|
|
register: k3s_reboot_required
|
|
|
|
- name: Node is up to date — nothing to do
|
|
ansible.builtin.debug:
|
|
msg: "{{ inventory_hostname }}: no reboot required, skipping."
|
|
when: not k3s_reboot_required.stat.exists
|
|
|
|
- name: Reboot this node if unattended-upgrades left one pending
|
|
when: k3s_reboot_required.stat.exists
|
|
block:
|
|
# --force: this is a homelab, not a cluster with a policy against bare
|
|
# pods — better to evict them than have a stray one block every rebuild.
|
|
# --delete-emptydir-data: emptyDir contents are expected to be
|
|
# disposable; anything that isn't shouldn't be using emptyDir.
|
|
- name: Cordon and drain the node
|
|
ansible.builtin.command:
|
|
argv:
|
|
- k3s
|
|
- kubectl
|
|
- drain
|
|
- "{{ inventory_hostname }}"
|
|
- --ignore-daemonsets
|
|
- --delete-emptydir-data
|
|
- --force
|
|
- --timeout=120s
|
|
delegate_to: "{{ groups['k3s_control_plane'][0] }}"
|
|
become: false
|
|
changed_when: true
|
|
|
|
- name: Reboot the node
|
|
ansible.builtin.reboot:
|
|
reboot_timeout: 300
|
|
|
|
# Polls rather than trusting the reboot handshake alone — the node can be
|
|
# reachable over SSH before k3s (and, if this is the control-plane node
|
|
# itself, the API server it just took down with it) has finished coming
|
|
# back up.
|
|
- name: Wait for the node to report Ready again
|
|
ansible.builtin.command:
|
|
argv:
|
|
- k3s
|
|
- kubectl
|
|
- wait
|
|
- --for=condition=Ready
|
|
- "node/{{ inventory_hostname }}"
|
|
- --timeout=20s
|
|
delegate_to: "{{ groups['k3s_control_plane'][0] }}"
|
|
become: false
|
|
register: k3s_node_ready
|
|
changed_when: false
|
|
failed_when: false
|
|
until: k3s_node_ready.rc == 0
|
|
retries: 12
|
|
delay: 15
|
|
|
|
- name: Fail if the node never came back Ready
|
|
ansible.builtin.fail:
|
|
msg: >-
|
|
{{ inventory_hostname }} rebooted but never reported Ready again —
|
|
it's left cordoned; check it by hand before re-running this against
|
|
the rest of the cluster.
|
|
when: k3s_node_ready.rc != 0
|
|
|
|
- name: Uncordon the node
|
|
ansible.builtin.command:
|
|
argv: [k3s, kubectl, uncordon, "{{ inventory_hostname }}"]
|
|
delegate_to: "{{ groups['k3s_control_plane'][0] }}"
|
|
become: false
|
|
changed_when: true
|
|
|
|
- name: Node rebooted and rejoined the cluster
|
|
ansible.builtin.debug:
|
|
msg: "{{ inventory_hostname }}: rebooted, drained and uncordoned cleanly."
|