feat(regrade): regrade a native server under memory pressure with a systemd actuator

.github/workflows/ci.yml

@@ -202,9 +202,15 @@ jobs:
           python3 -c 'import ansible.release; print("ansible-core", ansible.release.__version__)'
           mapfile -t modules < <(find tests/unit -maxdepth 1 -name 'test_*.py' -printf '%s %p\n' | LC_ALL=C sort -k1,1nr -k2,2 | cut -d' ' -f2- | awk -v shard="$SHARD" -v shards="$SHARDS" '(NR - 1) % shards == shard')
           echo "Shard ${SHARD} of ${SHARDS}: ${#modules[@]} module(s)"
+          # Every module runs whatever an earlier one did, so one failure never hides another's; the shard fails afterwards, naming each module that failed.
+          failed=()
           for module in "${modules[@]}"; do
-            python3 -m unittest discover -s tests/unit -p "$(basename "$module")" -v
+            python3 -m unittest discover -s tests/unit -p "$(basename "$module")" -v || failed+=("$module")
           done
+          if [ "${#failed[@]}" -gt 0 ]; then
+            echo "::error::Failed module(s): ${failed[*]}"
+            exit 1
+          fi
 
   ansible-lint:
     name: ansible-lint
@@ -244,7 +250,7 @@ jobs:
           tarball=$(ls exadev-github_runner-*.tar.gz)
           tar -tzf "$tarball" | grep -Fx roles/github_runner_cluster/files/docker-compose.yml
           tar -tzf "$tarball" | grep -Fx roles/github_runner_cluster/files/k3s/node-entrypoint.sh
-          for file in Dockerfile fleet-regrade.sh fleet-regrade-core.sh fleet-regrade-docker.sh; do
+          for file in Dockerfile fleet-regrade.sh fleet-regrade-core.sh fleet-regrade-k3s.sh fleet-regrade-docker.sh fleet-regrade-systemd.sh; do
             tar -tzf "$tarball" | grep -Fx "roles/github_runner_cluster/files/regrade/${file}"
           done
       - name: Install the built tarball and verify its content

roles/github_runner_cluster/defaults/main.yml

@@ -302,6 +302,20 @@ github_runner_cluster_native_config_dir: /etc/rancher/k3s
 github_runner_cluster_native_unit_dir: /etc/systemd/system
 github_runner_cluster_native_data_dir: /var/lib/rancher/k3s
 github_runner_cluster_native_tailscale_cli: /usr/bin/tailscale
+# Not settings, variables only so the tests can point them at a stand-in: run_dir is where systemd makes each unit's runtime directory (/run), and psi_file is the kernel's memory pressure, which the role reads to refuse regrading on a kernel that has none.
+github_runner_cluster_native_run_dir: /run
+github_runner_cluster_native_psi_file: /proc/pressure/memory
+# Derived, never set directly: the runtime directory of each unit (RuntimeDirectory=, under run_dir), which systemd keeps across a restart of the unit and removes when the unit stops. One per unit, never shared: systemd removes a unit's runtime directory whenever that unit leaves the active state, which includes a start that the unit's ExecCondition skips, so a start of the agent unit on a server (by hand, or by anything that starts every unit it finds) would otherwise remove the running server's resolver file and regrading record.
+github_runner_cluster_native_agent_runtime_dir: github-runner-k3s-agent
+github_runner_cluster_native_server_runtime_dir: github-runner-k3s-server
+# Derived, never set directly: the regrading level fleet-regrade last requested for a native server, on k3s's data directory so that it outlives a restart and a reboot, and the level its k3s started at, in the unit's runtime directory, so that it exists only while k3s runs. The server unit's ExecStartPre writes the second from the first (files/native/write-regrade-level.sh); the systemd actuator writes the first and reads the second (files/regrade/fleet-regrade-systemd.sh).
+github_runner_cluster_native_regrade_level_file: "{{ github_runner_cluster_native_data_dir }}/regrade-level"
+github_runner_cluster_native_regrade_started_file: "{{ github_runner_cluster_native_run_dir }}/{{ github_runner_cluster_native_server_runtime_dir }}/regrade-level"
+# Derived, never set directly: where a native server's regrade scripts and settings go, and the name of the unit that runs them.
+github_runner_cluster_native_regrade_dir: "{{ github_runner_cluster_native_lib_dir }}/regrade"
+github_runner_cluster_native_regrade_service: fleet-regrade
+# Not a setting: the commands a native server's regrading runs that a minimal host may lack, each with the Debian package that installs it.
+github_runner_cluster_native_regrade_commands: {jq: jq, ps: procps}
 # Not settings: where Tailscale's apt signing key and repository list file go, which are the paths Tailscale's own install instructions use. A host that has either already is left alone. Variables only so the tests can point them at a stand-in.
 github_runner_cluster_native_apt_keyring: /usr/share/keyrings/tailscale-archive-keyring.gpg
 github_runner_cluster_native_apt_source: /etc/apt/sources.list.d/tailscale.list

roles/github_runner_cluster/files/native/write-regrade-level.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+# Puts a native server at the regrading level last requested for it, from the server unit's ExecStartPre (templates/native/k3s.service.j2), before every start: what the k3s container's entrypoint does for a Docker server (set_regrade_flags in files/k3s/node-entrypoint.sh). The level is the one the systemd actuator of fleet-regrade (files/regrade/fleet-regrade-systemd.sh) wrote to LEVEL_FILE; a server that has never been stepped down has none and starts at full.
+#
+# Only etcd-only changes how k3s starts: it turns off the API server, controller manager, scheduler and agent, so the node keeps its etcd member, and its vote, and little else. full and control-plane run the same k3s and differ in the cordon and taint the regrade service puts on the Node. The level reaches k3s as the server configuration's drop-in DROPIN_FILE, which k3s reads after server.yaml and the drop-ins named before it (dotDFiles and readConfigFile in pkg/configfilearg/parser.go of k3s), and which holds those four keys at etcd-only and none at any other level. The level k3s is about to start at is then recorded in STARTED_FILE, which the actuator reads as the level the node is at, since k3s hides its arguments from /proc. A level that is none of the three starts k3s whole and is recorded as it was read, which the actuator reports as unknown, so it never steps a node from a level it cannot name.
+#
+# Usage: write-regrade-level.sh LEVEL_FILE DROPIN_FILE STARTED_FILE. Each file is replaced in one rename, so neither k3s nor the actuator ever reads half of one.
+set -eu
+
+level_file=${1:?usage: write-regrade-level.sh LEVEL_FILE DROPIN_FILE STARTED_FILE}
+dropin=${2:?usage: write-regrade-level.sh LEVEL_FILE DROPIN_FILE STARTED_FILE}
+started=${3:?usage: write-regrade-level.sh LEVEL_FILE DROPIN_FILE STARTED_FILE}
+
+level=full
+if [ -r "$level_file" ]; then
+  level=$(head -n 1 "$level_file")
+fi
+
+umask 077
+mkdir -p "$(dirname "$dropin")" "$(dirname "$started")"
+{
+  printf '# Written by write-regrade-level.sh before every start of the server unit, from %s: the regrading level %s.\n' "$level_file" "$level"
+  if [ "$level" = etcd-only ]; then
+    printf 'disable-apiserver: true\ndisable-controller-manager: true\ndisable-scheduler: true\ndisable-agent: true\n'
+  fi
+} > "${dropin}.tmp"
+mv "${dropin}.tmp" "$dropin"
+printf '%s\n' "$level" > "${started}.tmp"
+mv "${started}.tmp" "$started"

roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh

@@ -111,12 +111,47 @@ next_level_up() {
   echo "${LEVELS[$((position - 1))]}"
 }
 
-# kubectl against the cluster, as an administrator. KUBECTL may be more than one word (`k3s kubectl`). KUBE_SERVER, when set, names the API server to use instead of the one in the kubeconfig: a node that has switched its own API server off still has to reach the cluster's.
+# kubectl against the cluster, as an administrator. KUBECTL may be more than one word (`k3s kubectl`). KUBE_SERVER, when set, names the API server to use instead of the one in the kubeconfig: select_api_server sets it from KUBE_SERVERS, since a node that has switched its own API server off still has to reach the cluster's.
 kubectl_() {
   # shellcheck disable=SC2086 # KUBECTL is a command that may take a subcommand, so it is split on purpose.
   $KUBECTL --kubeconfig "$KUBECONFIG_PATH" ${KUBE_SERVER:+--server "$KUBE_SERVER"} "$@"
 }
 
+# How long one candidate API server is given to answer /readyz, in seconds. A serving API server answers it from memory, so one that takes longer is not one to drive a step through; the bound is what a candidate that drops packets (a host off the mesh) costs before the next is asked, instead of the connection's own retry time.
+READYZ_TIMEOUT_SECONDS=5
+
+# Succeeds when the API server $1 (a URL, or empty for the kubeconfig's own) answers /readyz.
+api_ready() {
+  local KUBE_SERVER=$1
+  kubectl_ get --raw=/readyz --request-timeout="${READYZ_TIMEOUT_SECONDS}s" > /dev/null 2>&1
+}
+
+# Sets KUBE_SERVER to the first of KUBE_SERVERS whose /readyz answers, and fails, leaving it as it was, when none does. KUBE_SERVERS is a list of API server URLs separated by spaces, every other server and every host that may become one, by the names their certificates carry and never this node's own: the servers change at run time, so the first of the inventory may be an agent by now, and the next that serves is used. Unset or empty, the kubeconfig's own server is the only candidate. Called before every coordinated step and again whenever a step waits for the API, so each step goes through a server that is serving at the time.
+select_api_server() {
+  local candidate
+  if [ -z "${KUBE_SERVERS:-}" ]; then
+    api_ready '' || return 1
+    KUBE_SERVER=''
+    return 0
+  fi
+  for candidate in $KUBE_SERVERS; do
+    if api_ready "$candidate"; then
+      KUBE_SERVER=$candidate
+      return 0
+    fi
+  done
+  return 1
+}
+
+# The candidates select_api_server tries, as a refusal names them.
+api_candidates_text() {
+  if [ -n "${KUBE_SERVERS:-}" ]; then
+    echo "$KUBE_SERVERS"
+  else
+    echo "the kubeconfig's own server"
+  fi
+}
+
 # Taints, cordons and evicts everything but daemonsets, while the API server is still up. Both steps are idempotent, so each step down runs it again before anything is switched off. The taint is applied here and not left to the node's own configuration, which takes effect only when a node registers. --force evicts pods that no controller owns too: the node is shedding load, and a pod that blocks the drain would stop the step.
 shed_workloads() {
   kubectl_ taint nodes "$NODE_NAME" "$CONTROL_PLANE_TAINT" --overwrite
@@ -130,8 +165,8 @@ restore_workloads() {
 
 wait_for_api() {
   local deadline=$((SECONDS + OPERATION_TIMEOUT_SECONDS))
-  until kubectl_ get --raw=/readyz > /dev/null 2>&1; do
-    [ "$SECONDS" -lt "$deadline" ] || die "The API server did not become ready within ${OPERATION_TIMEOUT_SECONDS}s."
+  until select_api_server; do
+    [ "$SECONDS" -lt "$deadline" ] || die "No API server became ready within ${OPERATION_TIMEOUT_SECONDS}s (tried: $(api_candidates_text))."
     sleep 2
   done
 }
@@ -231,13 +266,17 @@ peers_ready() {
   fi
 }
 
-# Runs a command only while this node may transition: its peers are Ready and it holds the Lease. The Lease is released afterwards whether or not the command succeeded, and the command's status is the result.
+# Runs a command only while this node may transition: an API server answers, its peers are Ready and it holds the Lease. The Lease is released afterwards whether or not the command succeeded, through an API server that answers then (the one the step started with may have gone away meanwhile), and the command's status is the result.
 coordinated() {
   local status=0
+  if ! select_api_server; then
+    log "cannot step: no API server answers (tried: $(api_candidates_text))"
+    return 1
+  fi
   peers_ready || return 1
   acquire_lease || return 1
   "$@" || status=$?
-  release_lease || log "the lease could not be released; it expires on its own"
+  { select_api_server && release_lease; } || log "the lease could not be released; it expires on its own"
   return "$status"
 }