diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a9eb2ee..44b00057 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -202,9 +202,15 @@ jobs: python3 -c 'import ansible.release; print("ansible-core", ansible.release.__version__)' mapfile -t modules < <(find tests/unit -maxdepth 1 -name 'test_*.py' -printf '%s %p\n' | LC_ALL=C sort -k1,1nr -k2,2 | cut -d' ' -f2- | awk -v shard="$SHARD" -v shards="$SHARDS" '(NR - 1) % shards == shard') echo "Shard ${SHARD} of ${SHARDS}: ${#modules[@]} module(s)" + # Every module runs whatever an earlier one did, so one failure never hides another's; the shard fails afterwards, naming each module that failed. + failed=() for module in "${modules[@]}"; do - python3 -m unittest discover -s tests/unit -p "$(basename "$module")" -v + python3 -m unittest discover -s tests/unit -p "$(basename "$module")" -v || failed+=("$module") done + if [ "${#failed[@]}" -gt 0 ]; then + echo "::error::Failed module(s): ${failed[*]}" + exit 1 + fi ansible-lint: name: ansible-lint @@ -244,7 +250,7 @@ jobs: tarball=$(ls exadev-github_runner-*.tar.gz) tar -tzf "$tarball" | grep -Fx roles/github_runner_cluster/files/docker-compose.yml tar -tzf "$tarball" | grep -Fx roles/github_runner_cluster/files/k3s/node-entrypoint.sh - for file in Dockerfile fleet-regrade.sh fleet-regrade-core.sh fleet-regrade-docker.sh; do + for file in Dockerfile fleet-regrade.sh fleet-regrade-core.sh fleet-regrade-k3s.sh fleet-regrade-docker.sh fleet-regrade-systemd.sh; do tar -tzf "$tarball" | grep -Fx "roles/github_runner_cluster/files/regrade/${file}" done - name: Install the built tarball and verify its content diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index f15ca8a8..6ae2a0ea 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -131,7 +131,6 @@ The container nodes point the kubelet at a resolver file naming only Tailscale's - **The image pin.** The k3s binary is pinned in its place. A Docker node ignores a changed pin until its image is rebuilt by hand; a native node's binary is replaced by the next run and the service restarted, after the version check has allowed it. Upgrade as for any agent: servers first, one minor at a time (see [Versions](#versions)), and cordon and drain the node first if its runner pods must not see the restart (they survive it, but the kubelet restarts). - **The node password volume.** k3s keeps the node's password in `/etc/rancher/node/password` on the host, which the role never reads, writes or deletes, so it survives restarts, upgrades and reruns without anything like the seeding in [Node passwords and recreating servers](#node-passwords-and-recreating-servers). It is lost only if that file is removed or the host is reinstalled, and then the node is "a genuinely new node under an old name" ([A replaced node](#a-replaced-node)): `github_runner_cluster_wait_for_nodes` on a server host clears the stale Secret and Node object, or delete them by hand. - **Removing a node.** Stopping a unit leaves the pods' containerd shims running. The role installs neither k3s's `k3s-killall.sh` nor its uninstall script (the installer writes them; see [Uninstalling](https://docs.k3s.io/installation/uninstall) ([archived](http://web.archive.org/web/2026/https://docs.k3s.io/installation/uninstall))). Drain the node, delete its Node object, then `systemctl disable --now k3s-agent k3s` and stop the shims. A server's etcd member is removed first, as for a demotion. -- **Memory-pressure regrading.** Its actuator drives a Docker server's container, so the role refuses `github_runner_cluster_regrade` on a native host. - **Tailnet names in pods** (above), and **a Docker-free host's `docker` tooling**: nothing in this mode runs `docker` on the host, so it need not be installed. ### A Wi-Fi-only host @@ -140,7 +139,7 @@ A node on Wi-Fi drops off the network whenever the radio does, and it then flaps ### Tests -`tests/unit/test_native_node.py` and `tests/unit/test_native_server.py` cover it without a real host: the filters in `plugins/filter/cluster_native.py`, the helper scripts, the defaults, argument specs and this README, and the role's tasks (`tests/native/run.yml`) against stand-ins (`tests/unit/native_stand_ins.py`) for `systemctl`, `tailscale`, `dpkg-query`, `apt-get`, `docker` and `kubectl` on `PATH`, a local server for the k3s release, and a server data directory laid out as k3s leaves it. For servers it checks a bootstrap with `cluster-init` and the Docker servers' flags, a join, the voter guard refusing a join and a restart (and the refused restart being made by the next run), a promotion in place, an abandoned promotion whose runner pods did not finish, a demotion refused while the member is a voter and made once it is removed, with the datastore moved aside and the identity kept, and a former member never starting a second cluster. `tests/mesh/run.sh native_server` runs a real native server as a privileged systemd container beside the Docker servers: it joins etcd as a learner, is promoted to a voter, and keeps the cluster writable throughout. It checks a first run, idempotence, `--check`, a changed setting restarting the service once, a client in another tailnet (refused, then logged out first when allowed), a client k3s did not log in, a refused downgrade, a wrong sum, an unsupported host, the secrets staying out of the output, and the node's readiness as a server reports it. +`tests/unit/test_native_node.py` and `tests/unit/test_native_server.py` cover it without a real host: the filters in `plugins/filter/cluster_native.py`, the helper scripts, the defaults, argument specs and this README, and the role's tasks (`tests/native/run.yml`) against stand-ins (`tests/unit/native_stand_ins.py`) for `systemctl`, `tailscale`, `dpkg-query`, `apt-get`, `docker` and `kubectl` on `PATH`, a local server for the k3s release, and a server data directory laid out as k3s leaves it. For servers it checks a bootstrap with `cluster-init` and the Docker servers' flags, a join, the voter guard refusing a join and a restart (and the refused restart being made by the next run), a promotion in place, an abandoned promotion whose runner pods did not finish, a demotion refused while the member is a voter and made once it is removed, with the datastore moved aside and the identity kept, and a former member never starting a second cluster. `tests/mesh/run.sh native_server` runs a real native server as a privileged systemd container beside the Docker servers: it joins etcd as a learner, is promoted to a voter, and keeps the cluster writable throughout; then, with regrading turned on, its `fleet-regrade` unit steps it down to etcd-only and back up under simulated memory pressure, with its membership unchanged, its vote kept (a write succeeds with a Docker server stopped), the level kept across `systemctl restart k3s` and across the host stopping and starting, and no pod's shim left running at etcd-only. It checks a first run, idempotence, `--check`, a changed setting restarting the service once, a client in another tailnet (refused, then logged out first when allowed), a client k3s did not log in, a refused downgrade, a wrong sum, an unsupported host, the secrets staying out of the output, and the node's readiness as a server reports it. ## Docker host disk @@ -276,7 +275,7 @@ A `regrade` service runs beside the server's k3s container (in its own Compose p With `github_runner_cluster_regrade_step_up: automatic` (the default) the service also steps back up by itself, one level at a time. A server counts as calm when the pressure is under the threshold and `MemAvailable` is at or above the floor, both, which is stricter than not being over the limit: a reading between the two ends the streak towards stepping down and the streak towards stepping up alike, so the server does not step both ways on the same noise. It steps up after being calm for `github_runner_cluster_regrade_sustain_seconds`, once the cooldown since the last step, in either direction, has passed, through the same Lease and peer check as a step down; stepping up restarts the server's etcd at the first level that leaves etcd-only, so it never starts while a peer is down. A server that cannot hold the higher level cycles down and up no faster than once per cooldown and sustain period, which is the sign it is too small for the workloads it is given. `manual` leaves stepping up to `fleet-regrade up`. -Steps are serialised across the whole cluster. A server starts a step only while it holds a Lease named `fleet-regrade` in `kube-system` and every other etcd-voting Node is Ready, so two servers never step at once, and a step never starts while a peer is down. A server at etcd-only is not Ready (it has no kubelet), which also means no other server steps down while one is etcd-only. The Lease expires on its own after `github_runner_cluster_regrade_lease_duration_seconds` if its holder never gives it back. Every call the service makes to the cluster goes to another server's API and never to the stepping server's own: at etcd-only the server has no API, and its own kubeconfig answers `apiserver disabled` on some servers. +Steps are serialised across the whole cluster. A server starts a step only while it holds a Lease named `fleet-regrade` in `kube-system` and every other etcd-voting Node is Ready, so two servers never step at once, and a step never starts while a peer is down. A server at etcd-only is not Ready (it has no kubelet), which also means no other server steps down while one is etcd-only. The Lease expires on its own after `github_runner_cluster_regrade_lease_duration_seconds` if its holder never gives it back. Every call the service makes to the cluster goes to another server's API and never to the stepping server's own: at etcd-only the server has no API, and its own kubeconfig answers `apiserver disabled` on some servers. The service is given every other server of the group and then every host marked `github_runner_cluster_voter_eligible`, by the addresses the servers' certificates carry, and uses the first whose `/readyz` answers, chosen again before each step, for the Lease's release and whenever a step waits for the API, since which hosts are servers can change at run time. A level is applied by the Docker actuator, which records it in a file on the k3s data volume, which the container's entrypoint reads at every start, and restarts the container when the change is between etcd-only and another level. Between `full` and `control-plane` nothing restarts, because both run the same k3s; the difference is the taint and cordon the service puts on the Node. The container is restarted, never created again, so the node's password and mesh identity are the ones the cluster already holds, and an Ansible run, which changes nothing a container is created from, cannot undo a step. A level survives the container being recreated and the Docker host restarting: a server stepped down stays down, with its cordon, until it is stepped up, whether by hand or, in `automatic`, by the service once memory has been calm. That includes a `docker restart` by hand and an Ansible run that recreates the container (a changed image or Compose setting): the server comes back at the level it was left at, and `fleet-regrade up` is the way back to `full`. The entrypoint also records, on the container's tmpfs, the level k3s started at; that is what the service reads as the level the node is at, since k3s hides its arguments from `/proc`. A server whose entrypoint predates this feature is at `full` until it next restarts. @@ -294,11 +293,19 @@ docker compose --profile regrade exec regrade fleet-regrade up # one step up Both go through the same Lease and peer check as the watcher. -The role refuses regrading where it could not work, before it changes anything: on an agent; where the server is the only one in the group, since a server at etcd-only needs another server's API; and on the bootstrap server of a `headscale_in_cluster` mesh, whose Headscale server is a static pod on the kubelet that etcd-only switches off. It also refuses to turn regrading off, or to remove the service, while the server is below `full`, because nothing else would step it up: step it up first, then run the role again. +A native server (`github_runner_cluster_node_mode: native`, see [Native servers](#native-servers)) regrades the same way, through the systemd actuator (`files/regrade/fleet-regrade-systemd.sh`) instead of the Docker one. The role runs it as the systemd unit `fleet-regrade` on the host, as root, with the host's own `/proc/pressure/memory` and `/proc/meminfo`, k3s's own `k3s kubectl` and the admin kubeconfig k3s wrote while it ran whole, from scripts and settings under `/usr/local/lib/github-runner-k3s/regrade`; it installs `jq` and `procps` from the distribution's packages when the host lacks them (Debian and Ubuntu; elsewhere it refuses until they are installed). A level is recorded in `/var/lib/rancher/k3s/regrade-level`, which the `k3s` unit's `ExecStartPre` (`files/native/write-regrade-level.sh`) reads at every start: it writes the drop-in `/etc/rancher/k3s/server.yaml.d/50-regrade.yaml`, which turns off the API server, controller manager, scheduler and agent at `etcd-only` and sets nothing at any other level, and records the level k3s starts at in `/run/github-runner-k3s-server/regrade-level`, the unit's runtime directory, which the actuator reads. The `ExecStartPre` runs whether or not regrading is on, so turning it on or off changes nothing in the server's unit and restarts nothing. A step into or out of `etcd-only` restarts the `k3s` unit and returns once etcd serves again (k3s's `ETCD server is now running`, read from the journal of the unit's new run by its invocation id) and, for a level that runs the agent, once the Node is Ready on a heartbeat later than the restart. The unit's `KillMode=process` leaves the pods' containerd shims running when k3s restarts without its agent, so entering `etcd-only` stops the shims that are left, with the containers they run, as k3s's `k3s-killall.sh` finds and stops them; leaving it starts the agent, whose kubelet starts the pods again. The level outlasts a restart of the unit and a reboot, as a Docker server's outlasts its container, and a demotion to an agent forgets it, so a later promotion starts the server whole. A server turn of the role that restarts a server at `etcd-only` waits for its etcd, not its Node, which has no kubelet. By hand, on the host: + +```sh +sudo fleet-regrade status +sudo fleet-regrade down # one step down +sudo fleet-regrade up # one step up +``` + +The role refuses regrading where it could not work, before it changes anything: on an agent (for a native host, the role it is to run after this run, which for a host marked `github_runner_cluster_voter_eligible` is the one recorded on it); where the server is the only one in the group, since a server at etcd-only needs another server's API; on the bootstrap server of a `headscale_in_cluster` mesh, whose Headscale server is a static pod on the kubelet that etcd-only switches off; and on a native host whose kernel exposes no memory pressure (built without `CONFIG_PSI` or booted with `psi=0`), which the service could not decide on. It also refuses to turn regrading off, or to remove the service or the `fleet-regrade` unit, while the server is below `full`, because nothing else would step it up: step it up first, then run the role again. The service mounts the Docker socket, which is a permission cost: whatever runs in that container can start containers on the host with any mount. `docker-prune` has the same mount for the same reason, and the regrade service's scripts are mounted read-only from the project directory, not fetched at run time. Its Compose profile is separate from `server` and `agent`, so a host that does not enable regrading never builds or starts it. The image is built locally by Compose from `regrade/Dockerfile` (Alpine with bash, jq and the Docker CLI), the first time the service starts. -The decision logic lives in `files/regrade/fleet-regrade-core.sh` and calls only three functions of an actuator: `levels`, `current_level` and `apply_level NAME`. The Docker actuator is `files/regrade/fleet-regrade-docker.sh`. Another platform supplies its own three functions and uses the core unchanged; a NixOS image can take the core from a pinned release of this collection. +The decision logic lives in `files/regrade/fleet-regrade-core.sh` and calls only three functions of an actuator: `levels`, `current_level` and `apply_level NAME`. The Docker actuator is `files/regrade/fleet-regrade-docker.sh`, over `files/regrade/fleet-regrade-k3s.sh`, which holds what any actuator of a k3s server needs (the levels, their records, the Node label and the restart into etcd-only and out of it) and leaves the platform only where k3s runs, how it restarts and how its log is read; `files/regrade/fleet-regrade.sh` runs the core with the actuator the settings name. Another platform supplies its own three functions and uses the core unchanged; a NixOS image can take the core from a pinned release of this collection. ## Role controller @@ -409,7 +416,7 @@ Each Compose project runs its own `docker-prune` service against the same daemon - `github_runner_cluster_node_taints`: Kubernetes taints (`key=value:Effect`, no spaces) the host's node registers with, empty by default. `NoSchedule` keeps pods without a matching toleration off it and `NoExecute` also evicts the ones already there. k3s applies them on every registration, so they come back if the node object is deleted, which a taint set with `kubectl` does not. - `github_runner_cluster_etcd_heartbeat_interval`, `github_runner_cluster_etcd_election_timeout`: etcd's timing in milliseconds for every server, empty by default (k3s's own). Set together, the same on every server, the election timeout at least five times the heartbeat interval; see [etcd timing](#etcd-timing). - `github_runner_cluster_role_controller` (default false), `github_runner_cluster_voters` (default 3), `github_runner_cluster_voter_eligible` and `github_runner_cluster_voter_pinned` (per host, default false): the role controller's inventory (see [Role controller](#role-controller)). `github_runner_cluster_voter_eligible_label` and `github_runner_cluster_voter_pinned_label` are the Node labels they are published under, derived, never set directly. -- `github_runner_cluster_regrade` (default false): run the `regrade` service, which steps this server down under memory pressure while it keeps its etcd vote (see [Memory-pressure regrading](#memory-pressure-regrading)). Servers only. `github_runner_cluster_regrade_pressure_threshold` (percent, default 10) and `github_runner_cluster_regrade_available_floor_percent` (default 15) are the limit; `github_runner_cluster_regrade_step_up` (`automatic`, the default, or `manual`), `github_runner_cluster_regrade_sample_seconds`, `github_runner_cluster_regrade_sustain_seconds`, `github_runner_cluster_regrade_cooldown_seconds`, `github_runner_cluster_regrade_operation_timeout_seconds` and `github_runner_cluster_regrade_lease_duration_seconds` are the timing, all derived from `github_runner_cluster_regrade_pressure_window_seconds` (the kernel's `avg300`, 300 seconds, not a setting) unless set. `github_runner_cluster_regrade_level_label` and `github_runner_cluster_regrade_taint` are derived, never set directly. +- `github_runner_cluster_regrade` (default false): run the `regrade` service, or on a native server the `fleet-regrade` unit, which steps this server down under memory pressure while it keeps its etcd vote (see [Memory-pressure regrading](#memory-pressure-regrading)). Servers only. `github_runner_cluster_regrade_pressure_threshold` (percent, default 10) and `github_runner_cluster_regrade_available_floor_percent` (default 15) are the limit; `github_runner_cluster_regrade_step_up` (`automatic`, the default, or `manual`), `github_runner_cluster_regrade_sample_seconds`, `github_runner_cluster_regrade_sustain_seconds`, `github_runner_cluster_regrade_cooldown_seconds`, `github_runner_cluster_regrade_operation_timeout_seconds` and `github_runner_cluster_regrade_lease_duration_seconds` are the timing, all derived from `github_runner_cluster_regrade_pressure_window_seconds` (the kernel's `avg300`, 300 seconds, not a setting) unless set. `github_runner_cluster_regrade_level_label` and `github_runner_cluster_regrade_taint` are derived, never set directly. - Per-host overrides: `github_runner_cluster_node_role` (`server` or `agent`), `github_runner_cluster_bootstrap` (true picks the bootstrap host, false excludes a host), `github_runner_cluster_server_url` (join this server instead, for a cluster whose servers are outside the inventory), `github_runner_cluster_tls_sans`, `github_runner_cluster_node_address`, and `github_runner_cluster_voter_eligible` (true stages a native host as a standby server, its server configuration written and its name on every server's certificate, without making it a server; see [Native servers](#native-servers)). The run fails if the result has an even number of servers, no servers, or more than one bootstrap host. - `github_runner_cluster_docker_context` (default empty): pins the Docker context this host's node runs in (see [Docker contexts](#docker-contexts)). - `github_runner_cluster_network_mode` (default `bridge`, or `host`): the network the node's container runs on, and with `host`, `github_runner_cluster_lan_cidr` (required: the LAN the nodes share) and `github_runner_cluster_network_probe_image` (default `busybox:1.37`, for reading the host's addresses) (see [Networking](#networking)). @@ -429,7 +436,7 @@ Each Compose project runs its own `docker-prune` service against the same daemon - `github_runner_cluster_native_tailscale_install` (default `true`): whether the role installs the pinned Tailscale package from Tailscale's apt repository (Debian and Ubuntu). `false` leaves the host's client as it is and only checks its login. - `github_runner_cluster_native_tailscale_repository_url` (default `https://pkgs.tailscale.com/stable`): where that repository's signing key and list file are fetched from, when the host has none. - `github_runner_cluster_native_tailscale_leave_other_tailnet` (default `false`): whether the role logs a client out of another tailnet so that k3s can join the cluster's; `false` stops the run instead. -- `github_runner_cluster_native_agent_service`, `github_runner_cluster_native_server_service`, `github_runner_cluster_native_service`, `github_runner_cluster_native_join_url`, `github_runner_cluster_native_join_host`, `github_runner_cluster_native_role_file` and `github_runner_cluster_native_restart_marker` are derived: the units' names (`k3s-agent` and `k3s`), the unit of the host's role, the URL the node joins and the host in it, which the Tailscale wrapper waits for, the role file both units' conditions read, and the prefix of the per-unit file that says a restart is waiting. `github_runner_cluster_native_bin_dir`, `github_runner_cluster_native_lib_dir`, `github_runner_cluster_native_config_dir`, `github_runner_cluster_native_unit_dir`, `github_runner_cluster_native_data_dir`, `github_runner_cluster_native_tailscale_cli`, `github_runner_cluster_native_apt_keyring` and `github_runner_cluster_native_apt_source` are where the native node's files live, which are not settings, and are variables only so the tests can point them at a stand-in. +- `github_runner_cluster_native_agent_service`, `github_runner_cluster_native_server_service`, `github_runner_cluster_native_service`, `github_runner_cluster_native_join_url`, `github_runner_cluster_native_join_host`, `github_runner_cluster_native_role_file` and `github_runner_cluster_native_restart_marker` are derived: the units' names (`k3s-agent` and `k3s`), the unit of the host's role, the URL the node joins and the host in it, which the Tailscale wrapper waits for, the role file both units' conditions read, and the prefix of the per-unit file that says a restart is waiting; `github_runner_cluster_native_regrade_level_file`, `github_runner_cluster_native_regrade_started_file`, `github_runner_cluster_native_regrade_dir`, `github_runner_cluster_native_regrade_service`, `github_runner_cluster_native_regrade_commands`, `github_runner_cluster_native_agent_runtime_dir` and `github_runner_cluster_native_server_runtime_dir` are derived too: a native server's requested and started regrading levels, where its regrade scripts and settings go, the unit that runs them, the commands regrading needs with the package of each, and each unit's own runtime directory under `/run` (one per unit, since systemd removes a unit's runtime directory when its condition skips a start, which on a shared one would take the running unit's resolver file). `github_runner_cluster_native_bin_dir`, `github_runner_cluster_native_lib_dir`, `github_runner_cluster_native_config_dir`, `github_runner_cluster_native_unit_dir`, `github_runner_cluster_native_data_dir`, `github_runner_cluster_native_tailscale_cli`, `github_runner_cluster_native_apt_keyring`, `github_runner_cluster_native_apt_source`, `github_runner_cluster_native_run_dir` and `github_runner_cluster_native_psi_file` are where the native node's files live, which are not settings, and are variables only so the tests can point them at a stand-in. ## Examples @@ -471,7 +478,7 @@ or read from 1Password by adding `exadev.github_runner.github_runner_secrets_one `tests/mesh/run.sh` brings up three k3s nodes in Docker on one machine and joins them through Headscale with this role, for the `headscale_hosted` and `headscale_in_cluster` providers, and checks the `headscale_existing` policy check against a real server. Each cluster scenario runs the role a second time and fails if any node container restarted. Both set the etcd timing and check each server's etcd config file carries it, and the `hosted` scenario then restarts a joined server against its initialised datastore and checks the cluster recovers with the timing still in place. The `compose_faulty` scenario checks the role refuses a Compose release that has that fault. With `GRTEST_PIN_CONTEXT=1` the cluster scenarios pin `github_runner_cluster_docker_context` while the current Docker context the playbooks see points at a socket that does not exist, so any docker call that ignored the pin fails, and they check the current context is unchanged afterwards; `tests/litestream/run.sh` takes the same setting for the Headscale hosts it delegates to and the promotion playbook. The `context_checks` scenario checks the role refuses a pinned context that does not exist, and one set alongside `DOCKER_HOST`, before writing anything. The `host_network` scenario (Linux only) gives each node a Docker daemon of its own at a fixed address on the test network, standing in for a VM with an address on the LAN, runs the role with `github_runner_cluster_network_mode: host`, and checks that every node's Tailscale client reaches every other directly at that address and that no port is published. `tests/unit/test_cluster_network.py` checks the rendered Compose project in both modes and the role's list of Compose files; `tests/unit/test_cluster_guard.py` checks the address and quorum filters, and `tests/unit/test_host_network_role.py` runs the role's `tasks/host_network.yml` against stand-in `docker` and `kubectl` commands. `.github/workflows/mesh-integration.yml` runs them on pull requests that touch the role, the hosted scenario on 2.39.0, the first fixed release, as well as on a current one. -`tests/regrade` tests the regrading: the core's decision, stepping up by itself, and the Lease and peer coordination (`regrade-core.sh`, `regrade-recover.sh`, `regrade-lease.sh`) and the Docker actuator (`regrade-docker.sh`), all against stand-ins for `docker` and `kubectl`, run by `tests/unit/test_regrade_shell.py`; `tests/unit/test_cluster_regrade.py` covers the role's checks, the settings file, the Compose service, the levels the entrypoint starts k3s at and the tasks that remove the service. +`tests/regrade` tests the regrading: the core's decision, stepping up by itself, and the Lease and peer coordination (`regrade-core.sh`, `regrade-recover.sh`, `regrade-lease.sh`) the Docker actuator (`regrade-docker.sh`), against stand-ins for `docker` and `kubectl`, and the systemd actuator with the server unit's `ExecStartPre` (`regrade-systemd.sh`), against stand-ins for `systemctl`, `journalctl`, `date`, `k3s` and `kubectl` and with real processes standing in for the pods' shims, all run by `tests/unit/test_regrade_shell.py`; `tests/unit/test_native_regrade.py` covers the role's regrading of a native server (its refusals, the unit and its settings, a server turn at etcd-only and turning it off); `tests/unit/test_cluster_regrade.py` covers the role's checks, the settings file, the Compose service, the levels the entrypoint starts k3s at and the tasks that remove the service. `tests/unit/test_runtime_autostart.py` covers the [container runtime tasks](#container-runtime-after-a-reboot-macos) without a Mac: it renders the launchd agent and daemon through `tests/runtime_autostart/render_plist.yml`, and runs `tasks/runtime.yml` and `tasks/end_checks.yml` through the role against stand-ins for `docker`, `colima`, `launchctl`, `plutil` and `sudo` on `PATH`, the stand-in `launchctl` refusing to change the system domain unless it was run through the stand-in `sudo`. It checks the variable defaults, that Linux hosts are left alone, that a second run changes nothing, that `--check` writes and starts nothing, and that each way the runtime or the node's container can be down fails with its message. `tests/unit/test_container_runtime.py` covers the filter that works out the runtime from the Docker context. Nothing there touches a real Colima, launchd or Docker Desktop, so loading the agent into a real login session, and the daemon starting a Colima VM at a real boot with no one logged in, are the parts only a run on a Mac can show. diff --git a/roles/github_runner_cluster/defaults/main.yml b/roles/github_runner_cluster/defaults/main.yml index e058b350..7e1d523a 100644 --- a/roles/github_runner_cluster/defaults/main.yml +++ b/roles/github_runner_cluster/defaults/main.yml @@ -302,6 +302,20 @@ github_runner_cluster_native_config_dir: /etc/rancher/k3s github_runner_cluster_native_unit_dir: /etc/systemd/system github_runner_cluster_native_data_dir: /var/lib/rancher/k3s github_runner_cluster_native_tailscale_cli: /usr/bin/tailscale +# Not settings, variables only so the tests can point them at a stand-in: run_dir is where systemd makes each unit's runtime directory (/run), and psi_file is the kernel's memory pressure, which the role reads to refuse regrading on a kernel that has none. +github_runner_cluster_native_run_dir: /run +github_runner_cluster_native_psi_file: /proc/pressure/memory +# Derived, never set directly: the runtime directory of each unit (RuntimeDirectory=, under run_dir), which systemd keeps across a restart of the unit and removes when the unit stops. One per unit, never shared: systemd removes a unit's runtime directory whenever that unit leaves the active state, which includes a start that the unit's ExecCondition skips, so a start of the agent unit on a server (by hand, or by anything that starts every unit it finds) would otherwise remove the running server's resolver file and regrading record. +github_runner_cluster_native_agent_runtime_dir: github-runner-k3s-agent +github_runner_cluster_native_server_runtime_dir: github-runner-k3s-server +# Derived, never set directly: the regrading level fleet-regrade last requested for a native server, on k3s's data directory so that it outlives a restart and a reboot, and the level its k3s started at, in the unit's runtime directory, so that it exists only while k3s runs. The server unit's ExecStartPre writes the second from the first (files/native/write-regrade-level.sh); the systemd actuator writes the first and reads the second (files/regrade/fleet-regrade-systemd.sh). +github_runner_cluster_native_regrade_level_file: "{{ github_runner_cluster_native_data_dir }}/regrade-level" +github_runner_cluster_native_regrade_started_file: "{{ github_runner_cluster_native_run_dir }}/{{ github_runner_cluster_native_server_runtime_dir }}/regrade-level" +# Derived, never set directly: where a native server's regrade scripts and settings go, and the name of the unit that runs them. +github_runner_cluster_native_regrade_dir: "{{ github_runner_cluster_native_lib_dir }}/regrade" +github_runner_cluster_native_regrade_service: fleet-regrade +# Not a setting: the commands a native server's regrading runs that a minimal host may lack, each with the Debian package that installs it. +github_runner_cluster_native_regrade_commands: {jq: jq, ps: procps} # Not settings: where Tailscale's apt signing key and repository list file go, which are the paths Tailscale's own install instructions use. A host that has either already is left alone. Variables only so the tests can point them at a stand-in. github_runner_cluster_native_apt_keyring: /usr/share/keyrings/tailscale-archive-keyring.gpg github_runner_cluster_native_apt_source: /etc/apt/sources.list.d/tailscale.list diff --git a/roles/github_runner_cluster/files/native/write-regrade-level.sh b/roles/github_runner_cluster/files/native/write-regrade-level.sh new file mode 100755 index 00000000..42e7dfb7 --- /dev/null +++ b/roles/github_runner_cluster/files/native/write-regrade-level.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# Puts a native server at the regrading level last requested for it, from the server unit's ExecStartPre (templates/native/k3s.service.j2), before every start: what the k3s container's entrypoint does for a Docker server (set_regrade_flags in files/k3s/node-entrypoint.sh). The level is the one the systemd actuator of fleet-regrade (files/regrade/fleet-regrade-systemd.sh) wrote to LEVEL_FILE; a server that has never been stepped down has none and starts at full. +# +# Only etcd-only changes how k3s starts: it turns off the API server, controller manager, scheduler and agent, so the node keeps its etcd member, and its vote, and little else. full and control-plane run the same k3s and differ in the cordon and taint the regrade service puts on the Node. The level reaches k3s as the server configuration's drop-in DROPIN_FILE, which k3s reads after server.yaml and the drop-ins named before it (dotDFiles and readConfigFile in pkg/configfilearg/parser.go of k3s), and which holds those four keys at etcd-only and none at any other level. The level k3s is about to start at is then recorded in STARTED_FILE, which the actuator reads as the level the node is at, since k3s hides its arguments from /proc. A level that is none of the three starts k3s whole and is recorded as it was read, which the actuator reports as unknown, so it never steps a node from a level it cannot name. +# +# Usage: write-regrade-level.sh LEVEL_FILE DROPIN_FILE STARTED_FILE. Each file is replaced in one rename, so neither k3s nor the actuator ever reads half of one. +set -eu + +level_file=${1:?usage: write-regrade-level.sh LEVEL_FILE DROPIN_FILE STARTED_FILE} +dropin=${2:?usage: write-regrade-level.sh LEVEL_FILE DROPIN_FILE STARTED_FILE} +started=${3:?usage: write-regrade-level.sh LEVEL_FILE DROPIN_FILE STARTED_FILE} + +level=full +if [ -r "$level_file" ]; then + level=$(head -n 1 "$level_file") +fi + +umask 077 +mkdir -p "$(dirname "$dropin")" "$(dirname "$started")" +{ + printf '# Written by write-regrade-level.sh before every start of the server unit, from %s: the regrading level %s.\n' "$level_file" "$level" + if [ "$level" = etcd-only ]; then + printf 'disable-apiserver: true\ndisable-controller-manager: true\ndisable-scheduler: true\ndisable-agent: true\n' + fi +} > "${dropin}.tmp" +mv "${dropin}.tmp" "$dropin" +printf '%s\n' "$level" > "${started}.tmp" +mv "${started}.tmp" "$started" diff --git a/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh b/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh index 33b5752b..dea370ec 100644 --- a/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh +++ b/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh @@ -111,12 +111,47 @@ next_level_up() { echo "${LEVELS[$((position - 1))]}" } -# kubectl against the cluster, as an administrator. KUBECTL may be more than one word (`k3s kubectl`). KUBE_SERVER, when set, names the API server to use instead of the one in the kubeconfig: a node that has switched its own API server off still has to reach the cluster's. +# kubectl against the cluster, as an administrator. KUBECTL may be more than one word (`k3s kubectl`). KUBE_SERVER, when set, names the API server to use instead of the one in the kubeconfig: select_api_server sets it from KUBE_SERVERS, since a node that has switched its own API server off still has to reach the cluster's. kubectl_() { # shellcheck disable=SC2086 # KUBECTL is a command that may take a subcommand, so it is split on purpose. $KUBECTL --kubeconfig "$KUBECONFIG_PATH" ${KUBE_SERVER:+--server "$KUBE_SERVER"} "$@" } +# How long one candidate API server is given to answer /readyz, in seconds. A serving API server answers it from memory, so one that takes longer is not one to drive a step through; the bound is what a candidate that drops packets (a host off the mesh) costs before the next is asked, instead of the connection's own retry time. +READYZ_TIMEOUT_SECONDS=5 + +# Succeeds when the API server $1 (a URL, or empty for the kubeconfig's own) answers /readyz. +api_ready() { + local KUBE_SERVER=$1 + kubectl_ get --raw=/readyz --request-timeout="${READYZ_TIMEOUT_SECONDS}s" > /dev/null 2>&1 +} + +# Sets KUBE_SERVER to the first of KUBE_SERVERS whose /readyz answers, and fails, leaving it as it was, when none does. KUBE_SERVERS is a list of API server URLs separated by spaces, every other server and every host that may become one, by the names their certificates carry and never this node's own: the servers change at run time, so the first of the inventory may be an agent by now, and the next that serves is used. Unset or empty, the kubeconfig's own server is the only candidate. Called before every coordinated step and again whenever a step waits for the API, so each step goes through a server that is serving at the time. +select_api_server() { + local candidate + if [ -z "${KUBE_SERVERS:-}" ]; then + api_ready '' || return 1 + KUBE_SERVER='' + return 0 + fi + for candidate in $KUBE_SERVERS; do + if api_ready "$candidate"; then + KUBE_SERVER=$candidate + return 0 + fi + done + return 1 +} + +# The candidates select_api_server tries, as a refusal names them. +api_candidates_text() { + if [ -n "${KUBE_SERVERS:-}" ]; then + echo "$KUBE_SERVERS" + else + echo "the kubeconfig's own server" + fi +} + # Taints, cordons and evicts everything but daemonsets, while the API server is still up. Both steps are idempotent, so each step down runs it again before anything is switched off. The taint is applied here and not left to the node's own configuration, which takes effect only when a node registers. --force evicts pods that no controller owns too: the node is shedding load, and a pod that blocks the drain would stop the step. shed_workloads() { kubectl_ taint nodes "$NODE_NAME" "$CONTROL_PLANE_TAINT" --overwrite @@ -130,8 +165,8 @@ restore_workloads() { wait_for_api() { local deadline=$((SECONDS + OPERATION_TIMEOUT_SECONDS)) - until kubectl_ get --raw=/readyz > /dev/null 2>&1; do - [ "$SECONDS" -lt "$deadline" ] || die "The API server did not become ready within ${OPERATION_TIMEOUT_SECONDS}s." + until select_api_server; do + [ "$SECONDS" -lt "$deadline" ] || die "No API server became ready within ${OPERATION_TIMEOUT_SECONDS}s (tried: $(api_candidates_text))." sleep 2 done } @@ -231,13 +266,17 @@ peers_ready() { fi } -# Runs a command only while this node may transition: its peers are Ready and it holds the Lease. The Lease is released afterwards whether or not the command succeeded, and the command's status is the result. +# Runs a command only while this node may transition: an API server answers, its peers are Ready and it holds the Lease. The Lease is released afterwards whether or not the command succeeded, through an API server that answers then (the one the step started with may have gone away meanwhile), and the command's status is the result. coordinated() { local status=0 + if ! select_api_server; then + log "cannot step: no API server answers (tried: $(api_candidates_text))" + return 1 + fi peers_ready || return 1 acquire_lease || return 1 "$@" || status=$? - release_lease || log "the lease could not be released; it expires on its own" + { select_api_server && release_lease; } || log "the lease could not be released; it expires on its own" return "$status" } diff --git a/roles/github_runner_cluster/files/regrade/fleet-regrade-docker.sh b/roles/github_runner_cluster/files/regrade/fleet-regrade-docker.sh index 03c222e6..c330f871 100644 --- a/roles/github_runner_cluster/files/regrade/fleet-regrade-docker.sh +++ b/roles/github_runner_cluster/files/regrade/fleet-regrade-docker.sh @@ -1,119 +1,29 @@ -# The Docker actuator for fleet-regrade, run by the regrade service of the cluster role's Compose file. The core, fleet-regrade-core.sh, is placed ahead of this file (concatenated, or sourced by the wrapper beside it); this file supplies the three functions of the actuator contract that the core describes, and runs it. +# The Docker actuator for fleet-regrade, run by the regrade service of the cluster role's Compose file, for a server whose k3s runs in a container. The core, fleet-regrade-core.sh, and the part both k3s actuators share, fleet-regrade-k3s.sh, are placed ahead of this file (sourced, in that order, by the wrapper beside it, fleet-regrade.sh, as FLEET_REGRADE_ACTUATOR=docker in the environment file selects); that file says what the levels are and how they are recorded, and this one says where: in the k3s container, whose entrypoint (files/k3s/node-entrypoint.sh, which uses the same two paths) reads the level at every start. # -# The levels are ways of running the one k3s container of this server. full and control-plane run the same k3s and differ only in the cordon and taint the core puts on the Node, so entering either from the other restarts nothing. etcd-only runs k3s with its API server, controller manager, scheduler and agent turned off: the server keeps its etcd member, and its vote, and nothing else, which it needs the container's entrypoint to start differently, so entering or leaving it restarts the container. +# LEVEL_FILE is on the k3s data volume, so it survives the container being recreated as well as restarted. STARTED_FILE is on the container's tmpfs, so it exists only while the container runs. A level that needs k3s to start differently restarts the container, which is never created again, so the node password, kept in the container's writable layer or in a volume, is the one the cluster already holds. # -# A level is recorded in the container, where the entrypoint (files/k3s/node-entrypoint.sh, which uses the same two paths) reads it at every start. LEVEL_FILE is on the k3s data volume: it survives the container being recreated and the host restarting, so a node stepped down stays stepped down until it is stepped up, and an Ansible run, which changes nothing the container is created from, cannot undo a step. STARTED_FILE is on the container's tmpfs: it exists only while the container runs and holds the level its k3s started at, which is what current_level reports, since k3s hides its arguments from /proc and a level cannot be read back from the process. The container is restarted, never created again, so the node password, kept in the container's writable layer or in a volume, is the one the cluster already holds. -# -# This differs from the NixOS actuator in one respect the core's contract allows: a reboot does not return the node to the first level. A server that restarts at etcd-only comes back at etcd-only, with the cordon the core put on its Node still in place, so the cluster's view and the node's agree. -# -# The Node carries a label, LEVEL_LABEL, with the level as its value, from the first step down until the node is back at the first level. Whatever judges nodes by their readiness can leave a stepped-down server out: at etcd-only it has no kubelet, so it is not Ready by design. -# -# It needs docker and the variables the core reads from the environment file, plus NODE_CONTAINER, the name of this server's k3s container, and LEVEL_LABEL. kubectl is the one in that container (KUBECTL in the environment file is `docker exec` of it), and KUBE_SERVER must name another server's API: the container's own, at etcd-only, answers `apiserver disabled` on some servers. +# It needs docker and the variables fleet-regrade-k3s.sh needs, plus NODE_CONTAINER, the name of this server's k3s container. kubectl is the one in that container (KUBECTL in the environment file is `docker exec` of it), and KUBE_SERVERS must name only other servers' APIs: the container's own, at etcd-only, answers `apiserver disabled` on some servers. LEVEL_FILE=${LEVEL_FILE:-/var/lib/rancher/k3s/regrade-level} STARTED_FILE=${STARTED_FILE:-/run/regrade-level} -# How often a restarted container is asked whether it is back, in seconds: the core's own pace for waiting on the API server. -POLL_SECONDS=2 - -# The line k3s logs once its embedded etcd is serving, at every level and on every start. Waiting for it is how a restart is known to have returned the server's vote, which the core's lease relies on: it is released as soon as apply_level returns, and another server that stepped before this one's etcd was back would stack on top of the outage. -ETCD_READY_LOG='ETCD server is now running' - -levels() { - printf '%s\n' full control-plane etcd-only -} - -# Runs a command in the server's container. -in_node() { +# Runs a command in the server's container, which fails while the container is not running. +on_server() { docker exec -i "$NODE_CONTAINER" "$@" } -# The level the running k3s started at, as its entrypoint recorded it; "unknown" while the container is not running, or when the record is none of the levels (for example one written by another version, or a restart in progress). A running container with neither a record nor a requested level is one whose entrypoint is from before regrading, which could only start k3s whole, so it is at full; its next restart picks up the new entrypoint. -current_level() { - local level - # shellcheck disable=SC2016 # The expansions are for the shell inside the container, which is given the two paths as arguments. - level=$(in_node sh -c 'if [ -r "$1" ]; then cat "$1"; elif [ ! -e "$2" ]; then echo full; fi' sh "$STARTED_FILE" "$LEVEL_FILE" 2> /dev/null) || { - echo unknown - return 0 - } - if levels | grep -Fqx -- "$level"; then - echo "$level" - else - echo unknown - fi -} - -# Puts the label for the level $1 on the Node, or takes it off for the first level. Done before the level is entered, so the label is never missing from a node that has stepped down; the cluster API is reached through another server, which is up at every level. -label_node() { - if [ "$1" = "$(levels | head -n 1)" ]; then - kubectl_ label node "$NODE_NAME" "${LEVEL_LABEL}-" - else - kubectl_ label node "$NODE_NAME" "${LEVEL_LABEL}=$1" --overwrite - fi -} - -# Replaces the file at $1, inside the container, with the level $2, atomically, so a reader never sees half a name. -write_level() { - # shellcheck disable=SC2016 # The expansions are for the shell inside the container, which is given the level and the path as arguments. - in_node sh -c 'umask 077 && printf "%s\n" "$1" > "$2.new" && mv "$2.new" "$2"' sh "$2" "$1" -} - -# How k3s runs at a level: as a server (every level but etcd-only), or as an etcd member alone. Only a change between the two changes the arguments k3s starts with, so only that needs a restart. -k3s_mode() { - if [ "$1" = etcd-only ]; then - echo etcd - else - echo server - fi -} - -# Whether the node has registered again after the container started at $1: it reports Ready and its kubelet has renewed the node lease since. Ready alone is not proof, since the API keeps the old container's Ready condition until the node controller's grace period runs out. Both are UTC, so their first nineteen characters, to the second, order as text. -node_ready_since() { - local started=$1 ready renewed - ready=$(kubectl_ get node "$NODE_NAME" --output 'jsonpath={.status.conditions[?(@.type=="Ready")].status}' 2> /dev/null) || return 1 - renewed=$(kubectl_ get lease --namespace kube-node-lease "$NODE_NAME" --output 'jsonpath={.spec.renewTime}' 2> /dev/null) || return 1 - [ "$ready" = True ] && [[ ${renewed:0:19} > "${started:0:19}" ]] +# The container is given the operation timeout to stop in, and its start time, as Docker records it, is the time of the restart. +restart_server() { + docker restart --time "$OPERATION_TIMEOUT_SECONDS" "$NODE_CONTAINER" > /dev/null || return 1 + RESTARTED_AT=$(docker inspect --format '{{.State.StartedAt}}' "$NODE_CONTAINER") } -# Restarts the container and returns once the server is back as the level $1 runs it: etcd serving again, and, when the level runs the agent, the node Ready on a heartbeat from the new container. Fails when that takes longer than the operation timeout, which leaves the node as it is, for current_level to report. -restart_node() { - local to=$1 started deadline - docker restart --time "$OPERATION_TIMEOUT_SECONDS" "$NODE_CONTAINER" > /dev/null || return 1 - started=$(docker inspect --format '{{.State.StartedAt}}' "$NODE_CONTAINER") || return 1 - deadline=$((SECONDS + OPERATION_TIMEOUT_SECONDS)) - until grep -Fq "$ETCD_READY_LOG" <<< "$(docker logs --since "$started" "$NODE_CONTAINER" 2>&1)"; do - if [ "$SECONDS" -ge "$deadline" ]; then - log "etcd did not come back within ${OPERATION_TIMEOUT_SECONDS}s" - return 1 - fi - sleep "$POLL_SECONDS" - done - [ "$(k3s_mode "$to")" = server ] || return 0 - until node_ready_since "$started"; do - if [ "$SECONDS" -ge "$deadline" ]; then - log "the node did not report Ready within ${OPERATION_TIMEOUT_SECONDS}s" - return 1 - fi - sleep "$POLL_SECONDS" - done +# Only what the container logged since it started counts: the line from before the restart does not. +etcd_serving() { + grep -Fq "$ETCD_READY_LOG" <<< "$(docker logs --since "$RESTARTED_AT" "$NODE_CONTAINER" 2>&1)" } +# Restarting the container stops everything in it, its pods' processes included, so a level needs nothing beyond the shared steps. apply_level() { - local to=$1 from - from=$(current_level) - if [ "$from" = unknown ]; then - log "cannot apply $to: the node is not running at a known level" - return 1 - fi - label_node "$to" || return 1 - write_level "$LEVEL_FILE" "$to" || return 1 - if [ "$(k3s_mode "$from")" = "$(k3s_mode "$to")" ]; then - write_level "$STARTED_FILE" "$to" - else - restart_node "$to" - fi + k3s_apply_level "$1" } - -if [ "${BASH_SOURCE[0]}" = "$0" ]; then - regrade_main "$@" -fi diff --git a/roles/github_runner_cluster/files/regrade/fleet-regrade-k3s.sh b/roles/github_runner_cluster/files/regrade/fleet-regrade-k3s.sh new file mode 100644 index 00000000..bcb548b7 --- /dev/null +++ b/roles/github_runner_cluster/files/regrade/fleet-regrade-k3s.sh @@ -0,0 +1,116 @@ +# What the two k3s actuators of fleet-regrade share: the Docker actuator (fleet-regrade-docker.sh), for a server whose k3s runs in a container under Compose, and the systemd actuator (fleet-regrade-systemd.sh), for a native server whose k3s runs as a systemd unit on the host. The core, fleet-regrade-core.sh, is placed ahead of this file and the platform's actuator after it (sourced in that order by the wrapper beside them, fleet-regrade.sh); between them they supply the three functions of the actuator contract that the core describes. +# +# THE LEVELS. full and control-plane run the same k3s and differ only in the cordon and taint the core puts on the Node, so entering either from the other restarts nothing. etcd-only runs k3s with its API server, controller manager, scheduler and agent turned off: the server keeps its etcd member, and its vote, and nothing else, which needs k3s to start with other arguments, so entering or leaving it restarts k3s. +# +# THE RECORDS. A level is recorded in LEVEL_FILE, on k3s's data directory, which whatever starts k3s (the container's entrypoint, or the unit's ExecStartPre) reads at every start: it survives k3s restarting, its container being recreated and the host restarting, so a node stepped down stays stepped down until it is stepped up, and an Ansible run, which changes nothing the level is read from, cannot undo a step. A reboot does not return the node to the first level, which the core's contract allows: a server that restarts at etcd-only comes back at etcd-only, with the cordon the core put on its Node still in place, so the cluster's view and the node's agree. STARTED_FILE is on a tmpfs that exists only while k3s runs and holds the level its k3s started at, which whatever starts k3s writes and which is what current_level reports, since k3s hides its arguments from /proc and a level cannot be read back from the process. Both are replaced atomically, so a reader never sees half a name. +# +# The Node carries a label, LEVEL_LABEL, with the level as its value, from the first step down until the node is back at the first level. Whatever judges nodes by their readiness can leave a stepped-down server out: at etcd-only it has no kubelet, so it is not Ready by design. +# +# THE PLATFORM. The actuator after this file sets LEVEL_FILE and STARTED_FILE and defines four functions: +# +# on_server CMD... runs a command where this server's k3s runs, and fails without running it while k3s is not running. +# +# restart_server restarts k3s, so that it starts at the level LEVEL_FILE names, and sets RESTARTED_AT to the time it restarted, as RFC 3339 in UTC by the clock the kubelet stamps its heartbeat with. +# +# etcd_serving succeeds once k3s has logged ETCD_READY_LOG since the last restart_server. +# +# apply_level NAME the third function of the core's contract, which calls k3s_apply_level and does whatever else its platform needs. +# +# It needs the variables the core reads from the environment file, plus LEVEL_LABEL. kubectl reaches the cluster through another server's API (KUBE_SERVERS in the environment file), which is up at every level. + +# How often a restarted k3s is asked whether it is back, in seconds: the core's own pace for waiting on the API server. +POLL_SECONDS=2 + +# The line k3s logs once its embedded etcd is serving, at every level and on every start. Waiting for it is how a restart is known to have returned the server's vote, which the core's lease relies on: it is released as soon as apply_level returns, and another server that stepped before this one's etcd was back would stack on top of the outage. +# shellcheck disable=SC2034 # Read by the actuator sourced after this file. +ETCD_READY_LOG='ETCD server is now running' + +levels() { + printf '%s\n' full control-plane etcd-only +} + +# The level the running k3s started at, as STARTED_FILE records it; "unknown" while k3s is not running, or when the record is none of the levels (for example one written by another version, or a restart in progress). A running k3s with neither a record nor a requested level is one started from before regrading, which could only start k3s whole, so it is at full; its next restart records its level. +current_level() { + local level + # shellcheck disable=SC2016 # The expansions are for the shell on_server runs, which is given the two paths as arguments. + level=$(on_server sh -c 'if [ -r "$1" ]; then cat "$1"; elif [ ! -e "$2" ]; then echo full; fi' sh "$STARTED_FILE" "$LEVEL_FILE" 2> /dev/null) || { + echo unknown + return 0 + } + if levels | grep -Fqx -- "$level"; then + echo "$level" + else + echo unknown + fi +} + +# Puts the label for the level $1 on the Node, or takes it off for the first level. Done before the level is entered, so the label is never missing from a node that has stepped down. +label_node() { + if [ "$1" = "$(levels | head -n 1)" ]; then + kubectl_ label node "$NODE_NAME" "${LEVEL_LABEL}-" + else + kubectl_ label node "$NODE_NAME" "${LEVEL_LABEL}=$1" --overwrite + fi +} + +# Replaces the file at $1, where k3s runs, with the level $2, atomically, so a reader never sees half a name. +write_level() { + # shellcheck disable=SC2016 # The expansions are for the shell on_server runs, which is given the level and the path as arguments. + on_server sh -c 'umask 077 && printf "%s\n" "$1" > "$2.new" && mv "$2.new" "$2"' sh "$2" "$1" +} + +# How k3s runs at a level: as a server (every level but etcd-only), or as an etcd member alone. Only a change between the two changes the arguments k3s starts with, so only that needs a restart. +k3s_mode() { + if [ "$1" = etcd-only ]; then + echo etcd + else + echo server + fi +} + +# Whether the node has registered again after k3s restarted at $1: it reports Ready and its kubelet has renewed the node lease since. Ready alone is not proof, since the API keeps the old kubelet's Ready condition until the node controller's grace period runs out. Both are UTC, so their first nineteen characters, to the second, order as text. +node_ready_since() { + local started=$1 ready renewed + ready=$(kubectl_ get node "$NODE_NAME" --output 'jsonpath={.status.conditions[?(@.type=="Ready")].status}' 2> /dev/null) || return 1 + renewed=$(kubectl_ get lease --namespace kube-node-lease "$NODE_NAME" --output 'jsonpath={.spec.renewTime}' 2> /dev/null) || return 1 + [ "$ready" = True ] && [[ ${renewed:0:19} > "${started:0:19}" ]] +} + +# Restarts k3s and returns once the server is back as the level $1 runs it: etcd serving again, and, when the level runs the agent, the node Ready on a heartbeat from the new k3s. Fails when that takes longer than the operation timeout, which leaves the node as it is, for current_level to report. +restart_into() { + local to=$1 deadline + restart_server || return 1 + deadline=$((SECONDS + OPERATION_TIMEOUT_SECONDS)) + until etcd_serving; do + if [ "$SECONDS" -ge "$deadline" ]; then + log "etcd did not come back within ${OPERATION_TIMEOUT_SECONDS}s" + return 1 + fi + sleep "$POLL_SECONDS" + done + [ "$(k3s_mode "$to")" = server ] || return 0 + until node_ready_since "$RESTARTED_AT"; do + if [ "$SECONDS" -ge "$deadline" ]; then + log "the node did not report Ready within ${OPERATION_TIMEOUT_SECONDS}s" + return 1 + fi + sleep "$POLL_SECONDS" + done +} + +# Takes the node to the level $1: labels the Node, records the level for the next start, and either records it as the running level too (a change that runs the same k3s) or restarts k3s into it. +k3s_apply_level() { + local to=$1 from + from=$(current_level) + if [ "$from" = unknown ]; then + log "cannot apply $to: the node is not running at a known level" + return 1 + fi + label_node "$to" || return 1 + write_level "$LEVEL_FILE" "$to" || return 1 + if [ "$(k3s_mode "$from")" = "$(k3s_mode "$to")" ]; then + write_level "$STARTED_FILE" "$to" + else + restart_into "$to" + fi +} diff --git a/roles/github_runner_cluster/files/regrade/fleet-regrade-systemd.sh b/roles/github_runner_cluster/files/regrade/fleet-regrade-systemd.sh new file mode 100644 index 00000000..76248fb5 --- /dev/null +++ b/roles/github_runner_cluster/files/regrade/fleet-regrade-systemd.sh @@ -0,0 +1,75 @@ +# The systemd actuator for fleet-regrade, run by the fleet-regrade unit of a native server (github_runner_cluster_node_mode: native), whose k3s runs on the host as the systemd unit K3S_SERVICE. The core, fleet-regrade-core.sh, and the part both k3s actuators share, fleet-regrade-k3s.sh, are placed ahead of this file (sourced, in that order, by the wrapper beside it, fleet-regrade.sh, as FLEET_REGRADE_ACTUATOR=systemd in the environment file selects); that file says what the levels are and how they are recorded, and this one says where: on the host, where the unit's ExecStartPre (files/native/write-regrade-level.sh, given the same two paths by templates/native/k3s.service.j2) reads the level at every start and renders the server configuration's drop-in from it. +# +# LEVEL_FILE is on k3s's data directory, so it survives the unit restarting and the host rebooting. STARTED_FILE is in the unit's runtime directory, which systemd keeps across a restart of the unit (RuntimeDirectoryPreserve=restart) and removes when the unit stops, so it exists only while k3s runs. +# +# The unit's KillMode=process stops only k3s itself when it restarts: the containerd shims of the node's pods keep running, as they do across any restart of a native node. k3s at etcd-only starts no agent, and so no kubelet and no containerd, to stop them or take them up again, and the pods that survived the drain (DaemonSet pods) would keep running and keep their memory, which defeats the step. Entering etcd-only therefore stops the shims that are left, with everything they run, as k3s's own k3s-killall.sh does; leaving it starts the agent, whose kubelet starts the pods the node should run. +# +# It needs systemctl, journalctl and ps, and the variables fleet-regrade-k3s.sh needs, plus K3S_SERVICE and K3S_DATA_DIR. kubectl is k3s's own (KUBECTL in the environment file is `k3s kubectl`) with the admin kubeconfig k3s wrote while it ran whole, and KUBE_SERVERS must name only other servers' APIs: this server has none at etcd-only. + +K3S_DATA_DIR=${K3S_DATA_DIR:-/var/lib/rancher/k3s} +LEVEL_FILE=${LEVEL_FILE:-$K3S_DATA_DIR/regrade-level} +STARTED_FILE=${STARTED_FILE:-/run/github-runner-k3s-server/regrade-level} + +# Runs a command on the host while the server's unit is active, and fails without running it otherwise, as `docker exec` does for a container that is not running. A unit that is still starting is not active yet (Type=exec reports activating until its ExecStartPre commands have run), so a level is never read from a record its start has not written. +on_server() { + systemctl is-active --quiet "$K3S_SERVICE" || return 1 + "$@" +} + +# The time is read just before the restart, by the host's own clock, which the kubelet stamps its heartbeat with: a heartbeat later than it is from the new k3s. systemctl returns once the unit's ExecStartPre commands have run and k3s has been started, or fails when they could not run. +restart_server() { + # shellcheck disable=SC2034 # Read by restart_into, in fleet-regrade-k3s.sh. + RESTARTED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ) + systemctl restart "$K3S_SERVICE" +} + +# Only what the unit's current run logged counts, read by its invocation id, which systemd gives every start of a unit: the line from the run before the restart has another. The journal is read whole before it is searched, since grep -q would stop reading at the first match and, under pipefail, fail a reader cut off by it. +etcd_serving() { + local invocation + invocation=$(systemctl show --property InvocationID --value "$K3S_SERVICE") || return 1 + [ -n "$invocation" ] || return 1 + grep -Fq "$ETCD_READY_LOG" <<< "$(journalctl --no-pager --output cat "_SYSTEMD_INVOCATION_ID=$invocation")" +} + +# The process ids of the containerd shims k3s started for the node's pods, found as k3s-killall.sh (the script k3s's install.sh writes) finds them: a process whose command is one of the shim binaries k3s unpacks under its data directory, data//bin/containerd-shim*. +pod_shims() { + ps -e -o pid= -o args= | awk -v data="$K3S_DATA_DIR/data/" ' + index($2, data) == 1 && substr($2, length(data) + 1) ~ /^[^\/]+\/bin\/containerd-shim[^\/]*$/ { print $1 }' +} + +# The process ids given as arguments and every process descended from them, as k3s-killall.sh's pstree walks the tree: a shim's children are the containers of its pod. +process_tree() { + ps -e -o pid= -o ppid= | awk -v roots="$*" ' + BEGIN { count = split(roots, list, " "); for (i = 1; i <= count; i++) keep[list[i]] = 1 } + { parent[$1] = $2 } + END { + do { + grew = 0 + for (pid in parent) if (!(pid in keep) && (parent[pid] in keep)) { keep[pid] = 1; grew = 1 } + } while (grew) + for (pid in keep) print pid + }' +} + +# Stops the shims that outlived k3s's agent, with everything they run, by SIGKILL as k3s-killall.sh does, and returns once none is left, failing when that takes longer than the operation timeout. The signals are not checked one by one: a process that exits between the listing and the signal is gone either way, and what decides the result is whether any shim is left. +stop_pod_shims() { + local shims deadline + shims=$(pod_shims) + [ -n "$shims" ] || return 0 + log "stopping the pods left without a kubelet at etcd-only: containerd shims $(echo "$shims" | tr '\n' ' ')" + # shellcheck disable=SC2046,SC2086 # One process id per word, on purpose. + kill -KILL $(process_tree $shims) 2> /dev/null || true + deadline=$((SECONDS + OPERATION_TIMEOUT_SECONDS)) + until [ -z "$(pod_shims)" ]; do + if [ "$SECONDS" -ge "$deadline" ]; then + log "containerd shims still running after ${OPERATION_TIMEOUT_SECONDS}s: $(pod_shims | tr '\n' ' ')" + return 1 + fi + sleep "$POLL_SECONDS" + done +} + +apply_level() { + k3s_apply_level "$1" || return 1 + [ "$1" != etcd-only ] || stop_pod_shims +} diff --git a/roles/github_runner_cluster/files/regrade/fleet-regrade.sh b/roles/github_runner_cluster/files/regrade/fleet-regrade.sh index 8b46f546..943a3ca1 100755 --- a/roles/github_runner_cluster/files/regrade/fleet-regrade.sh +++ b/roles/github_runner_cluster/files/regrade/fleet-regrade.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Runs fleet-regrade in the regrade service of the cluster role's Compose file: the platform-neutral core, then the Docker actuator, both from the directory the Compose file mounts beside this script, so a collection upgrade reaches the service when it restarts without the image being built again. The settings are in fleet-regrade.env in the same directory, which the role templates. +# Runs fleet-regrade: the platform-neutral core, the part both k3s actuators share, then the actuator FLEET_REGRADE_ACTUATOR names in the settings (docker, for the regrade service of the cluster role's Compose file, or systemd, for the fleet-regrade unit of a native server), all from the directory this script is in, so a collection upgrade reaches the service when it restarts, without the Compose service's image being built again. The settings are in fleet-regrade.env in the same directory, which the role templates. set -euo pipefail lib=$(dirname "$(readlink -f "$0")") @@ -7,14 +7,21 @@ FLEET_REGRADE_ENV=${FLEET_REGRADE_ENV:-$lib/fleet-regrade.env} # shellcheck source=/dev/null . "$lib/fleet-regrade-core.sh" + +case ${FLEET_REGRADE_ACTUATOR:-} in + docker | systemd) ;; + *) die "FLEET_REGRADE_ACTUATOR is '${FLEET_REGRADE_ACTUATOR:-}' in $FLEET_REGRADE_ENV; it must be docker or systemd." ;; +esac +# shellcheck source=/dev/null +. "$lib/fleet-regrade-k3s.sh" # shellcheck source=/dev/null -. "$lib/fleet-regrade-docker.sh" +. "$lib/fleet-regrade-$FLEET_REGRADE_ACTUATOR.sh" -# Memory pressure is what the core decides on. A kernel without it (built without CONFIG_PSI, or booted with psi=0) would fail on every sample, so the service stops at once and says why. +# Memory pressure is what the core decides on. A kernel without it (built without CONFIG_PSI, or booted with psi=0) would fail on every sample, so the service stops at once and says why. The role refuses to turn regrading on where it can read that the kernel has none. case ${1:-} in watch | status) - if [ ! -r "${PSI_FILE:-/proc/pressure/memory}" ]; then - echo "The kernel of this Docker host exposes no memory pressure (${PSI_FILE:-/proc/pressure/memory} is not readable): it needs CONFIG_PSI and no psi=0 on its command line. Regrading cannot decide without it." >&2 + if [ ! -r "$PSI_FILE" ]; then + echo "The kernel this server runs on exposes no memory pressure ($PSI_FILE is not readable): it needs CONFIG_PSI and no psi=0 on its command line. Regrading cannot decide without it." >&2 exit 1 fi ;; diff --git a/roles/github_runner_cluster/meta/argument_specs.yml b/roles/github_runner_cluster/meta/argument_specs.yml index 9ba35d3c..3b75a8a4 100644 --- a/roles/github_runner_cluster/meta/argument_specs.yml +++ b/roles/github_runner_cluster/meta/argument_specs.yml @@ -80,3 +80,11 @@ argument_specs: type: path default: /etc/apt/sources.list.d/tailscale.list description: Where Tailscale's apt repository list file goes. Not a setting, and a variable only so the tests can point it at a stand-in. + github_runner_cluster_native_run_dir: + type: path + default: /run + description: Where systemd makes the server and agent units' runtime directories, which hold the resolver file the kubelet reads and the regrading level a native server's k3s started at. Not a setting, and a variable only so the tests can point it at a stand-in. + github_runner_cluster_native_psi_file: + type: path + default: /proc/pressure/memory + description: The kernel's memory pressure, which the role reads to refuse memory-pressure regrading on a native host whose kernel has none. Not a setting, and a variable only so the tests can point it at a stand-in. diff --git a/roles/github_runner_cluster/tasks/native/config.yml b/roles/github_runner_cluster/tasks/native/config.yml index 12fd0f7a..3e87e4de 100644 --- a/roles/github_runner_cluster/tasks/native/config.yml +++ b/roles/github_runner_cluster/tasks/native/config.yml @@ -27,6 +27,7 @@ - {src: write-resolv-conf.sh, dest: "{{ github_runner_cluster_native_lib_dir }}/write-resolv-conf"} - {src: write-node-ip.sh, dest: "{{ github_runner_cluster_native_lib_dir }}/write-node-ip"} - {src: fleet-role-is.sh, dest: "{{ github_runner_cluster_native_lib_dir }}/fleet-role-is"} + - {src: write-regrade-level.sh, dest: "{{ github_runner_cluster_native_lib_dir }}/write-regrade-level"} loop_control: label: "{{ item.dest }}" register: github_runner_cluster_native_helpers diff --git a/roles/github_runner_cluster/tasks/native/demote.yml b/roles/github_runner_cluster/tasks/native/demote.yml index 85d805f6..eb2fa733 100644 --- a/roles/github_runner_cluster/tasks/native/demote.yml +++ b/roles/github_runner_cluster/tasks/native/demote.yml @@ -98,6 +98,13 @@ argv: [systemctl, disable, --now, "{{ github_runner_cluster_native_server_service }}"] changed_when: true + - name: Forget the regrading level the server was left at + # The level is a server's, and the agent unit never reads it; a later promotion must start the server whole, not at a level it was stepped to in an earlier term as a server. + become: true + ansible.builtin.file: + path: "{{ github_runner_cluster_native_regrade_level_file }}" + state: absent + - name: Move the server's datastore aside, keeping only the newest copy # A copy, not a deletion: it holds the snapshots this server took (server/db/snapshots). One copy only, the one from this demotion, so repeated switches do not fill the disk. become: true diff --git a/roles/github_runner_cluster/tasks/native/main.yml b/roles/github_runner_cluster/tasks/native/main.yml index edaac43a..222b56e5 100644 --- a/roles/github_runner_cluster/tasks/native/main.yml +++ b/roles/github_runner_cluster/tasks/native/main.yml @@ -54,11 +54,6 @@ host's own network with the host's own Tailscale client, and has no Docker network to choose. Remove github_runner_cluster_network_mode (or set it to bridge) for this host. bad: "{{ github_runner_cluster_network_mode == 'host' }}" - - msg: >- - github_runner_cluster_regrade is true for {{ inventory_hostname }}, which sets github_runner_cluster_node_mode: native. Memory-pressure - regrading steps a server down through its actuator, and the only one there is drives a Docker server's container; a native server has - none yet. Remove github_runner_cluster_regrade for this host. - bad: "{{ github_runner_cluster_regrade | bool }}" loop_control: label: "{{ item.msg | split(' ') | first }}" when: item.bad | bool @@ -91,6 +86,9 @@ - name: Decide which role the host runs, and whether a server bootstraps a new cluster ansible.builtin.include_tasks: native/role.yml +- name: Check the memory-pressure regrading settings, for the role the host is to run + ansible.builtin.include_tasks: regrade.yml + - name: Install the pinned Tailscale client and check the tailnet it is logged in to ansible.builtin.include_tasks: native/tailscale.yml @@ -102,3 +100,6 @@ - name: Run the unit of the host's role and check the node joined ansible.builtin.include_tasks: native/service.yml + +- name: Run the fleet-regrade unit beside this server, or remove it + ansible.builtin.include_tasks: native/regrade_service.yml diff --git a/roles/github_runner_cluster/tasks/native/regrade_service.yml b/roles/github_runner_cluster/tasks/native/regrade_service.yml new file mode 100644 index 00000000..537ab880 --- /dev/null +++ b/roles/github_runner_cluster/tasks/native/regrade_service.yml @@ -0,0 +1,173 @@ +--- +# Runs the fleet-regrade unit on a native server when github_runner_cluster_regrade is true, and removes it when it is not: the native counterpart of tasks/regrade_service.yml. The unit runs fleet-regrade's core with the systemd actuator, from scripts and settings placed under github_runner_cluster_native_regrade_dir, and `fleet-regrade` on the PATH runs them by hand (status, down, up). Run after service.yml has run the host's unit; tasks/regrade.yml has already checked the settings, so regrading is on only for a server. While github_runner_cluster_native_start is false the unit is left as it is found, running or not, as the k3s units are, and only its files are written. Under --check nothing is changed and each change a real run would make is reported. + +- name: Run the fleet-regrade unit + when: github_runner_cluster_regrade | bool + block: + - name: Install the packages regrading needs that the host lacks + # tasks/regrade.yml read which they are, and refused a host that is not Debian or Ubuntu that lacks any. The index is refreshed first, since a host's lists can be missing or stale. + when: github_runner_cluster_native_regrade_packages | length > 0 + block: + - name: Refresh the package index + become: true + ansible.builtin.command: + argv: [apt-get, update] + environment: + DEBIAN_FRONTEND: noninteractive + changed_when: true + + - name: Install the packages + become: true + ansible.builtin.command: + argv: "{{ ['apt-get', 'install', '--yes', '--no-install-recommends'] + github_runner_cluster_native_regrade_packages }}" + environment: + DEBIAN_FRONTEND: noninteractive + changed_when: true + + - name: Create the directory the regrade scripts and settings go in + become: true + ansible.builtin.file: + path: "{{ github_runner_cluster_native_regrade_dir }}" + state: directory + mode: "0755" + + - name: Place the regrade scripts + # Copied on every run, so a collection upgrade reaches the host; the unit reads them at its start, so a changed one restarts it, below. + become: true + ansible.builtin.copy: + src: "regrade/{{ item.name }}" + dest: "{{ github_runner_cluster_native_regrade_dir }}/{{ item.name }}" + mode: "{{ item.mode }}" + loop: + - {name: fleet-regrade.sh, mode: "0755"} + - {name: fleet-regrade-core.sh, mode: "0644"} + - {name: fleet-regrade-k3s.sh, mode: "0644"} + - {name: fleet-regrade-systemd.sh, mode: "0644"} + register: github_runner_cluster_native_regrade_scripts + + - name: Template the regrade settings + become: true + ansible.builtin.template: + src: regrade.env.j2 + dest: "{{ github_runner_cluster_native_regrade_dir }}/fleet-regrade.env" + mode: "0644" + register: github_runner_cluster_native_regrade_settings + + - name: Put fleet-regrade on the PATH, for stepping by hand + become: true + ansible.builtin.file: + src: "{{ github_runner_cluster_native_regrade_dir }}/fleet-regrade.sh" + dest: "{{ github_runner_cluster_native_bin_dir }}/fleet-regrade" + state: link + # Under --check the scripts above have not been placed, so the link's target may not exist yet. + force: true + + - name: Write the fleet-regrade unit + become: true + ansible.builtin.template: + src: native/fleet-regrade.service.j2 + dest: "{{ github_runner_cluster_native_unit_dir }}/{{ github_runner_cluster_native_regrade_service }}.service" + mode: "0644" + register: github_runner_cluster_native_regrade_unit + + - name: Reload systemd when the unit changed + become: true + ansible.builtin.command: + argv: [systemctl, daemon-reload] + changed_when: true + when: + - github_runner_cluster_native_regrade_unit is changed + - not ansible_check_mode + + - name: Read whether the unit is enabled and running + ansible.builtin.command: + argv: [systemctl, "{{ item }}", "{{ github_runner_cluster_native_regrade_service }}"] + loop: [is-enabled, is-active] + register: github_runner_cluster_native_regrade_state + changed_when: false + failed_when: false + check_mode: false + + - name: Enable the unit + become: true + ansible.builtin.command: + argv: [systemctl, enable, "{{ github_runner_cluster_native_regrade_service }}"] + changed_when: true + when: + - github_runner_cluster_native_start | bool + - github_runner_cluster_native_regrade_state.results[0].rc != 0 + + - name: Start the unit, or restart it since something it reads changed + # The watcher reads its scripts and settings once, at its start. A unit nothing changed for is left running, with its streaks. + become: true + ansible.builtin.command: + argv: + - systemctl + - "{{ 'restart' if github_runner_cluster_native_regrade_state.results[1].rc == 0 else 'start' }}" + - "{{ github_runner_cluster_native_regrade_service }}" + changed_when: true + when: + - github_runner_cluster_native_start | bool + - >- + github_runner_cluster_native_regrade_state.results[1].rc != 0 + or github_runner_cluster_native_regrade_scripts is changed + or github_runner_cluster_native_regrade_settings is changed + or github_runner_cluster_native_regrade_unit is changed + +- name: Stop regrading + when: + - not (github_runner_cluster_regrade | bool) + - github_runner_cluster_native_start | bool + block: + - name: Read the level this server's k3s started at + # Read-only, so it runs under --check too. Absent while the server unit is not running, on an agent, and on a server that has never started with the level's ExecStartPre. + become: true + ansible.builtin.slurp: + src: "{{ github_runner_cluster_native_regrade_started_file }}" + register: github_runner_cluster_native_regrade_level + failed_when: false + check_mode: false + + - name: Fail if the server is stepped down + # Without the unit nothing steps it back up, and the server unit's ExecStartPre would keep starting it at the level it was left at. Fatal for the whole play, as tasks/regrade_service.yml's refusal is, so that no other host removes its service while one server is still stepped down. + any_errors_fatal: true + ansible.builtin.fail: + msg: >- + github_runner_cluster_regrade is false, but {{ inventory_hostname }} is running at the level + + {{ github_runner_cluster_native_regrade_level.content | b64decode | trim }}, below full, and the fleet-regrade unit is what steps it + back up. Run `fleet-regrade up` on {{ inventory_hostname }} until it reports full (`fleet-regrade status` shows the level), then run + again. Nothing was changed. + when: + - github_runner_cluster_native_regrade_level.content is defined + - github_runner_cluster_native_regrade_level.content | b64decode | trim != 'full' + + - name: Read whether the unit is installed + ansible.builtin.stat: + path: "{{ github_runner_cluster_native_unit_dir }}/{{ github_runner_cluster_native_regrade_service }}.service" + register: github_runner_cluster_native_regrade_installed + + - name: Remove the unit, its scripts and settings, and the command on the PATH + when: github_runner_cluster_native_regrade_installed.stat.exists + block: + - name: Stop and disable the unit + become: true + ansible.builtin.command: + argv: [systemctl, disable, --now, "{{ github_runner_cluster_native_regrade_service }}"] + changed_when: true + + - name: Remove the unit's files + become: true + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - "{{ github_runner_cluster_native_unit_dir }}/{{ github_runner_cluster_native_regrade_service }}.service" + - "{{ github_runner_cluster_native_regrade_dir }}" + - "{{ github_runner_cluster_native_bin_dir }}/fleet-regrade" + + - name: Reload systemd, which still knows the removed unit + become: true + ansible.builtin.command: + argv: [systemctl, daemon-reload] + changed_when: true diff --git a/roles/github_runner_cluster/tasks/native/server_turn.yml b/roles/github_runner_cluster/tasks/native/server_turn.yml index 23d8f3d7..5cec7151 100644 --- a/roles/github_runner_cluster/tasks/native/server_turn.yml +++ b/roles/github_runner_cluster/tasks/native/server_turn.yml @@ -119,9 +119,40 @@ vars: github_runner_cluster_native_api_self: true + - name: Read the regrading level the server started at + # Written by the unit's ExecStartPre before k3s starts (files/native/write-regrade-level.sh), so it is there once the start has returned. + become: true + ansible.builtin.slurp: + src: "{{ github_runner_cluster_native_regrade_started_file }}" + register: github_runner_cluster_native_started_level + - name: Wait for this server's node to be Ready on a new heartbeat and an etcd voter # Ready alone is not enough, as for a recreated Docker server (tasks/server_ready.yml): the node's lease must have been renewed after the start. A voter, because k3s adds a new server as a learner and the turn is over only once etcd counts it: k3s's EtcdIsVoter condition is True on a voting member's Node (setEtcdStatusCondition in pkg/etcd/etcd.go). ansible.builtin.include_tasks: native/server_ready.yml + when: github_runner_cluster_native_started_level.content | b64decode | trim != 'etcd-only' + + - name: Wait for the etcd of a server that regrading has stepped down to etcd-only to serve again + # At etcd-only the server runs no kubelet, so its node is never Ready; what the restart has to give back is its etcd member, which k3s reports by logging that line once its etcd is serving, as the systemd actuator waits for it (files/regrade/fleet-regrade-systemd.sh). Only the unit's current run counts, read by its invocation id. + become: true + ansible.builtin.shell: | + set -eu + invocation=$(systemctl show --property InvocationID --value "$service") + [ -n "$invocation" ] + case "$(journalctl --no-pager --output cat "_SYSTEMD_INVOCATION_ID=${invocation}")" in + *'ETCD server is now running'*) ;; + *) exit 1 ;; + esac + environment: + service: "{{ github_runner_cluster_native_server_service }}" + register: github_runner_cluster_native_etcd_back + until: github_runner_cluster_native_etcd_back.rc == 0 + retries: "{{ ((github_runner_cluster_native_ready_timeout | int) / github_runner_cluster_native_poll) | round(0, 'ceil') | int }}" + delay: "{{ github_runner_cluster_native_poll }}" + vars: + # Seconds between reads, as tasks/native/server_ready.yml polls. + github_runner_cluster_native_poll: 5 + changed_when: false + when: github_runner_cluster_native_started_level.content | b64decode | trim == 'etcd-only' - name: Let the node run workloads again become: true diff --git a/roles/github_runner_cluster/tasks/regrade.yml b/roles/github_runner_cluster/tasks/regrade.yml index a124a9cb..91549d7c 100644 --- a/roles/github_runner_cluster/tasks/regrade.yml +++ b/roles/github_runner_cluster/tasks/regrade.yml @@ -1,5 +1,5 @@ --- -# Checks the memory-pressure regrading settings (github_runner_cluster_regrade and the settings beside it) before the role changes anything on the host. Run after the topology is known: whether a step down can work depends on the other servers. Off is the default and passes. +# Checks the memory-pressure regrading settings (github_runner_cluster_regrade and the settings beside it) before the role changes anything on the host. Run after the topology is known: whether a step down can work depends on the other servers. On a native host (tasks/native/main.yml) it runs once the role the host is to run is decided too (tasks/native/role.yml), and also checks what the host's kernel and tools give the systemd actuator. Off is the default and passes. - name: Fail if regrading is not true or false ansible.builtin.assert: @@ -11,9 +11,9 @@ when: github_runner_cluster_regrade | bool block: - name: Fail if this host is not a server - # An agent has no control plane to shed and no etcd vote to keep. + # An agent has no control plane to shed and no etcd vote to keep. A native host is judged by the role it is to run after this run, which for a host marked github_runner_cluster_voter_eligible is the one recorded on the host, not the inventory's. ansible.builtin.assert: - that: github_runner_cluster_effective_node_role == 'server' + that: (github_runner_cluster_native_role if github_runner_cluster_node_mode == 'native' else github_runner_cluster_effective_node_role) == 'server' quiet: true fail_msg: >- github_runner_cluster_regrade is true, but {{ inventory_hostname }} is an agent. Regrading steps a server down while it @@ -99,3 +99,46 @@ github_runner_cluster_regrade_lease_duration_seconds ({{ github_runner_cluster_regrade_lease_duration_seconds }}) must be at least three times github_runner_cluster_regrade_operation_timeout_seconds ({{ github_runner_cluster_regrade_operation_timeout_seconds }}), or another server could take the Lease while this one is still stepping. Nothing was changed. + + - name: Check what a native server's kernel and tools give regrading + when: github_runner_cluster_node_mode == 'native' + block: + - name: Read whether the kernel exposes memory pressure, and which of the commands regrading needs are installed + # Read-only, so it runs under --check too. The core decides on the kernel's memory pressure and parses the cluster's answers with jq, and the systemd actuator lists the node's processes with ps. + ansible.builtin.shell: | + set -u + if [ -r "$psi" ]; then echo psi; fi + for command in $commands; do + if command -v "$command" >/dev/null; then echo "$command"; fi + done + environment: + psi: "{{ github_runner_cluster_native_psi_file }}" + commands: "{{ github_runner_cluster_native_regrade_commands.keys() | join(' ') }}" + register: github_runner_cluster_native_regrade_host + changed_when: false + check_mode: false + + - name: Record the packages regrading needs that are not installed + ansible.builtin.set_fact: + github_runner_cluster_native_regrade_packages: >- + {{ github_runner_cluster_native_regrade_commands | dict2items + | rejectattr('key', 'in', github_runner_cluster_native_regrade_host.stdout_lines) | map(attribute='value') | list }} + + - name: Fail if the kernel exposes no memory pressure + # The service would stop at its first sample (files/regrade/fleet-regrade.sh), so it is refused here, before anything changes. + ansible.builtin.fail: + msg: >- + github_runner_cluster_regrade is true, but the kernel of {{ inventory_hostname }} exposes no memory pressure + ({{ github_runner_cluster_native_psi_file }} is not readable): it needs CONFIG_PSI and no psi=0 on its command line, and + regrading cannot decide without it. Nothing was changed. + when: "'psi' not in github_runner_cluster_native_regrade_host.stdout_lines" + + - name: Fail if a command regrading needs is missing on a host the role cannot install it on + ansible.builtin.fail: + msg: >- + github_runner_cluster_regrade is true, but {{ inventory_hostname }} lacks the packages + {{ github_runner_cluster_native_regrade_packages | join(', ') }}, which regrading needs, and the role installs them from the + distribution's packages on Debian and Ubuntu only. Install them and run again. Nothing was changed. + when: + - github_runner_cluster_native_regrade_packages | length > 0 + - ansible_facts['os_family'] != 'Debian' diff --git a/roles/github_runner_cluster/tasks/regrade_service.yml b/roles/github_runner_cluster/tasks/regrade_service.yml index 4620598e..b1edb6d2 100644 --- a/roles/github_runner_cluster/tasks/regrade_service.yml +++ b/roles/github_runner_cluster/tasks/regrade_service.yml @@ -20,6 +20,7 @@ - {name: Dockerfile, mode: "0644"} - {name: fleet-regrade.sh, mode: "0755"} - {name: fleet-regrade-core.sh, mode: "0644"} + - {name: fleet-regrade-k3s.sh, mode: "0644"} - {name: fleet-regrade-docker.sh, mode: "0644"} register: github_runner_cluster_regrade_scripts diff --git a/roles/github_runner_cluster/templates/native/agent.yaml.j2 b/roles/github_runner_cluster/templates/native/agent.yaml.j2 index b36ccc9e..b277b065 100644 --- a/roles/github_runner_cluster/templates/native/agent.yaml.j2 +++ b/roles/github_runner_cluster/templates/native/agent.yaml.j2 @@ -4,7 +4,7 @@ server: {{ github_runner_cluster_native_join_url | to_json }} token-file: {{ (github_runner_cluster_native_config_dir ~ '/token') | to_json }} node-name: {{ github_runner_cluster_effective_node_name | to_json }} vpn-auth-file: {{ (github_runner_cluster_native_config_dir ~ '/vpn-auth') | to_json }} -resolv-conf: "/run/github-runner-k3s/resolv.conf" +resolv-conf: {{ (github_runner_cluster_native_run_dir ~ '/' ~ github_runner_cluster_native_agent_runtime_dir ~ '/resolv.conf') | to_json }} {% if github_runner_cluster_node_taints | length > 0 %} node-taint: {% for taint in github_runner_cluster_node_taints %} diff --git a/roles/github_runner_cluster/templates/native/fleet-regrade.service.j2 b/roles/github_runner_cluster/templates/native/fleet-regrade.service.j2 new file mode 100644 index 00000000..d423e492 --- /dev/null +++ b/roles/github_runner_cluster/templates/native/fleet-regrade.service.j2 @@ -0,0 +1,21 @@ +# Managed by the exadev.github_runner collection's github_runner_cluster role (node mode native, github_runner_cluster_regrade: true). Do not edit by hand: change this host's variables and run the role again. +# Steps this native server down through levels of reduced function when memory runs short, and back up, while it keeps its etcd vote; see the role README's Memory-pressure regrading. It runs fleet-regrade's core with the systemd actuator (files/regrade/fleet-regrade-systemd.sh), which reads the host's own memory pressure and applies a level by recording it for the {{ github_runner_cluster_native_server_service }} unit's ExecStartPre and restarting that unit, so it runs as root, as that unit does. +[Unit] +Description=fleet-regrade memory-pressure regrading of this native k3s server +Documentation=https://github.com/ExaDev/github-runner/tree/main/roles/github_runner_cluster#memory-pressure-regrading +# Ordered after the server unit and bound to nothing: it restarts that unit itself, and must keep running while it does. +After={{ github_runner_cluster_native_server_service }}.service + +[Service] +Type=exec +User=root +Environment="FLEET_REGRADE_ENV={{ github_runner_cluster_native_regrade_dir }}/fleet-regrade.env" +# k3s, for `k3s kubectl`, is in the role's binary directory. +Environment="PATH={{ github_runner_cluster_native_bin_dir }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +ExecStart={{ github_runner_cluster_native_regrade_dir }}/fleet-regrade.sh watch +# The watcher keeps its streaks in memory, so a restart only starts them again; a kernel without memory pressure stops it at once, with the reason in the journal, and the role refuses to turn regrading on there in the first place. It is restarted after one sample period, the pace it reads at in any case. +Restart=always +RestartSec={{ github_runner_cluster_regrade_sample_seconds }} + +[Install] +WantedBy=multi-user.target diff --git a/roles/github_runner_cluster/templates/native/k3s-agent.service.j2 b/roles/github_runner_cluster/templates/native/k3s-agent.service.j2 index 02bbc8a2..d31c7918 100644 --- a/roles/github_runner_cluster/templates/native/k3s-agent.service.j2 +++ b/roles/github_runner_cluster/templates/native/k3s-agent.service.j2 @@ -26,14 +26,14 @@ TasksMax=infinity TimeoutStartSec=0 Restart=always RestartSec=5s -# Kept across restarts, so the file the kubelet reads is there while k3s starts again; removed when the service stops. -RuntimeDirectory=github-runner-k3s +# Kept across restarts, so the file the kubelet reads is there while k3s starts again; removed when the service stops, and when a start is skipped by the condition below, so it is this unit's alone and never shared with the other k3s unit. +RuntimeDirectory={{ github_runner_cluster_native_agent_runtime_dir }} RuntimeDirectoryPreserve=restart # Starts only while the role file names this host an agent (files/native/fleet-role-is.sh), so this unit and the k3s server unit never run together and Restart=always never starts the agent on a host that is now a server. ExecCondition={{ github_runner_cluster_native_lib_dir }}/fleet-role-is agent {{ github_runner_cluster_native_role_file }} ExecStartPre=-/sbin/modprobe br_netfilter ExecStartPre=-/sbin/modprobe overlay -ExecStartPre={{ github_runner_cluster_native_lib_dir }}/write-resolv-conf /run/github-runner-k3s/resolv.conf +ExecStartPre={{ github_runner_cluster_native_lib_dir }}/write-resolv-conf {{ github_runner_cluster_native_run_dir }}/{{ github_runner_cluster_native_agent_runtime_dir }}/resolv.conf ExecStart={{ github_runner_cluster_native_bin_dir }}/k3s agent --config {{ github_runner_cluster_native_config_dir }}/agent.yaml [Install] diff --git a/roles/github_runner_cluster/templates/native/k3s.service.j2 b/roles/github_runner_cluster/templates/native/k3s.service.j2 index 46d60d13..16fe3cbb 100644 --- a/roles/github_runner_cluster/templates/native/k3s.service.j2 +++ b/roles/github_runner_cluster/templates/native/k3s.service.j2 @@ -26,16 +26,18 @@ TasksMax=infinity TimeoutStartSec=0 Restart=always RestartSec=5s -# Kept across restarts, so the file the kubelet reads is there while k3s starts again; removed when the service stops. -RuntimeDirectory=github-runner-k3s +# Kept across restarts, so the file the kubelet reads is there while k3s starts again; removed when the service stops, and when a start is skipped by the condition below, so it is this unit's alone and never shared with the other k3s unit. +RuntimeDirectory={{ github_runner_cluster_native_server_runtime_dir }} RuntimeDirectoryPreserve=restart # Starts only while the role file names this host a server (files/native/fleet-role-is.sh), so this unit and the k3s agent unit never run together and Restart=always never starts a server that has been demoted, which would join the cluster again as a learner. ExecCondition={{ github_runner_cluster_native_lib_dir }}/fleet-role-is server {{ github_runner_cluster_native_role_file }} ExecStartPre=-/sbin/modprobe br_netfilter ExecStartPre=-/sbin/modprobe overlay -ExecStartPre={{ github_runner_cluster_native_lib_dir }}/write-resolv-conf /run/github-runner-k3s/resolv.conf +ExecStartPre={{ github_runner_cluster_native_lib_dir }}/write-resolv-conf {{ github_runner_cluster_native_run_dir }}/{{ github_runner_cluster_native_server_runtime_dir }}/resolv.conf # etcd's peer URL is the host's Tailscale address, written as node-ip into the configuration's drop-in directory before each start (files/native/write-node-ip.sh). ExecStartPre={{ github_runner_cluster_native_lib_dir }}/write-node-ip {{ github_runner_cluster_native_config_dir }}/server.yaml.d/00-node-ip.yaml {{ github_runner_cluster_native_config_dir }}/vpn-auth +# The regrading level fleet-regrade last recorded for this server (files/native/write-regrade-level.sh): the drop-in that turns all but etcd off at etcd-only, and the record of the level k3s starts at, which the systemd actuator (files/regrade/fleet-regrade-systemd.sh) reads. Written before every start whether or not regrading is on, so that turning it on or off changes nothing in this unit; a server that has never been stepped down starts at full. +ExecStartPre={{ github_runner_cluster_native_lib_dir }}/write-regrade-level {{ github_runner_cluster_native_regrade_level_file }} {{ github_runner_cluster_native_config_dir }}/server.yaml.d/50-regrade.yaml {{ github_runner_cluster_native_regrade_started_file }} ExecStart={{ github_runner_cluster_native_bin_dir }}/k3s server --config {{ github_runner_cluster_native_config_dir }}/server.yaml [Install] diff --git a/roles/github_runner_cluster/templates/native/server.yaml.j2 b/roles/github_runner_cluster/templates/native/server.yaml.j2 index f32613e9..9f7b15ad 100644 --- a/roles/github_runner_cluster/templates/native/server.yaml.j2 +++ b/roles/github_runner_cluster/templates/native/server.yaml.j2 @@ -22,7 +22,7 @@ etcd-arg: - {{ ('heartbeat-interval=' ~ github_runner_cluster_etcd_heartbeat_interval) | to_json }} - {{ ('election-timeout=' ~ github_runner_cluster_etcd_election_timeout) | to_json }} {% endif %} -resolv-conf: "/run/github-runner-k3s/resolv.conf" +resolv-conf: {{ (github_runner_cluster_native_run_dir ~ '/' ~ github_runner_cluster_native_server_runtime_dir ~ '/resolv.conf') | to_json }} {% if github_runner_cluster_node_taints | length > 0 %} node-taint: {% for taint in github_runner_cluster_node_taints %} diff --git a/roles/github_runner_cluster/templates/regrade.env.j2 b/roles/github_runner_cluster/templates/regrade.env.j2 index b1484237..719b94f1 100644 --- a/roles/github_runner_cluster/templates/regrade.env.j2 +++ b/roles/github_runner_cluster/templates/regrade.env.j2 @@ -1,4 +1,4 @@ -# Generated by the exadev.github_runner collection's github_runner_cluster role on every run. Do not edit by hand: change this host's ansible/host_vars file and rerun ansible-playbook. Read by the regrade service (docker-compose.yml, regrade/); see the role README's Memory-pressure regrading. +# Generated by the exadev.github_runner collection's github_runner_cluster role on every run. Do not edit by hand: change this host's ansible/host_vars file and rerun ansible-playbook. Read by the regrade service (docker-compose.yml, regrade/) or, on a native server, the fleet-regrade unit (templates/native/fleet-regrade.service.j2); see the role README's Memory-pressure regrading. PRESSURE_THRESHOLD={{ github_runner_cluster_regrade_pressure_threshold }} AVAILABLE_FLOOR_PERCENT={{ github_runner_cluster_regrade_available_floor_percent }} SAMPLE_SECONDS={{ github_runner_cluster_regrade_sample_seconds }} @@ -8,10 +8,25 @@ OPERATION_TIMEOUT_SECONDS={{ github_runner_cluster_regrade_operation_timeout_sec LEASE_DURATION_SECONDS={{ github_runner_cluster_regrade_lease_duration_seconds }} STEP_UP={{ github_runner_cluster_regrade_step_up }} NODE_NAME={{ github_runner_cluster_effective_node_name }} -NODE_CONTAINER={{ github_runner_cluster_node_container }} CONTROL_PLANE_TAINT={{ github_runner_cluster_regrade_taint }} LEVEL_LABEL={{ github_runner_cluster_regrade_level_label }} +{% if github_runner_cluster_node_mode == 'native' %} +{# A native server: k3s is the unit on this host, kubectl is k3s's own, and the kubeconfig is the admin one k3s writes, which outlasts a start at etcd-only, where k3s writes none. #} +FLEET_REGRADE_ACTUATOR=systemd +K3S_SERVICE={{ github_runner_cluster_native_server_service }} +K3S_DATA_DIR={{ github_runner_cluster_native_data_dir }} +LEVEL_FILE={{ github_runner_cluster_native_regrade_level_file }} +STARTED_FILE={{ github_runner_cluster_native_regrade_started_file }} +KUBECTL="{{ github_runner_cluster_native_bin_dir }}/k3s kubectl" +KUBECONFIG_PATH={{ github_runner_cluster_native_config_dir }}/k3s.yaml +{% else %} +FLEET_REGRADE_ACTUATOR=docker +NODE_CONTAINER={{ github_runner_cluster_node_container }} KUBECTL="docker exec -i {{ github_runner_cluster_node_container }} kubectl" KUBECONFIG_PATH=/etc/rancher/k3s/k3s.yaml -{# Another server's API, never this one's: at etcd-only this server has none, and its own answers "apiserver disabled" on some servers. The first other server in inventory order, by the address the servers' certificates carry. #} -KUBE_SERVER=https://{{ github_runner_cluster_topology.hosts[github_runner_cluster_topology.servers | reject('equalto', inventory_hostname) | first].address }}:{{ github_runner_cluster_api_port }} +{% endif %} +{# The API servers the service may reach the cluster through, never this one's: at etcd-only this server has none, and its own answers "apiserver disabled" on some servers. Every other server, then every other host that may become one (github_runner_cluster_voter_eligible), each in inventory order and by the address the servers' certificates carry; the core uses the first whose /readyz answers, since the servers can change at run time. #} +{% set github_runner_cluster_regrade_others = github_runner_cluster_topology.hosts | dict2items | rejectattr('key', 'equalto', inventory_hostname) | list %} +{% set github_runner_cluster_regrade_candidates = (github_runner_cluster_regrade_others | selectattr('value.role', 'equalto', 'server') | list) + + (github_runner_cluster_regrade_others | rejectattr('value.role', 'equalto', 'server') | selectattr('value.eligible') | list) %} +KUBE_SERVERS="{% for candidate in github_runner_cluster_regrade_candidates %}{{ ' ' if not loop.first else '' }}https://{{ candidate.value.address }}:{{ github_runner_cluster_api_port }}{% endfor %}" diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index 79999b8c..7081a9c7 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -28,6 +28,8 @@ native_host=grtest-native native_image=grtest-native-host native_ip="${subnet_prefix}.20" native_role="" +# Whether the inventory turns memory-pressure regrading on for the native host alone; native_server's last part sets it. +native_regrade=false # The Docker hosts the inventory lists, by index; all of them unless a scenario leaves one out (native_server's second run, where node 3 keeps running as a voter outside the inventory). inventory_nodes="" # Where tc comes from for the role_controller scenario: a throwaway container in a node's network namespace, since the node's own image has none. @@ -255,6 +257,7 @@ EOF echo " ansible_python_interpreter: /usr/bin/python3" echo " github_runner_cluster_node_mode: native" echo " github_runner_cluster_node_role: ${native_role}" + echo " github_runner_cluster_regrade: ${native_regrade}" fi } > "${work}/inventory.yml" } @@ -811,9 +814,9 @@ assert_released() { if regrade_on "$1" cat "$2" | grep -F 'lease could not be released' >/dev/null; then regrade_on "$1" cat "$2"; fail "$(node_name "$1") could not release the Lease"; fi } node_is() { [ "$(node_status "$1" "$2")" = "$3" ]; } -# Whether the regrade service of node $1 can read the cluster's nodes as a step does, with the kubectl and kubeconfig of its own server's container and the API of another server. A recreated container has no kubeconfig until its k3s has written one, some seconds after the container starts, and a step before then is refused for it. +# Whether the regrade service of node $1 can read the cluster's nodes as a step does, with the kubectl and kubeconfig of its own server's container and the API of another server (the first of its candidates, which every other server is up to serve here). A recreated container has no kubeconfig until its k3s has written one, some seconds after the container starts, and a step before then is refused for it. # shellcheck disable=SC2016 # The expansions are for the shell in the service container, which sources the service's own settings and splits KUBECTL, a command of several words, on purpose. -can_list_nodes() { regrade_on "$1" sh -c '. /usr/local/lib/fleet-regrade/fleet-regrade.env && $KUBECTL --kubeconfig "$KUBECONFIG_PATH" --server "$KUBE_SERVER" get nodes' >/dev/null 2>&1; } +can_list_nodes() { regrade_on "$1" sh -c '. /usr/local/lib/fleet-regrade/fleet-regrade.env && $KUBECTL --kubeconfig "$KUBECONFIG_PATH" --server "${KUBE_SERVERS%% *}" get nodes' >/dev/null 2>&1; } # The numbers of the nodes whose regrade service reports a level below full. A read that gives no level, or unknown, is not a step: a test node's k3s container can restart by itself under the strain, and its status then reads unknown or fails for a moment, which says nothing about whether the server stepped. stepped_nodes() { @@ -1114,12 +1117,16 @@ scenario_native_server() { # Restart=always must not start the agent unit again, whose condition now fails. native_exec systemctl start k3s-agent || true native_unit_is k3s-agent inactive || fail "k3s-agent started on a server" + # And the skipped start must leave the server's own runtime directory alone, which holds the resolver file its kubelet reads. + native_exec test -s /run/github-runner-k3s-server/resolv.conf 0' >/dev/null; then fail "a node was annotated for removal"; fi } +# --- Memory-pressure regrading of the native server (native_server's last part) --- + +# The level the native server's fleet-regrade reports, read with its own command, as an operator would. +native_level() { native_exec fleet-regrade status /dev/null | sed -n 's/^level: \([^ ]*\) .*/\1/p'; } +native_at_level() { [ "$(native_level)" = "$1" ]; } +native_node_is() { [ "$(kubectl_on 1 get node "$native_host" --no-headers 2>/dev/null | awk '{print $2}')" = "$1" ]; } +native_level_label() { kubectl_on 1 get node "$native_host" -o jsonpath='{.metadata.labels.github-runner\.exadev\.com/regrade-level}'; } +# The keys the regrading drop-in sets, sorted, one space after each; a comment is not a key. +native_dropin_keys() { native_exec sh -c "grep -v '^#' /etc/rancher/k3s/server.yaml.d/50-regrade.yaml | cut -d: -f1 | sort | tr '\n' ' '" /dev/null; } +native_member_ids() { etcd_members | awk -F', ' '{print $1}' | sort | tr '\n' ' '; } +# Whether the member list names the native host's member as started and not a learner. +native_voter_started() { etcd_members | grep -F ", ${native_host}-" | grep -E ', started, .*, false$' >/dev/null; } +# The fleet-regrade unit's log, for its steps and for a failure. +native_regrade_log() { native_exec journalctl -u fleet-regrade --no-pager --output cat /dev/null; } +grtest_daemonset_ready() { [ "$(kubectl_on 1 get daemonset grtest-native-pod -o 'jsonpath={.status.numberReady}' 2>/dev/null)" = 1 ]; } +# Whether the etcd of the native host's k3s is serving in the unit's current run, as the actuator reads it. +native_etcd_serving() { + # shellcheck disable=SC2016 # The expansions are for the shell in the native host. + native_exec sh -c 'journalctl --no-pager --output cat "_SYSTEMD_INVOCATION_ID=$(systemctl show --property InvocationID --value k3s)" | grep -F "ETCD server is now running" >/dev/null' /dev/null || fail "the native settings do not name the systemd actuator" + if native_exec grep -F "${native_host}." /usr/local/lib/github-runner-k3s/regrade/fleet-regrade.env > /etc/grtest-pressure.env + printf "some avg10=0.00 avg60=0.00 avg300=50.00 total=1\n" > /etc/grtest-psi && printf "MemTotal: 1000000 kB\nMemAvailable: 1000 kB\n" > /etc/grtest-meminfo + mkdir -p /etc/systemd/system/fleet-regrade.service.d + printf "[Service]\nEnvironment=FLEET_REGRADE_ENV=/etc/grtest-pressure.env PSI_FILE=/etc/grtest-psi MEMINFO_FILE=/etc/grtest-meminfo\n" > /etc/systemd/system/fleet-regrade.service.d/grtest.conf + systemctl daemon-reload && systemctl restart fleet-regrade' control-plane' >/dev/null || { native_regrade_log; fail "the watcher did not step down through control-plane first"; } + wait_for "the Lease to be released after ${native_host}'s step down" 120 lease_free + native_regrade_log + + log "At etcd-only ${native_host} keeps its seat and its vote, runs all but etcd off, and runs no pod" + native_at_level etcd-only || fail "${native_host} does not read etcd-only" + [ "$(native_dropin_keys)" = "$native_etcd_only_keys" ] || fail "the drop-in does not turn all but etcd off: $(native_dropin_keys)" + [ "$(native_exec cat /run/github-runner-k3s-server/regrade-level /dev/null || fail "the actuator did not say it stopped the pods" + + log "The stepped-down native server still votes: the cluster takes writes with a Docker server stopped" + docker stop "$(container 3)" >/dev/null + wait_for "a write through $(node_name 1) with $(node_name 3) stopped" 180 write_marker + docker start "$(container 3)" >/dev/null + wait_for "$(node_name 3) to be Ready again" 600 node_is 3 3 Ready + + log "The level stays across a restart of the server unit by hand" + native_exec systemctl restart k3s /dev/null + docker start "$native_host" >/dev/null + wait_for "systemd to be running in ${native_host} again" 120 native_systemd_up + wait_for "${native_host}'s etcd to serve again after the host started" 300 native_etcd_serving + wait_for "${native_host} to read etcd-only after the host started" 120 native_at_level etcd-only + [ "$(native_dropin_keys)" = "$native_etcd_only_keys" ] || fail "the host's start lost the drop-in" + [ "$(native_shims)" = 0 ] || fail "a pod came back on ${native_host} after the host started at etcd-only" + wait_for "${native_host}'s member to be a started voter again" 300 native_voter_started + [ "$(native_member_ids)" = "$ids" ] || fail "the etcd members changed across the restarts: $(native_member_ids) (was ${ids})" + native_unit_is fleet-regrade active || fail "fleet-regrade did not start with the host" + + log "The pressure passes: the watcher steps back up by itself, one level at a time, to the same node" + native_exec sh -c 'printf "some avg10=0.00 avg60=0.00 avg300=0.00 total=1\n" > /etc/grtest-psi && printf "MemTotal: 1000000 kB\nMemAvailable: 900000 kB\n" > /etc/grtest-meminfo' control-plane' >/dev/null || { native_regrade_log; fail "the watcher did not step up through control-plane first"; } + native_regrade_log + wait_for "${native_host} to be Ready and schedulable" 300 native_node_is Ready + [ -z "$(native_level_label)" ] || fail "the level label is still on the Node at full" + [ "$(native_dropin_keys)" = "" ] || fail "the drop-in still turns something off at full: $(native_dropin_keys)" + [ "$(native_member_ids)" = "$ids" ] || fail "the etcd members changed after stepping up" + [ "$(native_exec sha256sum /etc/rancher/node/password /dev/null + + log "With the server at full, turning regrading off removes the unit and leaves the server running" + native_exec sh -c 'rm -rf /etc/systemd/system/fleet-regrade.service.d /etc/grtest-pressure.env /etc/grtest-psi /etc/grtest-meminfo && systemctl daemon-reload' &1) || { echo "$output"; fail "turning regrading off failed"; } + [ "$before" = "$(native_server_started)" ] || fail "turning regrading off restarted ${native_host}'s server" + if native_exec test -e /etc/systemd/system/fleet-regrade.service /dev/null || echo 0) + printf '2026-01-01T00:%02d:%02dZ\n' $((ticks / 60)) $((ticks % 60)) +else + exec /bin/date "$@" +fi diff --git a/tests/regrade/fake-journalctl.sh b/tests/regrade/fake-journalctl.sh new file mode 100755 index 00000000..9b6244aa --- /dev/null +++ b/tests/regrade/fake-journalctl.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# A stand-in for journalctl for the systemd actuator's tests: prints what one run of the unit logged, from the directory FAKE_HOST that fake-systemctl.sh keeps, for the only query the actuator makes, a run's lines by its invocation id with no metadata. +set -u + +host=${FAKE_HOST:?FAKE_HOST must be set} +echo "$*" >> "$host/calls" +[ "${1:-} ${2:-} ${3:-}" = "--no-pager --output cat" ] || { echo "unexpected journalctl call: $*" >&2; exit 2; } +case ${4:-} in + _SYSTEMD_INVOCATION_ID=?*) cat "$host/journal-${4#_SYSTEMD_INVOCATION_ID=}" 2> /dev/null || true ;; + *) echo "unexpected journalctl call: $*" >&2; exit 2 ;; +esac diff --git a/tests/regrade/fake-k3s.sh b/tests/regrade/fake-k3s.sh new file mode 100755 index 00000000..5196565b --- /dev/null +++ b/tests/regrade/fake-k3s.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# A stand-in for k3s for the systemd actuator's tests: `k3s kubectl` answers what the actuator asks about the node and its lease from the directory FAKE_HOST that fake-systemctl.sh keeps, and passes every other kubectl call to the kubectl on the PATH, which the test makes fake-kubectl.sh. Each call is recorded in FAKE_HOST/calls beside the unit's own, as `kubectl` and its arguments without the options that say where, so the test can read the order of the two. +set -u + +host=${FAKE_HOST:?FAKE_HOST must be set} +[ "${1:-}" = kubectl ] || { echo "unexpected k3s call: $*" >&2; exit 2; } +shift +original=("$@") +args=() +while [ $# -gt 0 ]; do + case $1 in + --kubeconfig | --server) shift 2 ;; + *) args+=("$1"); shift ;; + esac +done +echo "kubectl ${args[*]}" >> "$host/calls" +case "${args[0]:-} ${args[1]:-}" in + "get node") cat "$host/node-ready" ;; + "get lease") if [ "${args[2]:-}" = --namespace ]; then cat "$host/lease-renewed"; else exec kubectl "${original[@]}"; fi ;; + *) exec kubectl "${original[@]}" ;; +esac diff --git a/tests/regrade/fake-kubectl.sh b/tests/regrade/fake-kubectl.sh index 327f9968..f8c15e91 100755 --- a/tests/regrade/fake-kubectl.sh +++ b/tests/regrade/fake-kubectl.sh @@ -1,16 +1,25 @@ #!/usr/bin/env bash -# A stand-in for kubectl for the regrade tests, keeping a cluster in the directory FAKE_DIR: lease.json (the Lease, absent when there is none) and nodes.json (what listing the etcd-voting nodes returns). It understands only what the core asks of it, answers as the real one does for the cases that matter (a Lease that is not found, one that already exists, a replacement of a Lease that has changed), and records every call in FAKE_DIR/calls. The FAKE_* variables make a call fail: FAKE_LEASE_READ_ERROR, FAKE_NODES_ERROR, FAKE_CREATE_RACE, FAKE_REPLACE_CONFLICT. +# A stand-in for kubectl for the regrade tests, keeping a cluster in the directory FAKE_DIR: lease.json (the Lease, absent when there is none), nodes.json (what listing the etcd-voting nodes returns) and unready (the API servers, one URL per line, `default` for the kubeconfig's own, whose /readyz does not answer). It understands only what the core asks of it, answers as the real one does for the cases that matter (a Lease that is not found, one that already exists, a replacement of a Lease that has changed), and records every call in FAKE_DIR/calls, without the options that say where and how long, and the API server each call went to (its --server, or `default`) in FAKE_DIR/servers, a line for each call. The FAKE_* variables make a call fail: FAKE_LEASE_READ_ERROR, FAKE_NODES_ERROR, FAKE_CREATE_RACE, FAKE_REPLACE_CONFLICT. set -u dir=${FAKE_DIR:?FAKE_DIR must be set} +server=default args=() while [ $# -gt 0 ]; do case $1 in - --kubeconfig | --server) shift 2 ;; + --kubeconfig) shift 2 ;; + --server) server=$2; shift 2 ;; + --request-timeout=*) shift ;; *) args+=("$1"); shift ;; esac done echo "${args[*]}" >> "$dir/calls" +echo "$server ${args[*]}" >> "$dir/servers" + +if [ "${args[*]}" = "get --raw=/readyz" ] && grep -Fqx -- "$server" "$dir/unready" 2> /dev/null; then + echo "Unable to connect to the server: dial tcp: i/o timeout" >&2 + exit 1 +fi lease="$dir/lease.json" case "${args[0]:-} ${args[1]:-}" in diff --git a/tests/regrade/fake-systemctl.sh b/tests/regrade/fake-systemctl.sh new file mode 100755 index 00000000..dbb642d4 --- /dev/null +++ b/tests/regrade/fake-systemctl.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# A stand-in for systemctl for the systemd actuator's tests, keeping one unit, the k3s server, in the directory FAKE_HOST: `active` (present while it runs), `invocation` (the id of its current run, a number that rises with each start), `journal-` (what that run logged), `ticks` (the fake clock that fake-date.sh reads), and `lease-renewed` and `node-ready` (what kubectl reports about the node). A start or restart does what systemd does with the real unit's ExecStartPre: it runs FAKE_PRESTART, the command the test gives it (the real write-regrade-level.sh with the unit's three paths), then logs that etcd is serving and has the kubelet renew its lease a second later. A stop removes the unit's runtime directory, FAKE_RUNTIME_DIR, as RuntimeDirectory= does. Each start also records how many processes were running whose command starts with FAKE_SHIM_PREFIX, so the test can tell what ran across it. Every call is recorded in FAKE_HOST/calls. The FAKE_* variables make a restart go wrong: FAKE_RESTART_FAILS, FAKE_NO_ETCD (etcd never serves), FAKE_NODE_STALE (the kubelet never renews its lease), FAKE_NODE_NOT_READY. +set -u + +host=${FAKE_HOST:?FAKE_HOST must be set} +echo "$*" >> "$host/calls" + +# The fake clock, advanced by $1 seconds, as an RFC 3339 time. +clock() { + local ticks + ticks=$(($(cat "$host/ticks" 2> /dev/null || echo 0) + $1)) + echo "$ticks" > "$host/ticks" + printf '2026-01-01T00:%02d:%02dZ\n' $((ticks / 60)) $((ticks % 60)) +} + +start() { + local invocation processes + [ -z "${FAKE_RESTART_FAILS:-}" ] || exit 1 + # The listing is read whole first, so the search does not count itself. + processes=$(ps -e -o args=) + echo "running shims: $(grep -c "^${FAKE_SHIM_PREFIX:?FAKE_SHIM_PREFIX must be set}" <<< "$processes" || true)" >> "$host/calls" + sh -c "${FAKE_PRESTART:?FAKE_PRESTART must be set}" || exit 1 + invocation=$(($(cat "$host/invocation" 2> /dev/null || echo 0) + 1)) + echo "$invocation" > "$host/invocation" + touch "$host/active" + : > "$host/journal-$invocation" + clock 10 > /dev/null + [ -n "${FAKE_NO_ETCD:-}" ] || echo "ETCD server is now running" >> "$host/journal-$invocation" + echo "k3s is up and running" >> "$host/journal-$invocation" + [ -n "${FAKE_NODE_STALE:-}" ] || clock 1 > "$host/lease-renewed" + if [ -z "${FAKE_NODE_NOT_READY:-}" ]; then echo True > "$host/node-ready"; else echo Unknown > "$host/node-ready"; fi +} + +case "${1:-} ${2:-}" in + "is-active --quiet") [ -f "$host/active" ] ;; + "restart k3s" | "start k3s") start ;; + "stop k3s") + rm -f "$host/active" + rm -rf "${FAKE_RUNTIME_DIR:?FAKE_RUNTIME_DIR must be set}" + ;; + "show --property") + [ "${3:-} ${4:-} ${5:-}" = "InvocationID --value k3s" ] || { echo "unexpected systemctl call: $*" >&2; exit 2; } + cat "$host/invocation" 2> /dev/null || echo + ;; + *) + echo "unexpected systemctl call: $*" >&2 + exit 2 + ;; +esac diff --git a/tests/regrade/regrade-docker.sh b/tests/regrade/regrade-docker.sh index 2b60a871..25b6cf95 100644 --- a/tests/regrade/regrade-docker.sh +++ b/tests/regrade/regrade-docker.sh @@ -1,10 +1,11 @@ -# Tests for the Docker actuator of fleet-regrade (files/regrade/fleet-regrade-docker.sh): which level the node is at, and how a level is entered. Run as `bash tests/regrade/regrade-docker.sh `. docker is a stand-in that keeps one container in a directory (tests/regrade/fake-docker.sh) and kubectl is the lease test's (tests/regrade/fake-kubectl.sh), so nothing here touches a container or a cluster. The core calls the functions defined here by name, and reads the variables set here, so shellcheck sees them as unused. +# Tests for the Docker actuator of fleet-regrade (files/regrade/fleet-regrade-docker.sh, over the part both k3s actuators share, fleet-regrade-k3s.sh): which level the node is at, and how a level is entered. Run as `bash tests/regrade/regrade-docker.sh `. docker is a stand-in that keeps one container in a directory (tests/regrade/fake-docker.sh) and kubectl is the lease test's (tests/regrade/fake-kubectl.sh), so nothing here touches a container or a cluster. The core calls the functions defined here by name, and reads the variables set here, so shellcheck sees them as unused. # shellcheck disable=SC2329,SC2034 set -euo pipefail -core=${1:?usage: regrade-docker.sh } -actuator=${2:?usage: regrade-docker.sh } +core=${1:?usage: regrade-docker.sh } +k3s_actuator=${2:?usage: regrade-docker.sh } +actuator=${3:?usage: regrade-docker.sh } here=$(cd "$(dirname "$0")" && pwd) work=$(mktemp -d) trap 'rm -rf "$work"' EXIT @@ -27,7 +28,7 @@ CONTROL_PLANE_TAINT=node-role.kubernetes.io/control-plane=true:NoSchedule NODE_CONTAINER=test-k3s KUBECTL="docker exec -i test-k3s kubectl" KUBECONFIG_PATH=/etc/rancher/k3s/k3s.yaml -KUBE_SERVER=https://other.example.test:6443 +KUBE_SERVERS="https://other.example.test:6443 https://another.example.test:6443" LEVEL_LABEL=example.test/regrade-level ENV export FLEET_REGRADE_ENV=$work/env LEVEL_FILE=$work/node/level-file STARTED_FILE=$work/node/started-file @@ -37,6 +38,8 @@ export FLEET_REGRADE_ENV=$work/env LEVEL_FILE=$work/node/level-file STARTED_FILE # shellcheck source=/dev/null . "$core" # shellcheck source=/dev/null +. "$k3s_actuator" +# shellcheck source=/dev/null . "$actuator" POLL_SECONDS=1 diff --git a/tests/regrade/regrade-lease.sh b/tests/regrade/regrade-lease.sh index fead9340..4921b5e6 100644 --- a/tests/regrade/regrade-lease.sh +++ b/tests/regrade/regrade-lease.sh @@ -26,8 +26,8 @@ NOW=1000000000 now_epoch() { echo "$NOW"; } lease="$FAKE_DIR/lease.json" reset_cluster() { - rm -f "$lease" "$FAKE_DIR/calls" - unset FAKE_LEASE_READ_ERROR FAKE_NODES_ERROR FAKE_CREATE_RACE FAKE_REPLACE_CONFLICT + rm -f "$lease" "$FAKE_DIR/calls" "$FAKE_DIR/servers" "$FAKE_DIR/unready" + unset FAKE_LEASE_READ_ERROR FAKE_NODES_ERROR FAKE_CREATE_RACE FAKE_REPLACE_CONFLICT KUBE_SERVERS KUBE_SERVER nodes_json True True } # The voters: peer-a and peer-b with the given Ready condition statuses (True, False or Unknown), and this node, Ready. @@ -123,7 +123,7 @@ reset_cluster peers_ready check "peers: only etcd-voting nodes are asked about" grep -qF -- "--selector node-role.kubernetes.io/etcd=true" "$FAKE_DIR/calls" -# Coordinated: the peers first, then the lease, then the command, then the release. +# Coordinated: an API server that answers first, then the peers, then the lease, then the command, then the release. ran='' step_recorder() { ran="$ran step"; } failing_step() { ran="$ran failing"; return 3; } @@ -153,6 +153,72 @@ run_coordinated step_recorder || true check "lease held elsewhere: the command does not run" test "$ran" = "" check "lease held elsewhere: it is left as it was" test "$(holder)" = other-node +# Which API server a step goes through: the first of the candidates whose /readyz answers, never the kubeconfig's own (this node's) while there are candidates, chosen again for the release and whenever a step waits for the API. +first_api=https://first.example.test:6443 +second_api=https://second.example.test:6443 +unready() { printf '%s\n' "$@" > "$FAKE_DIR/unready"; } +# The servers the calls other than the readiness probes went to, each once, in the order first used. +used_servers() { grep -v ' get --raw=/readyz$' "$FAKE_DIR/servers" | cut -d' ' -f1 | awk '!seen[$0]++' | tr '\n' ' '; } +to_own_server() { grep -q '^default ' "$FAKE_DIR/servers"; } + +reset_cluster +KUBE_SERVERS="$first_api $second_api" +run_coordinated step_recorder +check "candidates: the step runs through the first that answers" test "$ran" = " step" +check "candidates: every call goes to it" test "$(used_servers)" = "$first_api " +check_not "candidates: nothing goes to this node's own server" to_own_server + +reset_cluster +KUBE_SERVERS="$first_api $second_api" +unready "$first_api" +run_coordinated step_recorder +check "a candidate that does not answer is passed over" test "$ran" = " step" +check "and every call goes to the next" test "$(used_servers)" = "$second_api " +check "the lease is given back through it" test "$(holder)" = "" + +reset_cluster +KUBE_SERVERS="$first_api $second_api" +unready "$first_api" "$second_api" +ran='' +refused_output=$(coordinated step_recorder 2>&1) && refused_status=0 || refused_status=$? +check_not "no candidate answers: the step is refused" test "$refused_status" = 0 +check "and the command does not run" test "$ran" = "" +check "and no lease is taken" test ! -e "$lease" +check "and the refusal names the candidates" grep -Fq "no API server answers (tried: $first_api $second_api)" <<< "$refused_output" +check_not "and this node's own server is never tried in their place" to_own_server + +reset_cluster +KUBE_SERVERS="$first_api $second_api" +first_goes_away() { ran="$ran step"; unready "$first_api"; } +run_coordinated first_goes_away +check "a server that goes away during a step: the release goes through the next" test "$(holder)" = "" +check "and the calls after it went there" test "$(used_servers)" = "$first_api $second_api " + +reset_cluster +run_coordinated step_recorder +check "no candidates: the kubeconfig's own server is used" test "$(used_servers)" = "default " + +reset_cluster +unready default +run_coordinated step_recorder || true +check "no candidates and the kubeconfig's own does not answer: the step is refused" test "$ran" = "" + +reset_cluster +KUBE_SERVERS="$first_api $second_api" +unready "$first_api" +OPERATION_TIMEOUT_SECONDS=1 wait_for_api +# shellcheck disable=SC2153 # KUBE_SERVER is the core's, set by select_api_server. +check "waiting for the API settles on a candidate that answers" test "$KUBE_SERVER" = "$second_api" +unready "$first_api" "$second_api" +waits_out() { (OPERATION_TIMEOUT_SECONDS=1 wait_for_api) > /dev/null 2>&1; } +check_not "waiting for the API gives up when none answers" waits_out +gave_up_text() { + local output + output=$(OPERATION_TIMEOUT_SECONDS=1 wait_for_api 2>&1) && return 1 + grep -Fq "No API server became ready within 1s (tried: $first_api $second_api)" <<< "$output" +} +check "and says which it tried" gave_up_text + # Two nodes never step at once: while one holds the lease, a second asking as another node is refused. reset_cluster acquire_lease > /dev/null diff --git a/tests/regrade/regrade-systemd.sh b/tests/regrade/regrade-systemd.sh new file mode 100644 index 00000000..f6934b59 --- /dev/null +++ b/tests/regrade/regrade-systemd.sh @@ -0,0 +1,296 @@ +# Tests for the systemd actuator of fleet-regrade (files/regrade/fleet-regrade-systemd.sh, over the part both k3s actuators share, fleet-regrade-k3s.sh) and the server unit's ExecStartPre that it relies on (files/native/write-regrade-level.sh): which level the node is at, how a level is entered and left in each direction, the waits after a restart, and the pods stopped at etcd-only. Run as `bash tests/regrade/regrade-systemd.sh `. systemctl, journalctl, date and k3s are stand-ins that keep one unit in a directory (tests/regrade/fake-systemctl.sh and the files beside it) and kubectl is the lease test's (tests/regrade/fake-kubectl.sh), so nothing here touches a unit or a cluster; the pods' containerd shims are real processes of this test's own, started from a stand-in data directory, and the processes it stops are only those. The core calls the functions defined here by name, and reads the variables set here, so shellcheck sees them as unused. + +# shellcheck disable=SC2329,SC2034 +set -euo pipefail + +usage="usage: regrade-systemd.sh " +core=${1:?$usage} +k3s_actuator=${2:?$usage} +actuator=${3:?$usage} +prestart=${4:?$usage} +here=$(cd "$(dirname "$0")" && pwd) +work=$(mktemp -d) + +# The test's own shims, and the containers they run, stopped on the way out whatever the result. +cleanup() { + local pids + pids=$(cat "$work/spawned" 2> /dev/null || true) + # shellcheck disable=SC2046,SC2086 # One process id per word, on purpose. + [ -z "$pids" ] || kill -KILL $(process_tree $pids) 2> /dev/null || true + rm -rf "$work" +} +trap cleanup EXIT + +mkdir -p "$work/bin" "$work/host" "$work/cluster" "$work/data" "$work/config/server.yaml.d" "$work/run" +cp "$here/fake-systemctl.sh" "$work/bin/systemctl" +cp "$here/fake-journalctl.sh" "$work/bin/journalctl" +cp "$here/fake-date.sh" "$work/bin/date" +cp "$here/fake-k3s.sh" "$work/bin/k3s" +cp "$here/fake-kubectl.sh" "$work/bin/kubectl" +export PATH="$work/bin:$PATH" FAKE_HOST=$work/host FAKE_DIR=$work/cluster FAKE_RUNTIME_DIR=$work/run FAKE_SHIM_PREFIX=$work/data/data/ + +level_file=$work/data/regrade-level +dropin=$work/config/server.yaml.d/50-regrade.yaml +started_file=$work/run/regrade-level +# What the unit's ExecStartPre runs, as templates/native/k3s.service.j2 writes it. +export FAKE_PRESTART="$prestart $level_file $dropin $started_file" + +cat > "$work/env" < "$level_file" + systemctl start k3s + : > "$FAKE_HOST/calls" + printf '{"items":[{"metadata":{"name":"peer-a"},"status":{"conditions":[{"type":"Ready","status":"True"}]}},{"metadata":{"name":"testnode"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}\n' > "$FAKE_DIR/nodes.json" +} +restarts() { grep -c '^restart k3s$' "$FAKE_HOST/calls" || true; } +# Whether the first call matching $1 in the file $3 (the unit's calls by default) was made before the last one matching $2. +call_before() { + local first last calls=${3:-$FAKE_HOST/calls} + first=$(grep -n -- "$1" "$calls" | head -n 1 | cut -d: -f1) + last=$(grep -n -- "$2" "$calls" | tail -n 1 | cut -d: -f1) + [ -n "$first" ] && [ -n "$last" ] && [ "$first" -lt "$last" ] +} +requested() { cat "$level_file"; } +started_at_level() { cat "$started_file"; } +# The keys the drop-in sets, sorted, one space after each; comments are no keys. +dropin_keys() { grep -v '^#' "$dropin" | cut -d: -f1 | sort | tr '\n' ' '; } +etcd_only_keys="disable-agent disable-apiserver disable-controller-manager disable-scheduler " +no_leftovers() { [ -z "$(find "$work/data" "$work/config" "$work/run" -name '*.new' -o -name '*.tmp')" ]; } + +check "the levels, most capable first" test "$(levels | tr '\n' ' ')" = "full control-plane etcd-only " + +# The unit's ExecStartPre: what k3s is configured to start as, and the record of it. +unit_at full +rm "$level_file" +systemctl restart k3s +check "a server never stepped down starts at full" test "$(started_at_level)" = full +check "with a drop-in that sets nothing" test "$(dropin_keys)" = "" +for level in full control-plane; do + unit_at "$level" + check "$level starts k3s whole" test "$(dropin_keys)" = "" + check "$level is recorded as started" test "$(started_at_level)" = "$level" +done +unit_at etcd-only +check "etcd-only turns off all but etcd" test "$(dropin_keys)" = "$etcd_only_keys" +check "the drop-in is valid YAML that turns each one on" test "$(grep -c ': true$' "$dropin")" = 4 +check "etcd-only is recorded as started" test "$(started_at_level)" = etcd-only +unit_at full +echo unheard-of > "$level_file" +systemctl restart k3s +check "a level that is no level starts k3s whole" test "$(dropin_keys)" = "" +check "and is recorded as read, for the actuator to call unknown" test "$(current_level)" = unknown +check "the start writes each file in one rename" no_leftovers + +# Which level the node is at is what the unit's start recorded for the running k3s. +for level in full control-plane etcd-only; do + unit_at "$level" + check "current_level reports $level" test "$(current_level)" = "$level" +done +unit_at full +rm "$started_file" "$level_file" +check "current_level is full for a server started from before regrading" test "$(current_level)" = full +unit_at full +rm "$started_file" +check "current_level is unknown with a requested level but no record of what started" test "$(current_level)" = unknown +unit_at etcd-only +systemctl stop k3s +check "current_level is unknown while the unit is not running" test "$(current_level)" = unknown +unit_at etcd-only +# A unit that is starting again is not active until its ExecStartPre has run, and its runtime directory, kept across a restart, still holds the record of the run before. +rm "$FAKE_HOST/active" +check "current_level is unknown while the unit is not active, whatever record its runtime directory still holds" test "$(current_level)" = unknown +unit_at full +echo etcd-only > "$level_file" +check "current_level reads what runs, not what was last requested" test "$(current_level)" = full + +# Between the levels that run the same k3s nothing restarts: the level is recorded as requested and as running. +unit_at full +apply_level control-plane +check "full to control-plane restarts nothing" test "$(restarts)" = 0 +check "full to control-plane is recorded as requested" test "$(requested)" = control-plane +check "full to control-plane is recorded as running" test "$(started_at_level)" = control-plane +apply_level full +check "control-plane to full restarts nothing" test "$(restarts)" = 0 +check "control-plane to full is recorded" test "$(requested)-$(started_at_level)" = full-full +check "the records are written in one rename" no_leftovers + +# The Node is labelled with the level it is taken to, before the level is entered, and unlabelled on the way back to the first. +unit_at full +apply_level control-plane +check "stepping down labels the Node with the level" grep -Fq "label node testnode example.test/regrade-level=control-plane --overwrite" "$FAKE_DIR/calls" +apply_level full +check "returning to the first level takes the label off" grep -Fq "label node testnode example.test/regrade-level-" "$FAKE_DIR/calls" +label_fails() { (kubectl_() { return 1; }; apply_level control-plane > /dev/null 2>&1); } +check_not "a label that cannot be put on fails the step" label_fails +check "and nothing was recorded" test "$(requested)" = full + +# Entering etcd-only restarts the unit once the level is recorded for its ExecStartPre to read, and waits for etcd in the new run. +unit_at control-plane +apply_level etcd-only +check "control-plane to etcd-only restarts the unit once" test "$(restarts)" = 1 +check "the level is recorded before the restart" test "$(requested)" = etcd-only +check "the new run started at etcd-only" test "$(started_at_level)" = etcd-only +check "with all but etcd turned off" test "$(dropin_keys)" = "$etcd_only_keys" +check "current_level follows" test "$(current_level)" = etcd-only +check "etcd is waited for in the new run's journal" grep -Fqx -- "--no-pager --output cat _SYSTEMD_INVOCATION_ID=2" "$FAKE_HOST/calls" +check "after the restart" call_before "^restart k3s$" "_SYSTEMD_INVOCATION_ID=2" + +# Leaving it restarts the unit and also waits for the node to report Ready on a heartbeat from the new run. +for level in control-plane full; do + unit_at etcd-only + apply_level "$level" + check "etcd-only to $level restarts the unit once" test "$(restarts)" = 1 + check "the new run started at $level" test "$(started_at_level)" = "$level" + check "with k3s whole again" test "$(dropin_keys)" = "" +done +check "the node's heartbeat is read after the restart" call_before "^restart k3s$" "^kubectl get lease --namespace kube-node-lease testnode" + +# The level stays across a restart by hand and across a stop and a start, since the requested level is read at every start. +unit_at control-plane +apply_level etcd-only +systemctl restart k3s +check "a restart by hand keeps etcd-only" test "$(current_level)-$(dropin_keys)" = "etcd-only-$etcd_only_keys" +systemctl stop k3s +systemctl start k3s +check "a stop and a start keep etcd-only" test "$(current_level)-$(dropin_keys)" = "etcd-only-$etcd_only_keys" + +# A restart that did not bring the server back fails the step, and says what is running. +unit_at full +export FAKE_NO_ETCD=1 +no_etcd() { apply_level etcd-only > "$work/out" 2>&1; } +check_not "etcd that never serves fails the step" no_etcd +check "and the failure says so" grep -Fq "etcd did not come back within 2s" "$work/out" +check "and the node is left at the level it restarted at" test "$(current_level)" = etcd-only + +# Waiting for etcd counts only the run since the restart: the line the run before it logged does not satisfy it. +unit_at control-plane +check "the run before the restart did log that etcd was serving" grep -Fq "ETCD server is now running" "$FAKE_HOST/journal-1" +export FAKE_NO_ETCD=1 +stale_journal() { apply_level etcd-only > "$work/out" 2>&1; } +check_not "etcd's line from the run before the restart does not count" stale_journal + +unit_at etcd-only +export FAKE_NODE_STALE=1 +node_stale() { apply_level full > "$work/out" 2>&1; } +check_not "a kubelet that never renews its lease fails the step" node_stale +check "and the failure says so" grep -Fq "the node did not report Ready within 2s" "$work/out" + +unit_at etcd-only +export FAKE_NODE_NOT_READY=1 +node_not_ready() { apply_level full > "$work/out" 2>&1; } +check_not "a node that is not Ready fails the step" node_not_ready + +unit_at full +export FAKE_RESTART_FAILS=1 +restart_fails() { apply_level etcd-only > "$work/out" 2>&1; } +check_not "a restart systemd refuses fails the step" restart_fails + +# A node that is not running at a known level is not touched. +unit_at full +systemctl stop k3s +refused() { apply_level etcd-only > "$work/out" 2>&1; } +check_not "a level cannot be applied to a server whose unit is not running" refused +check "and nothing is recorded" test "$(requested)" = full +check "and the failure says why" grep -Fq "not running at a known level" "$work/out" + +# The pods that outlive k3s's agent: shims started from the k3s data directory's shim binary, each running a container process, and one shim of the same name from elsewhere, which is not k3s's and is left alone. +mkdir -p "$work/data/data/0123abcd/bin" "$work/elsewhere/bin" +ln -sf "$(command -v bash)" "$work/data/data/0123abcd/bin/containerd-shim-runc-v2" +ln -sf "$(command -v bash)" "$work/elsewhere/bin/containerd-shim-runc-v2" +# Starts a shim from $1 whose container is a sleep of $2 seconds, a duration the test uses as that process's name, and returns once the container runs, which the shim says by writing the container's pid to a file named for it. The shim is left to the system's init, as a real one is, and its pid is kept for the clean-up. +spawn_shim() { + local ready=$work/ready-$2 shim + rm -f "$ready" + shim=$( ("$1" -c "sleep $2 & echo \$! > $ready; wait" > /dev/null 2>&1 < /dev/null & echo $!)) + echo "$shim" >> "$work/spawned" + until [ -s "$ready" ]; do + kill -0 "$shim" 2> /dev/null || die "the stand-in shim for $2 exited before its container started" + sleep 1 + done +} +# Read whole before it is searched: grep -q stops reading at the first match, which under pipefail would fail the listing. +running() { + local processes + processes=$(ps -e -o args=) + grep -Eq "^(/bin/)?sleep $1\$" <<< "$processes" +} +spawn_pods() { + spawn_shim "$work/data/data/0123abcd/bin/containerd-shim-runc-v2" 7001 + spawn_shim "$work/data/data/0123abcd/bin/containerd-shim-runc-v2" 7002 + spawn_shim "$work/elsewhere/bin/containerd-shim-runc-v2" 7003 +} +spawn_pods +check "the stand-in shims are found as k3s's" test "$(pod_shims | wc -l | tr -d ' ')" = 2 +unit_at full +apply_level control-plane +check "a step that keeps the agent stops no pod" running 7001 +unit_at control-plane +apply_level etcd-only > "$work/out" +check "entering etcd-only stops k3s's shims" test -z "$(pod_shims)" +check_not "and the container of the first" running 7001 +check_not "and the container of the second" running 7002 +check "and leaves a shim that is not k3s's alone" running 7003 +check "and says which it stopped" grep -Fq "stopping the pods left without a kubelet at etcd-only" "$work/out" +check "after the restart that left the agent off, across which they still ran" grep -Fqx "running shims: 2" "$FAKE_HOST/calls" +spawn_shim "$work/data/data/0123abcd/bin/containerd-shim-runc-v2" 7004 +apply_level control-plane +check "leaving etcd-only stops nothing" running 7004 +apply_level full +# A shim that does not go fails the step. +unit_at control-plane +kill_ignored() { (kill() { :; }; apply_level etcd-only > "$work/out" 2>&1); } +check_not "a shim that outlives the signal fails the step" kill_ignored +check "and the failure names it" grep -Fq "containerd shims still running after 2s" "$work/out" + +# Through the core: down and up by hand, with the coordination real (the lease and the peers' readiness), the workloads shed before a level is applied and restored only on reaching the first level, and every call to the cluster through another server. +unit_at full +cmd_down > /dev/null +check "down by hand reaches control-plane" test "$(current_level)" = control-plane +cmd_down > /dev/null +check "down by hand reaches etcd-only" test "$(current_level)" = etcd-only +check "the drain came before the restart that switched the API server off" call_before "^kubectl drain " "^restart k3s$" +cmd_up > /dev/null +check "up by hand leaves etcd-only for control-plane" test "$(current_level)" = control-plane +check_not "which does not yet restore the workloads" grep -Fq "uncordon" "$FAKE_DIR/calls" +cmd_up > /dev/null +check "up by hand reaches full" test "$(current_level)" = full +check "which restores the workloads" test "$(grep -c 'uncordon testnode' "$FAKE_DIR/calls")" = 1 +check_not "and nothing went to this server's own API" grep -q '^default ' "$FAKE_DIR/servers" + +finish diff --git a/tests/regrade/render_env.yml b/tests/regrade/render_env.yml index 7ef4212a..cb6991f0 100644 --- a/tests/regrade/render_env.yml +++ b/tests/regrade/render_env.yml @@ -1,5 +1,5 @@ --- -# Renders the cluster role's regrade.env.j2 for one server of a three-server group (the one the inventory names, which the test gives as the only host), with the role's defaults and placeholder values for what the template does not look at, writing it to env_output, for tests/unit/test_cluster_regrade.py. Role variables are passed as extra variables, which outrank the defaults loaded here. +# Renders the cluster role's regrade.env.j2 for one server of a three-server group with a standby (an agent marked eligible) and an agent (the one the inventory names, which the test gives as the only host), with the role's defaults and placeholder values for what the template does not look at, writing it to env_output, for tests/unit/test_cluster_regrade.py. Role variables are passed as extra variables, which outrank the defaults loaded here. - name: Render the regrade service's settings hosts: all connection: local @@ -12,9 +12,11 @@ github_runner_cluster_topology: servers: [node1, node2, node3] hosts: - node1: {address: node1.example.internal} - node2: {address: node2.example.internal} - node3: {address: node3.example.internal} + node1: {address: node1.example.internal, role: server, eligible: false} + node2: {address: node2.example.internal, role: server, eligible: false} + node3: {address: node3.example.internal, role: server, eligible: false} + standby: {address: standby.example.internal, role: agent, eligible: true} + agent: {address: agent.example.internal, role: agent, eligible: false} tasks: - name: Render regrade.env.j2 ansible.builtin.copy: diff --git a/tests/unit/native_stand_ins.py b/tests/unit/native_stand_ins.py index 35c6669e..dd3e1144 100644 --- a/tests/unit/native_stand_ins.py +++ b/tests/unit/native_stand_ins.py @@ -1,4 +1,4 @@ -"""Shared stand-ins and a base class for the tests that run the github_runner_cluster role against a native node (test_native_node.py, its agent, and test_native_server.py, its server): stand-ins for systemctl, tailscale, dpkg-query, apt-get, docker and kubectl first on the playbook's PATH, a local server for the k3s release, a k3s data directory laid out as a server leaves it, and the helpers that run the role through tests/native/run.yml, so that no real service, package, tailnet or cluster is touched.""" +"""Shared stand-ins and a base class for the tests that run the github_runner_cluster role against a native node (test_native_node.py, its agent, test_native_server.py, its server, and test_native_regrade.py, its regrading): stand-ins for systemctl, journalctl, tailscale, dpkg-query, apt-get, docker and kubectl first on the playbook's PATH, a local server for the k3s release, a k3s data directory laid out as a server leaves it, and the helpers that run the role through tests/native/run.yml, so that no real service, package, tailnet or cluster is touched.""" from __future__ import annotations @@ -93,6 +93,8 @@ def setUp(self) -> None: "github_runner_cluster_native_config_dir": self.root / "etc" / "rancher" / "k3s", "github_runner_cluster_native_unit_dir": self.root / "etc" / "systemd" / "system", "github_runner_cluster_native_data_dir": self.root / "var" / "lib" / "rancher" / "k3s", + "github_runner_cluster_native_run_dir": self.root / "run", + "github_runner_cluster_native_psi_file": self.root / "proc" / "pressure" / "memory", "github_runner_cluster_native_apt_keyring": self.root / "usr" / "share" / "keyrings" / "tailscale-archive-keyring.gpg", "github_runner_cluster_native_apt_source": self.root / "etc" / "apt" / "sources.list.d" / "tailscale.list", } @@ -101,9 +103,20 @@ def setUp(self) -> None: # Directories every host has already: the role puts files in them and creates no others of the system's. self.paths["github_runner_cluster_native_bin_dir"].mkdir(exist_ok=True) self.paths["github_runner_cluster_native_unit_dir"].mkdir(exist_ok=True) + # A kernel with memory pressure, as every kernel the role supports has unless it was built or booted without it. + self.paths["github_runner_cluster_native_psi_file"].write_text("some avg10=0.00 avg60=0.00 avg300=0.00 total=0\nfull avg10=0.00 avg60=0.00 avg300=0.00 total=0\n") self.make_stand_ins() self.set_tailscale(None, None) self.facts = dict(FACTS) + # What the host holds before the role runs, which written() leaves out. + self.host_files = self.files_on_host() + + def files_on_host(self) -> set[Path]: + return {path for path in self.root.rglob("*") if path.is_file()} + + def written(self) -> list[Path]: + """The files on the stand-in host that were not there before the role ran, in path order.""" + return sorted(self.files_on_host() - self.host_files) def make_stand_ins(self) -> None: bin_dir = self.stubs / "bin" @@ -136,11 +149,25 @@ def make_stand_ins(self) -> None: if [ -n "$now" ]; then touch "$state/active-$unit"; case "$unit" in k3s|k3s-agent) join ;; esac; fi ;; disable) rm -f "$state/enabled-$unit"; if [ -n "$now" ]; then rm -f "$state/active-$unit"; fi ;; start|restart) touch "$state/active-$unit"; case "$unit" in k3s|k3s-agent) join ;; esac ;; + # Each start of a unit is a new run, with an invocation id of its own. + show) cat "$state/invocation-$unit" 2>/dev/null || echo 0 ;; +esac +# A start of the server unit runs its ExecStartPre that records the regrading level, as systemd would, from the unit the role wrote: the real helper the role placed, with the paths the unit gives it. +case "$command:$unit" in + enable:k3s|start:k3s|restart:k3s) + if [ "$command" != enable ] || [ -n "$now" ]; then + echo $(( $(cat "$state/invocation-k3s" 2>/dev/null || echo 0) + 1 )) > "$state/invocation-k3s" + pre=$(sed -n 's/^ExecStartPre=\(.*write-regrade-level .*\)$/\1/p' "$UNIT_DIR/k3s.service") + [ -z "$pre" ] || sh -c "$pre" + fi ;; esac # What the role file named when a k3s unit was started, which is what its ExecCondition would have read. case "$command:$unit" in enable:k3s*|start:k3s*|restart:k3s*) if [ "$command" != enable ] || [ -n "$now" ]; then echo "$command $unit $(cat "$ROLE_FILE" 2>/dev/null || echo none)" >> "$STUB/starts.log"; fi ;; esac +""") + # What the journal holds for a run of a unit: the line k3s logs once its etcd serves, unless the state says it never came. + stand_in(bin_dir / "journalctl", log + r"""[ -e "$STUB/state/no-etcd" ] || echo "ETCD server is now running" """) stand_in(bin_dir / "tailscale", log + r"""case "$1" in status) [ -e "$STUB/state/tailscale.json" ] || exit 1; cat "$STUB/state/tailscale.json" ;; @@ -228,6 +255,7 @@ def run_role(self, *args: str, host_vars: dict[str, Any] | None = None, check: b "PATH": f"{self.stubs / 'bin'}:{os.environ['PATH']}", "STUB": str(self.stubs), "ROLE_FILE": str(self.file("github_runner_cluster_native_data_dir", "fleet-role")), + "UNIT_DIR": str(self.paths["github_runner_cluster_native_unit_dir"]), "ANSIBLE_COLLECTIONS_PATH": str(ROOT / "playbooks" / "collections"), "ANSIBLE_NOCOLOR": "1", "ANSIBLE_FORCE_COLOR": "0", @@ -243,11 +271,11 @@ def run_role(self, *args: str, host_vars: dict[str, Any] | None = None, check: b def calls(self) -> list[str]: """The commands the stand-ins were run with, one per call (a call whose arguments hold a line break logs more lines, which are not calls).""" log = self.stubs / "calls.log" - return [line for line in log.read_text().splitlines() if re.match(r"^(systemctl|tailscale|apt-get|dpkg-query|docker|kubectl) ", line)] if log.exists() else [] + return [line for line in log.read_text().splitlines() if re.match(r"^(systemctl|journalctl|tailscale|apt-get|dpkg-query|docker|kubectl) ", line)] if log.exists() else [] def mutations(self, since: int = 0) -> list[str]: """The calls that change something, from the ``since``-th call on.""" - return [call for call in self.calls()[since:] if call.split()[0] in ("apt-get", "systemctl") and call.split()[1] not in ("is-enabled", "is-active")] + [call for call in self.calls()[since:] if call in ("tailscale logout",) or (call.startswith("kubectl ") and call.split()[1] in ("cordon", "uncordon", "drain", "label", "annotate"))] + return [call for call in self.calls()[since:] if call.split()[0] in ("apt-get", "systemctl") and call.split()[1] not in ("is-enabled", "is-active", "show")] + [call for call in self.calls()[since:] if call in ("tailscale logout",) or (call.startswith("kubectl ") and call.split()[1] in ("cordon", "uncordon", "drain", "label", "annotate"))] def file(self, key: str, name: str) -> Path: return self.paths[key] / name @@ -282,7 +310,7 @@ def starts(self) -> list[str]: return log.read_text().splitlines() if log.exists() else [] def unit_calls(self, since: int = 0) -> list[str]: - return [call for call in self.calls()[since:] if call.startswith("systemctl") and " is-" not in call and "daemon-reload" not in call] + return [call for call in self.calls()[since:] if call.startswith("systemctl") and " is-" not in call and "daemon-reload" not in call and not call.startswith("systemctl show ")] def running_as(self, role: str) -> None: """Leave the host as a run of the role leaves one that runs ``role``: the role file names it, and its unit is enabled and active. For a test whose subject is the next run, so that it need not spend a run of the role getting there; the role writes every file it reads afresh.""" diff --git a/tests/unit/regrade_settings.py b/tests/unit/regrade_settings.py new file mode 100644 index 00000000..9803377d --- /dev/null +++ b/tests/unit/regrade_settings.py @@ -0,0 +1,18 @@ +"""Reads a regrade settings file (templates/regrade.env.j2) back with a real shell, for the tests of the role's regrading (test_cluster_regrade.py and test_native_regrade.py): what a shell that sources the file, as fleet-regrade.sh does, makes of each variable it sets.""" + +from __future__ import annotations + +import subprocess + + +def sourced(text: str) -> dict[str, str]: + """Every variable the settings file sets, as bash makes of it after sourcing the file under ``set -eu``, in an environment that holds nothing else. + + Raises ``subprocess.CalledProcessError`` when the file does not source cleanly. + """ + # set -a exports every assignment the file makes, so env -0 lists them; the variables bash itself exports into an empty environment are left out. + script = "set -eu\nset -a\n. /dev/stdin <<'ENV'\n" + text + "ENV\nset +a\nenv -0\n" + shell = subprocess.run(["bash", "-c", "env -0"], capture_output=True, text=True, check=True, env={}).stdout + result = subprocess.run(["bash", "-c", script], capture_output=True, text=True, check=True, env={}) + own = {entry.split("=", 1)[0] for entry in shell.split("\0") if "=" in entry} + return {name: value for name, value in (entry.split("=", 1) for entry in result.stdout.split("\0") if "=" in entry) if name not in own} diff --git a/tests/unit/test_cluster_regrade.py b/tests/unit/test_cluster_regrade.py index f3f37b16..bbfca073 100644 --- a/tests/unit/test_cluster_regrade.py +++ b/tests/unit/test_cluster_regrade.py @@ -15,12 +15,15 @@ import yaml +from regrade_settings import sourced + ROOT = Path(__file__).resolve().parents[2] ROLE = ROOT / "roles" / "github_runner_cluster" FILES = ROLE / "files" ENTRYPOINT = FILES / "k3s" / "node-entrypoint.sh" COMPOSE = FILES / "docker-compose.yml" ACTUATOR = FILES / "regrade" / "fleet-regrade-docker.sh" +K3S_ACTUATOR = FILES / "regrade" / "fleet-regrade-k3s.sh" ANSIBLE_PLAYBOOK = os.environ.get("ANSIBLE_PLAYBOOK", "ansible-playbook") DEFAULTS: dict[str, Any] = yaml.safe_load((ROLE / "defaults" / "main.yml").read_text()) @@ -191,20 +194,42 @@ def render(self, *extra: str, host: str = "node2") -> str: def sourced(self, text: str, *names: str) -> dict[str, str]: """What a shell that sources the file, as the core does, makes of these variables.""" - script = "set -eu\n. /dev/stdin <<'ENV'\n" + text + "ENV\n" + "".join(f'printf "%s\\0" "${{{name}}}"\n' for name in names) - result = subprocess.run(["bash", "-c", script], capture_output=True, text=True, check=False) - self.assertEqual(result.returncode, 0, result.stderr) - return dict(zip(names, result.stdout.split("\0"), strict=False)) - - def test_the_core_reads_every_setting_it_needs(self) -> None: - core = (FILES / "regrade" / "fleet-regrade-core.sh").read_text() - referenced = set(re.findall(r"\$\{?([A-Z][A-Z_]+)\b", core)) - assigned = set(re.findall(r"\b([A-Z][A-Z_]+)=", core)) - # What the core takes from the environment: referenced, never assigned by the core itself, and not a bash variable. The paths and names with a default (PSI_FILE and the like) are assigned with one, so they are not needed here. - needed = referenced - assigned - {"SECONDS", "BASH_SOURCE"} - provided = {line.split("=", 1)[0] for line in self.render().splitlines() if re.match(r"[A-Z_]+=", line)} - self.assertEqual(sorted(needed - provided), []) + values = sourced(text) + return {name: values[name] for name in names} + + def needed(self, actuator: str) -> set[str]: + """The settings the core, the part both k3s actuators share and the named actuator take from the environment: referenced, never assigned by the scripts themselves, and not a bash variable. The paths and names with a default (PSI_FILE and the like) are assigned with one, so they are not needed. A shell variable's name is letters, digits and underscores, not starting with a digit.""" + scripts = "".join((FILES / "regrade" / name).read_text() for name in ("fleet-regrade-core.sh", "fleet-regrade-k3s.sh", f"fleet-regrade-{actuator}.sh")) + referenced = set(re.findall(r"\$\{?([A-Z_][A-Z0-9_]*)\b", scripts)) + assigned = set(re.findall(r"\b([A-Z_][A-Z0-9_]*)=", scripts)) + return referenced - assigned - {"SECONDS", "BASH_SOURCE"} + + def test_the_core_and_the_actuator_read_every_setting_they_need(self) -> None: + needed = self.needed("docker") + self.assertEqual(sorted(needed - set(sourced(self.render()))), []) self.assertIn("PRESSURE_THRESHOLD", needed, "the check finds the settings at all") + self.assertIn("NODE_CONTAINER", needed, "the check reads the actuator too") + + def test_the_wrapper_runs_the_docker_actuator(self) -> None: + self.assertEqual(self.sourced(self.render(), "FLEET_REGRADE_ACTUATOR"), {"FLEET_REGRADE_ACTUATOR": "docker"}) + + def test_a_native_servers_core_and_actuator_read_every_setting_they_need(self) -> None: + needed = self.needed("systemd") + self.assertEqual(sorted(needed - set(sourced(self.render("-e", "github_runner_cluster_node_mode=native")))), []) + self.assertIn("K3S_SERVICE", needed, "the check reads names with a digit in them, and the actuator too") + + def test_a_native_server_runs_the_systemd_actuator_with_k3s_and_its_admin_kubeconfig(self) -> None: + values = self.sourced(self.render("-e", "github_runner_cluster_node_mode=native"), "FLEET_REGRADE_ACTUATOR", "KUBECTL", "KUBECONFIG_PATH", "K3S_SERVICE", "K3S_DATA_DIR", "LEVEL_FILE", "STARTED_FILE") + self.assertEqual(values, { + "FLEET_REGRADE_ACTUATOR": "systemd", + "KUBECTL": "/usr/local/bin/k3s kubectl", + "KUBECONFIG_PATH": "/etc/rancher/k3s/k3s.yaml", + "K3S_SERVICE": "k3s", + "K3S_DATA_DIR": "/var/lib/rancher/k3s", + "LEVEL_FILE": "/var/lib/rancher/k3s/regrade-level", + "STARTED_FILE": "/run/github-runner-k3s-server/regrade-level", + }) + self.assertNotIn("NODE_CONTAINER=", self.render("-e", "github_runner_cluster_node_mode=native")) def test_the_values_are_the_role_variables(self) -> None: values = self.sourced(self.render(), "PRESSURE_THRESHOLD", "AVAILABLE_FLOOR_PERCENT", "SAMPLE_SECONDS", "SUSTAIN_SECONDS", "COOLDOWN_SECONDS", "OPERATION_TIMEOUT_SECONDS", "LEASE_DURATION_SECONDS", "NODE_NAME", "STEP_UP") @@ -227,15 +252,19 @@ def test_kubectl_is_the_one_in_the_nodes_container_and_stays_one_command_for_the self.assertEqual(values["KUBECONFIG_PATH"], "/etc/rancher/k3s/k3s.yaml") self.assertEqual(values["CONTROL_PLANE_TAINT"], "node-role.kubernetes.io/control-plane=true:NoSchedule") - def test_the_api_it_uses_is_another_servers_never_its_own(self) -> None: - for host, other in (("node1", "node2"), ("node2", "node1"), ("node3", "node1")): + def test_the_apis_it_may_use_are_every_other_server_then_every_standby_never_its_own(self) -> None: + for host, others in (("node1", ("node2", "node3")), ("node2", ("node1", "node3")), ("node3", ("node1", "node2")), ("standby", ("node1", "node2", "node3"))): with self.subTest(host=host): - values = self.sourced(self.render(host=host), "KUBE_SERVER") - self.assertEqual(values["KUBE_SERVER"], f"https://{other}.example.internal:6443") + values = self.sourced(self.render(host=host), "KUBE_SERVERS") + expected = [f"https://{other}.example.internal:6443" for other in others] + ([] if host == "standby" else ["https://standby.example.internal:6443"]) + self.assertEqual(values["KUBE_SERVERS"].split(" "), expected) + + def test_an_agent_that_may_not_become_a_server_is_never_a_candidate(self) -> None: + self.assertNotIn("agent.example.internal", self.sourced(self.render(), "KUBE_SERVERS")["KUBE_SERVERS"]) def test_it_follows_the_api_port(self) -> None: - values = self.sourced(self.render("-e", "github_runner_cluster_api_port=7443"), "KUBE_SERVER") - self.assertTrue(values["KUBE_SERVER"].endswith(":7443")) + values = self.sourced(self.render("-e", "github_runner_cluster_api_port=7443"), "KUBE_SERVERS") + self.assertTrue(all(url.endswith(":7443") for url in values["KUBE_SERVERS"].split(" "))) class ComposeServiceTest(unittest.TestCase): @@ -271,11 +300,11 @@ def test_the_service_files_are_where_the_mount_and_the_image_expect_them(self) - directory = FILES / "regrade" dockerfile = (directory / "Dockerfile").read_text() self.assertIn("/usr/local/lib/fleet-regrade/fleet-regrade.sh", dockerfile) - for name in ("fleet-regrade.sh", "fleet-regrade-core.sh", "fleet-regrade-docker.sh"): + for name in ("fleet-regrade.sh", "fleet-regrade-core.sh", "fleet-regrade-k3s.sh", "fleet-regrade-docker.sh"): self.assertTrue((directory / name).is_file(), name) self.assertTrue(os.access(directory / "fleet-regrade.sh", os.X_OK)) tasks = (ROLE / "tasks" / "regrade_service.yml").read_text() - for name in ("Dockerfile", "fleet-regrade.sh", "fleet-regrade-core.sh", "fleet-regrade-docker.sh"): + for name in ("Dockerfile", "fleet-regrade.sh", "fleet-regrade-core.sh", "fleet-regrade-k3s.sh", "fleet-regrade-docker.sh"): self.assertIn(f"name: {name}", tasks) @@ -468,7 +497,7 @@ def test_a_server_below_full_stops_the_play_before_any_host_removes_its_service( def test_the_actuator_labels_the_node_with_the_key_the_role_selects_on(self) -> None: self.assertIn("LEVEL_LABEL={{ github_runner_cluster_regrade_level_label }}", (ROLE / "templates" / "regrade.env.j2").read_text()) - self.assertIn('"${LEVEL_LABEL}=$1"', ACTUATOR.read_text()) + self.assertIn('"${LEVEL_LABEL}=$1"', K3S_ACTUATOR.read_text()) self.assertTrue(DEFAULTS["github_runner_cluster_regrade_level_label"].startswith(DEFAULTS["github_runner_cluster_deployed_version_annotation"].split("/")[0] + "/")) diff --git a/tests/unit/test_native_node.py b/tests/unit/test_native_node.py index c8823f6e..9020a381 100644 --- a/tests/unit/test_native_node.py +++ b/tests/unit/test_native_node.py @@ -301,9 +301,9 @@ def test_an_address_that_is_not_ipv4_fails(self) -> None: self.assertNotEqual(result.returncode, 0) self.assertIn("reports no IPv4 address", result.stderr) - def test_both_scripts_are_executable_and_shellcheck_clean(self) -> None: + def test_every_script_is_executable_and_shellcheck_clean(self) -> None: scripts = sorted((ROLE / "files" / "native").glob("*.sh")) - self.assertEqual([script.name for script in scripts], ["fleet-role-is.sh", "tailscale-up-wait.sh", "write-node-ip.sh", "write-resolv-conf.sh"]) + self.assertEqual([script.name for script in scripts], ["fleet-role-is.sh", "tailscale-up-wait.sh", "write-node-ip.sh", "write-regrade-level.sh", "write-resolv-conf.sh"]) for script in scripts: self.assertTrue(os.access(script, os.X_OK), script) if shutil.which("shellcheck"): @@ -332,6 +332,13 @@ def test_every_native_default_is_declared_with_the_same_default_and_described_in "github_runner_cluster_native_join_host", "github_runner_cluster_native_role_file", "github_runner_cluster_native_restart_marker", + "github_runner_cluster_native_regrade_level_file", + "github_runner_cluster_native_regrade_started_file", + "github_runner_cluster_native_regrade_dir", + "github_runner_cluster_native_regrade_service", + "github_runner_cluster_native_regrade_commands", + "github_runner_cluster_native_agent_runtime_dir", + "github_runner_cluster_native_server_runtime_dir", } self.assertEqual(set(SPECS), native - derived) for name, spec in SPECS.items(): @@ -355,7 +362,7 @@ class TemplateTest(unittest.TestCase): def test_the_units_and_configurations_are_templates_the_role_has(self) -> None: templates = sorted(path.name for path in (ROLE / "templates" / "native").iterdir()) - self.assertEqual(templates, ["agent.yaml.j2", "join.yaml.j2", "k3s-agent.service.j2", "k3s.service.j2", "server.yaml.j2"]) + self.assertEqual(templates, ["agent.yaml.j2", "fleet-regrade.service.j2", "join.yaml.j2", "k3s-agent.service.j2", "k3s.service.j2", "server.yaml.j2"]) config = (ROLE / "tasks" / "native" / "config.yml").read_text() for name in ("agent.yaml.j2", "server.yaml.j2", "join.yaml.j2", "native/{{ item }}.service.j2"): self.assertIn(name, config) @@ -408,7 +415,7 @@ def test_a_first_run_installs_configures_starts_and_checks_the_node(self) -> Non "token-file": str(self.file(config_dir, "token")), "node-name": "laptop", "vpn-auth-file": str(self.file(config_dir, "vpn-auth")), - "resolv-conf": "/run/github-runner-k3s/resolv.conf", + "resolv-conf": str(self.paths["github_runner_cluster_native_run_dir"] / "github-runner-k3s-agent" / "resolv.conf"), }) self.assertEqual(self.file(config_dir, "token").read_text(), TOKEN) self.assertEqual(self.file(config_dir, "vpn-auth").read_text(), f"name=tailscale,joinKey={JOIN_KEY}") @@ -444,11 +451,11 @@ def test_the_unit_runs_the_installed_binary_under_the_wrapper_with_the_hosts_joi service = sections["Service"] self.assertIn(f"ExecStart={bin_dir}/k3s agent --config {self.paths['github_runner_cluster_native_config_dir']}/agent.yaml", service) self.assertIn(f"ExecCondition={lib}/fleet-role-is agent {self.paths['github_runner_cluster_native_data_dir']}/fleet-role", service) - self.assertIn(f"ExecStartPre={lib}/write-resolv-conf /run/github-runner-k3s/resolv.conf", service) + self.assertIn(f"ExecStartPre={lib}/write-resolv-conf {self.paths['github_runner_cluster_native_run_dir']}/github-runner-k3s-agent/resolv.conf", service) self.assertIn(f'Environment="PATH={lib}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"', service) self.assertIn(f'Environment="GITHUB_RUNNER_TAILSCALE={self.stubs / "bin" / "tailscale"}"', service) self.assertIn(f'Environment="GITHUB_RUNNER_JOIN_HOST=server1.{DOMAIN}"', service) - for setting in ("Type=exec", "KillMode=process", "Delegate=yes", "Restart=always", "TimeoutStartSec=0", "RuntimeDirectory=github-runner-k3s", "RuntimeDirectoryPreserve=restart"): + for setting in ("Type=exec", "KillMode=process", "Delegate=yes", "Restart=always", "TimeoutStartSec=0", "RuntimeDirectory=github-runner-k3s-agent", "RuntimeDirectoryPreserve=restart"): self.assertIn(setting, service) self.assertIn("Wants=network-online.target tailscaled.service", sections["Unit"]) self.assertIn("After=network-online.target tailscaled.service", sections["Unit"]) @@ -521,7 +528,7 @@ def test_a_check_run_changes_nothing_and_says_what_a_real_run_would_do(self) -> result = self.run_role(check=True) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertEqual(self.mutations(), []) - self.assertEqual([path for path in self.root.rglob("*") if path.is_file()], []) + self.assertEqual(self.written(), []) self.assertIn("apt-get install tailscale=1.102.4 (installed: none)", result.stdout) self.assertIn("enable start k3s-agent", result.stdout) self.assertEqual(self.recap(result)["failed"], 0) @@ -601,7 +608,7 @@ def test_an_unsupported_architecture_stops_the_run_before_anything_is_changed(se self.assertNotEqual(result.returncode, 0) self.assertIn("not an architecture a native k3s node supports", result.stdout) self.assertEqual(self.calls(), []) - self.assertEqual([path for path in self.root.rglob("*") if path.is_file()], []) + self.assertEqual(self.written(), []) def test_a_host_that_is_not_linux_under_systemd_is_refused(self) -> None: for change in ({"system": "Darwin"}, {"service_mgr": "openrc"}): @@ -625,7 +632,6 @@ def test_settings_a_native_node_needs_are_checked_first(self) -> None: ({"github_runner_cluster_native_k3s_sha256": "ABC"}, "must be empty or 64 lower-case hexadecimal digits"), ({"github_runner_cluster_k3s_token": ""}, "github_runner_cluster_k3s_token is not set"), ({"github_runner_cluster_k3s_version": "latest"}, "must be a pinned rancher/k3s tag"), - ({"github_runner_cluster_regrade": True}, "a native server has none yet"), ({"github_runner_cluster_node_mode": "container"}, "must be one of: compose, native"), ({"github_runner_cluster_network_mode": "host"}, "which sets github_runner_cluster_node_mode: native"), ] diff --git a/tests/unit/test_native_regrade.py b/tests/unit/test_native_regrade.py new file mode 100644 index 00000000..99e93002 --- /dev/null +++ b/tests/unit/test_native_regrade.py @@ -0,0 +1,165 @@ +"""Tests the github_runner_cluster role's memory-pressure regrading of a native server (tests/native/run.yml) against the stand-ins in native_stand_ins.py: the role's refusals, the fleet-regrade unit and the settings it runs with, the server unit's ExecStartPre that records the level, a server turn at etcd-only, and turning regrading off. The systemd actuator itself is tested in tests/regrade/regrade-systemd.sh, run by test_regrade_shell.py.""" + +from __future__ import annotations + +import re +import subprocess +import unittest +from typing import Any + +from native_stand_ins import DOMAIN, ROLE, NativeRoleHarness +from regrade_settings import sourced + +ON = {"github_runner_cluster_regrade": True} + + +class NativeRegradeTest(NativeRoleHarness): + """A native server that regrades, and one that is refused it.""" + + def run_server(self, check: bool = False, **laptop: Any) -> subprocess.CompletedProcess[str]: + """Run laptop as a native server beside the Docker servers server1 and server2, all three voters Ready.""" + (self.state / "voters").write_text("server1,True,true\nserver2,True,true\nlaptop,True,true\n") + return self.run_three(laptop=laptop, check=check) + + def settings(self) -> dict[str, str]: + return sourced(self.file("github_runner_cluster_native_lib_dir", "regrade/fleet-regrade.env").read_text()) + + def regrade_unit(self) -> str: + return self.file("github_runner_cluster_native_unit_dir", "fleet-regrade.service").read_text() + + def started_level(self) -> str: + return self.file("github_runner_cluster_native_run_dir", "github-runner-k3s-server/regrade-level").read_text().strip() + + def test_a_native_server_runs_the_fleet_regrade_unit_with_the_systemd_actuator(self) -> None: + result = self.run_server(**ON) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + lib = self.paths["github_runner_cluster_native_lib_dir"] / "regrade" + for name in ("fleet-regrade.sh", "fleet-regrade-core.sh", "fleet-regrade-k3s.sh", "fleet-regrade-systemd.sh"): + self.assertEqual((lib / name).read_text(), (ROLE / "files" / "regrade" / name).read_text(), name) + settings = self.settings() + self.assertEqual(settings["FLEET_REGRADE_ACTUATOR"], "systemd") + self.assertEqual(settings["KUBECTL"], f"{self.paths['github_runner_cluster_native_bin_dir']}/k3s kubectl") + self.assertEqual(settings["KUBECONFIG_PATH"], f"{self.paths['github_runner_cluster_native_config_dir']}/k3s.yaml") + self.assertEqual(settings["K3S_SERVICE"], "k3s") + self.assertEqual(settings["K3S_DATA_DIR"], str(self.paths["github_runner_cluster_native_data_dir"])) + self.assertEqual(settings["LEVEL_FILE"], str(self.file("github_runner_cluster_native_data_dir", "regrade-level"))) + self.assertEqual(settings["STARTED_FILE"], str(self.file("github_runner_cluster_native_run_dir", "github-runner-k3s-server/regrade-level"))) + self.assertEqual(settings["NODE_NAME"], "laptop") + self.assertNotIn("NODE_CONTAINER", settings) + # Every other server, never this one's own API. + self.assertEqual(settings["KUBE_SERVERS"].split(" "), [f"https://server1.{DOMAIN}:6443", f"https://server2.{DOMAIN}:6443"]) + unit = self.regrade_unit() + self.assertIn(f"ExecStart={lib}/fleet-regrade.sh watch", unit) + self.assertIn(f'Environment="FLEET_REGRADE_ENV={lib}/fleet-regrade.env"', unit) + self.assertNotRegex(unit, r"(?m)^(BindsTo|PartOf|Requires)=", "the watcher restarts k3s and must outlive it") + link = self.file("github_runner_cluster_native_bin_dir", "fleet-regrade") + self.assertTrue(link.is_symlink()) + self.assertEqual(link.resolve(), (lib / "fleet-regrade.sh").resolve()) + self.assertIn("systemctl enable fleet-regrade", self.calls()) + self.assertIn("systemctl start fleet-regrade", self.calls()) + self.assertLess(self.calls().index("systemctl start k3s"), self.calls().index("systemctl start fleet-regrade")) + + def test_a_rerun_leaves_the_unit_running_and_a_changed_setting_restarts_it(self) -> None: + self.assertEqual(self.run_server(**ON).returncode, 0) + before = len(self.calls()) + self.assertEqual(self.run_server(**ON).returncode, 0) + self.assertEqual([call for call in self.unit_calls(before) if "fleet-regrade" in call], []) + before = len(self.calls()) + result = self.run_server(**ON, github_runner_cluster_regrade_pressure_threshold=25) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual([call for call in self.unit_calls(before) if "fleet-regrade" in call], ["systemctl restart fleet-regrade"]) + self.assertEqual(self.settings()["PRESSURE_THRESHOLD"], "25") + + def test_the_server_unit_records_the_level_before_every_start_whether_regrading_is_on_or_not(self) -> None: + self.assertEqual(self.run_server().returncode, 0) + off = self.file("github_runner_cluster_native_unit_dir", "k3s.service").read_text() + lib, config = self.paths["github_runner_cluster_native_lib_dir"], self.paths["github_runner_cluster_native_config_dir"] + line = f"ExecStartPre={lib}/write-regrade-level {self.file('github_runner_cluster_native_data_dir', 'regrade-level')} {config}/server.yaml.d/50-regrade.yaml {self.file('github_runner_cluster_native_run_dir', 'github-runner-k3s-server/regrade-level')}" + self.assertIn(line, off.splitlines()) + self.assertEqual(self.file("github_runner_cluster_native_lib_dir", "write-regrade-level").read_text(), (ROLE / "files" / "native" / "write-regrade-level.sh").read_text()) + # The start the stand-in made ran it: a server never stepped down starts at full, with a drop-in that sets nothing. + self.assertEqual(self.started_level(), "full") + self.assertEqual([line for line in self.file("github_runner_cluster_native_config_dir", "server.yaml.d/50-regrade.yaml").read_text().splitlines() if not line.startswith("#")], []) + self.assertEqual(self.run_server(**ON).returncode, 0) + self.assertEqual(self.file("github_runner_cluster_native_unit_dir", "k3s.service").read_text(), off) + + def test_regrading_is_refused_where_it_could_not_work_before_anything_changes(self) -> None: + cases: list[tuple[str, dict[str, Any], str]] = [ + ("an agent", {"github_runner_cluster_node_role": "agent"}, "is an agent"), + ("a kernel without memory pressure", {"github_runner_cluster_native_psi_file": str(self.root / "proc" / "none")}, "exposes no memory pressure"), + ] + for name, laptop, message in cases: + with self.subTest(name): + result = self.run_server(**ON, **laptop) + self.assertNotEqual(result.returncode, 0) + self.assertIn(message, re.sub(r"\s+", " ", result.stdout)) + self.assertEqual(self.mutations(), []) + + def test_the_only_server_is_refused(self) -> None: + result = self.run_role(host_vars={"github_runner_cluster_bootstrap": True, **ON}, only_the_node=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("is the only server", re.sub(r"\s+", " ", result.stdout)) + self.assertEqual(self.mutations(), []) + + def test_an_eligible_host_that_runs_as_an_agent_is_refused_whatever_the_inventory_says(self) -> None: + # The role the host is to run is the one recorded on it, which whatever moves votes between hosts sets at run time. + self.running_as("agent") + result = self.run_server(**ON, github_runner_cluster_voter_eligible=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("is an agent", re.sub(r"\s+", " ", result.stdout)) + + def test_turning_regrading_off_at_full_removes_the_unit_and_leaves_the_server_running(self) -> None: + self.assertEqual(self.run_server(**ON).returncode, 0) + before = len(self.calls()) + result = self.run_server() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.unit_calls(before), ["systemctl disable --now fleet-regrade"]) + self.assertFalse(self.file("github_runner_cluster_native_unit_dir", "fleet-regrade.service").exists()) + self.assertFalse((self.paths["github_runner_cluster_native_lib_dir"] / "regrade").exists()) + self.assertFalse(self.file("github_runner_cluster_native_bin_dir", "fleet-regrade").is_symlink()) + self.assertTrue((self.state / "active-k3s").exists()) + before = len(self.calls()) + self.assertEqual(self.run_server().returncode, 0) + self.assertEqual(self.unit_calls(before), []) + + def test_turning_regrading_off_below_full_is_refused_and_removes_nothing(self) -> None: + self.assertEqual(self.run_server(**ON).returncode, 0) + self.file("github_runner_cluster_native_run_dir", "github-runner-k3s-server/regrade-level").write_text("etcd-only\n") + before = len(self.calls()) + result = self.run_server() + self.assertNotEqual(result.returncode, 0) + self.assertIn("is running at the level etcd-only, below full", re.sub(r"\s+", " ", result.stdout)) + self.assertEqual(self.unit_calls(before), []) + self.assertTrue(self.file("github_runner_cluster_native_unit_dir", "fleet-regrade.service").exists()) + + def test_a_server_restarted_at_etcd_only_is_waited_for_by_its_etcd_not_its_node(self) -> None: + self.assertEqual(self.run_server(**ON).returncode, 0) + self.file("github_runner_cluster_native_data_dir", "regrade-level").write_text("etcd-only\n") + # At etcd-only there is no kubelet, so the node is never Ready again. + (self.state / "node-ready").write_text("Unknown,True") + before = len(self.calls()) + result = self.run_server(**ON, github_runner_cluster_node_taints=["dedicated=ci:NoSchedule"], github_runner_cluster_native_ready_timeout=5) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("systemctl restart k3s", self.unit_calls(before)) + self.assertEqual(self.started_level(), "etcd-only") + self.assertIn("journalctl --no-pager --output cat _SYSTEMD_INVOCATION_ID=2", self.calls()[before:]) + self.assertFalse(any(call.startswith("kubectl get lease --namespace kube-node-lease") for call in self.calls()[before:])) + + def test_a_server_restarted_at_etcd_only_whose_etcd_does_not_come_back_fails_the_turn(self) -> None: + self.assertEqual(self.run_server(**ON).returncode, 0) + self.file("github_runner_cluster_native_data_dir", "regrade-level").write_text("etcd-only\n") + (self.state / "no-etcd").touch() + result = self.run_server(**ON, github_runner_cluster_node_taints=["dedicated=ci:NoSchedule"], github_runner_cluster_native_ready_timeout=5) + self.assertNotEqual(result.returncode, 0) + + def test_a_demoted_server_forgets_its_level(self) -> None: + level = self.file("github_runner_cluster_native_data_dir", "regrade-level") + level.parent.mkdir(parents=True, exist_ok=True) + level.write_text("control-plane\n") + result = self.demote() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertFalse(level.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_native_server.py b/tests/unit/test_native_server.py index 2cd558bd..4501aa98 100644 --- a/tests/unit/test_native_server.py +++ b/tests/unit/test_native_server.py @@ -27,7 +27,7 @@ def test_a_native_bootstrap_server_starts_a_new_cluster_with_the_docker_servers_ "write-kubeconfig-mode": "0600", "tls-san": [f"laptop.{DOMAIN}"], "etcd-arg": ["heartbeat-interval=500", "election-timeout=5000"], - "resolv-conf": "/run/github-runner-k3s/resolv.conf", + "resolv-conf": str(self.paths["github_runner_cluster_native_run_dir"] / "github-runner-k3s-server" / "resolv.conf"), }) self.assertEqual(yaml.safe_load(self.file(config_dir, "server.yaml.d/10-join.yaml").read_text()), {"cluster-init": True}) for name in ("server.yaml", "server.yaml.d/10-join.yaml"): @@ -48,9 +48,13 @@ def test_the_server_unit_runs_k3s_server_under_its_role_condition_with_the_mesh_ self.assertIn(f"ExecCondition={lib}/fleet-role-is server {self.paths['github_runner_cluster_native_data_dir']}/fleet-role", service) self.assertIn(f"ExecStartPre={lib}/write-node-ip {config}/server.yaml.d/00-node-ip.yaml {config}/vpn-auth", service) self.assertIn(f"ExecStart={self.paths['github_runner_cluster_native_bin_dir']}/k3s server --config {config}/server.yaml", service) + # Each unit has a runtime directory of its own: systemd removes a unit's when a start of it is skipped by its condition, which would otherwise take the running unit's files. + self.assertIn("RuntimeDirectory=github-runner-k3s-server", service) + self.assertIn(f"ExecStartPre={lib}/write-resolv-conf {self.paths['github_runner_cluster_native_run_dir']}/github-runner-k3s-server/resolv.conf", service) # Everything else is the agent unit's: one runs at a time, under the same limits. agent = [line for line in self.file("github_runner_cluster_native_unit_dir", "k3s-agent.service").read_text().split("[Service]")[1].split("[Install]")[0].splitlines() if line and not line.startswith("#")] - self.assertEqual([line for line in service if not line.startswith(("ExecCondition=", "ExecStart"))], [line for line in agent if not line.startswith(("ExecCondition=", "ExecStart"))]) + own = ("ExecCondition=", "ExecStart", "RuntimeDirectory=") + self.assertEqual([line for line in service if not line.startswith(own)], [line for line in agent if not line.startswith(own)]) for helper in ("write-node-ip", "fleet-role-is"): self.assertEqual(self.file("github_runner_cluster_native_lib_dir", helper).read_text(), (ROLE / "files" / "native" / f"{helper}.sh").read_text()) diff --git a/tests/unit/test_regrade_shell.py b/tests/unit/test_regrade_shell.py index 06ea8d4e..13471147 100644 --- a/tests/unit/test_regrade_shell.py +++ b/tests/unit/test_regrade_shell.py @@ -12,7 +12,10 @@ FILES = ROOT / "roles" / "github_runner_cluster" / "files" / "regrade" TESTS = ROOT / "tests" / "regrade" CORE = FILES / "fleet-regrade-core.sh" +K3S_ACTUATOR = FILES / "fleet-regrade-k3s.sh" DOCKER_ACTUATOR = FILES / "fleet-regrade-docker.sh" +SYSTEMD_ACTUATOR = FILES / "fleet-regrade-systemd.sh" +PRESTART = ROOT / "roles" / "github_runner_cluster" / "files" / "native" / "write-regrade-level.sh" SERVICE = FILES / "fleet-regrade.sh" @@ -40,7 +43,14 @@ def test_lease_and_peer_coordination(self) -> None: class DockerActuatorTest(unittest.TestCase): def test_levels_current_level_and_the_restart_between_them(self) -> None: - result = run_script("regrade-docker.sh", CORE, DOCKER_ACTUATOR) + result = run_script("regrade-docker.sh", CORE, K3S_ACTUATOR, DOCKER_ACTUATOR) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("all passed", result.stdout) + + +class SystemdActuatorTest(unittest.TestCase): + def test_levels_current_level_the_restart_between_them_and_the_pods_left_at_etcd_only(self) -> None: + result = run_script("regrade-systemd.sh", CORE, K3S_ACTUATOR, SYSTEMD_ACTUATOR, PRESTART) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertIn("all passed", result.stdout) @@ -54,7 +64,7 @@ def run_service(self, *args: str, **env: str) -> subprocess.CompletedProcess[str stub.parent.mkdir() stub.write_text("#!/bin/sh\necho \"docker called: $*\" >&2\nexit 1\n") stub.chmod(0o755) - full_env = {**os.environ, "PATH": f"{stub.parent}:{os.environ['PATH']}", "FLEET_REGRADE_ENV": str(Path(work) / "none"), **env} + full_env = {**os.environ, "PATH": f"{stub.parent}:{os.environ['PATH']}", "FLEET_REGRADE_ENV": str(Path(work) / "none"), "FLEET_REGRADE_ACTUATOR": "docker", **env} return subprocess.run(["bash", str(SERVICE), *args], capture_output=True, text=True, check=False, env=full_env) def test_watching_without_memory_pressure_stops_with_the_reason(self) -> None: @@ -71,6 +81,14 @@ def test_stepping_by_hand_does_not_need_the_pressure_file(self) -> None: result = self.run_service("down", PSI_FILE="/nonexistent/pressure") self.assertNotIn("exposes no memory pressure", result.stderr) + def test_an_actuator_that_is_neither_docker_nor_systemd_is_refused_before_anything_runs(self) -> None: + for actuator in ("", "nix"): + with self.subTest(actuator=actuator): + result = self.run_service("status", FLEET_REGRADE_ACTUATOR=actuator) + self.assertEqual(result.returncode, 1) + self.assertIn("it must be docker or systemd", result.stderr) + self.assertNotIn("docker called", result.stderr) + def test_a_readable_pressure_file_passes_the_check(self) -> None: with tempfile.NamedTemporaryFile("w") as psi: psi.write("some avg10=0.00 avg60=0.00 avg300=0.00 total=1\n")