diff --git a/.github/workflows/mesh-integration.yml b/.github/workflows/mesh-integration.yml index d13962bc..a8216377 100644 --- a/.github/workflows/mesh-integration.yml +++ b/.github/workflows/mesh-integration.yml @@ -1,6 +1,6 @@ name: Mesh integration -# Brings up three k3s nodes in Docker and joins them through Headscale with the cluster role (tests/mesh/run.sh), once per scenario: headscale_hosted, headscale_in_cluster, and the headscale_existing policy check, plus the role's refusal of a Docker Compose release that recreates the containers it has just built. headscale_in_cluster also runs with github_runner_cluster_docker_context pinned while the current Docker context points at nothing, which covers the cluster role's own docker calls and the recovery playbook's, and context_checks checks the role refuses a pin it cannot honour. regrade turns memory-pressure regrading on and steps a server down to etcd-only under (simulated) pressure and back up, checking that it keeps its etcd seat and vote and that no other server steps meanwhile. native_server joins a native host (a systemd container) as an agent beside three Docker servers, then promotes it to a server in place: it must join etcd as a learner, be promoted to a voter, and keep the cluster writable throughout. role_controller adds a standby agent, runs the role controller's probe and controller from the image built from this checkout, makes a voter slow with tc netem and pauses another, and checks every probe reports the link medium the cluster role labelled its node with, the controller reports the voter unfit with the swap named, reports that it would not act with a voter paused, and changes nothing. Only on changes that can affect how nodes join the mesh. +# Brings up three k3s nodes in Docker and joins them through Headscale with the cluster role (tests/mesh/run.sh), once per scenario: headscale_hosted, headscale_in_cluster, and the headscale_existing policy check, plus the role's refusal of a Docker Compose release that recreates the containers it has just built. headscale_in_cluster also runs with github_runner_cluster_docker_context pinned while the current Docker context points at nothing, which covers the cluster role's own docker calls and the recovery playbook's, and context_checks checks the role refuses a pin it cannot honour. regrade turns memory-pressure regrading on and steps a server down to etcd-only under (simulated) pressure and back up, checking that it keeps its etcd seat and vote and that no other server steps meanwhile. native_server joins a native host (a systemd container) as an agent beside three Docker servers, then promotes it to a server in place: it must join etcd as a learner, be promoted to a voter, and keep the cluster writable throughout. rejoin removes the bootstrap Docker server's etcd member and checks the server idles on its tombstone, refuses to start a second cluster from its moved-aside datastore with no server to join, joins the existing cluster from a join URL in its role directory with the same cluster ID, and idles as a server whose role file names the agent. role_controller adds a standby agent, runs the role controller's probe and controller from the image built from this checkout, makes a voter slow with tc netem and pauses another, and checks every probe reports the link medium the cluster role labelled its node with, the controller reports the voter unfit with the swap named, reports that it would not act with a voter paused, and changes nothing. Only on changes that can affect how nodes join the mesh. # # Runs on GitHub-hosted ubuntu-latest, whose Docker daemon runs the privileged containers the test needs (k3s, and Tailscale inside it). The work directory sits under runner.temp, which the job always has write access to. @@ -44,6 +44,7 @@ jobs: - {scenario: regrade, compose: v5.5.1} - {scenario: native_server, compose: v5.5.1} - {scenario: role_controller, compose: v5.5.1} + - {scenario: rejoin, compose: v5.5.1} - {scenario: hosted, compose: v2.39.0} - {scenario: compose_faulty, compose: v2.38.2} - {scenario: in_cluster, compose: v5.5.1, pin: true} diff --git a/README.md b/README.md index d55826c8..eca1e1a1 100644 --- a/README.md +++ b/README.md @@ -246,7 +246,7 @@ Commits follow Conventional Commits (`fix:`, `feat:`, `tune:`, and so on). - **etcd's own peer traffic does not follow vpn-auth's `--node-ip`/`--advertise-address` the way every other kind of cluster traffic does - it stays on the container's own internal, cross-machine-unroutable Docker-bridge IP by default.** k3s logs this outright ("Etcd IP (PrivateIP) remains the local IP. Running etcd traffic over VPN is not recommended due to performance issues") and deliberately avoids the VPN interface for etcd specifically, assuming a real local network is the better alternative - an assumption that doesn't hold here, since a container's own bridge IP is never reachable from another machine at all. Confirmed live: a second server's join failed outright with `MemberAdd request timed out`, because it was trying to reach the first server's own unroutable bridge IP. An `--etcd-arg="initial-advertise-peer-urls=..."` passthrough looks like the fix but isn't: it only overrides etcd's own raw flag, while k3s independently builds its own `--initial-cluster` string from `config.PrivateIP`, computed separately and earlier - the two then disagree and etcd's bootstrap validation rejects the mismatch outright (`"--initial-cluster has ...= but missing from --initial-advertise-peer-urls="`). The actual fix, traced through k3s's own source (`pkg/cli/server/server.go`, `pkg/executor/embed/embed.go`): `config.PrivateIP` is populated from `--node-ip`'s own value if set, and this happens *before* `--vpn-auth`'s own executor-layer code later rebuilds `NodeIP` from its own Tailscale detection - so passing `--node-ip` explicitly ourselves, to the same real Tailscale IP vpn-auth would itself have detected, is what actually makes `PrivateIP` (and therefore etcd's peer-advertise address) correct, with no second, disagreeing source of truth for the same value. `docker-compose.yml`'s `k3s` service entrypoint brings Tailscale up itself and reads `tailscale ip -4` before exec'ing k3s specifically to learn this IP first, since `exec` replaces the script's own process and nothing can react afterward to what `--vpn-auth`'s own internal call would have produced. - **A global `--etcd-arg` override is not scoped to the "real" server's own etcd instance - it also applies to k3s's separate, internal "temporary etcd", spun up on every *restart* (never on first boot) to reconcile with an already-initialized on-disk datastore before the real instance starts.** An earlier `--etcd-arg="listen-peer-urls=https://0.0.0.0:2380"` (kept as a "harmless" belt-and-braces bind override after fixing the peer-URL issue above) forced that temporary instance onto the same TLS `0.0.0.0:2380` bind it has no certificates for by design (it normally listens on a local, ephemeral, non-TLS port). Confirmed live: this only broke on a restart against an already-initialized datastore, never on a fresh `--cluster-init`, which is exactly why it wasn't caught until well after the first working deploy - fatal: `"cannot listen on TLS for [::]:2380: KeyFile and CertFile are not presented"`. `--node-ip` alone (above) is sufficient; etcd's own default `listen-peer-urls` already derives from the same `PrivateIP` correctly, so there's nothing left for an explicit override to add. - **Calling `tailscale up --authkey=...` unconditionally on every container restart re-registers a brand new tailnet device each time, instead of reconnecting the persisted one - even with the join key's own state volume intact.** A reusable Tailscale auth key is designed to authorize provisioning *new* devices, and a naive `tailscale up` call on every restart (needed here to learn this node's own Tailscale IP before `exec`ing k3s - see above) doesn't check whether the daemon is already authenticated first, so it keeps minting new devices under names like `node1`, `node1-1`, `node1-2`... k3s's own `--vpn-auth` integration avoids exactly this from v1.35.0 on, by checking the Tailscale backend state first and skipping its own `tailscale up` call entirely once already `"Running"` (`StartVPN()` in `pkg/vpn/vpn.go`; the pinned v1.34 releases run `tailscale up --authkey ... --reset` on every start whatever the state) - the manual pre-empt call needs the identical guard: check `tailscale ip -4` succeeds *before* calling `tailscale up` at all, only calling it when not already authenticated. -- **Tailscale assigns an IP and wires up its own MagicDNS resolver as two separate steps, not atomically - a node having a real IP is not proof DNS resolution against another tailnet member is working yet.** A joining server needs to resolve the bootstrap host's own MagicDNS hostname (`K3S_JOIN_SERVER_URL`) before it can bootstrap against it at all. Confirmed live: k3s failed outright with `"dial tcp: lookup k3s-server.example.ts.net: no such host"` immediately after `tailscale ip -4` had already returned a real address for the *local* node. Fix: wait for a real `nslookup` success against the join server's own hostname specifically, not just local Tailscale readiness, before invoking k3s (the `k3s` service's entrypoint only does this when `K3S_JOIN_SERVER_URL` is actually set - the bootstrap host has nothing else to resolve). +- **Tailscale assigns an IP and wires up its own MagicDNS resolver as two separate steps, not atomically - a node having a real IP is not proof DNS resolution against another tailnet member is working yet.** A joining server needs to resolve the bootstrap host's own MagicDNS hostname (`K3S_JOIN_SERVER_URL`) before it can bootstrap against it at all. Confirmed live: k3s failed outright with `"dial tcp: lookup k3s-server.example.ts.net: no such host"` immediately after `tailscale ip -4` had already returned a real address for the *local* node. Fix: wait for a real `nslookup` success against the join server's own hostname specifically, not just local Tailscale readiness, before invoking k3s (the `k3s` service's entrypoint only does this when it has a server to join, `K3S_JOIN_SERVER_URL` or the role directory's `join-url` - the bootstrap host has nothing else to resolve). - **A server that memory-pressure regrading has stepped to `etcd-only` is `NotReady` by design, and must stay registered.** With `github_runner_cluster_regrade` on, a server under sustained memory pressure steps down (`full`, `control-plane`, `etcd-only`) while it keeps its etcd member and vote; at `etcd-only` k3s runs with `--disable-apiserver --disable-controller-manager --disable-scheduler --disable-agent`, so there is no kubelet to keep its Node Ready, and the Node goes `NotReady` with `Unknown` conditions and stays so. That is not a failure to clear. Its Node object must not be deleted (k3s removes that server's etcd member with it), so anything that culls long-`NotReady` nodes must leave a Node carrying the `github-runner.exadev.com/regrade-level` label alone, and the role's wait for every node to be Ready already does. `kubectl` on such a server's own container is unreliable (its API server is off); the regrade service always reaches the cluster through another server. See the cluster role's README, Memory-pressure regrading. Confirmed against a three-server k3s cluster: the same etcd member ids before, during and after, the cluster taking writes with the stepped-down server as one of the two live voters, and the same Node and node password after stepping back up. - **A single remaining healthy etcd server can get stuck indefinitely if another member is registered but permanently unreachable (e.g. a partial join that never completed), even though that member is a non-voting learner and shouldn't block quorum.** Symptom: `"failed to publish local member to cluster through raft"` with `context deadline exceeded`, repeating forever, alongside `kubectl` returning `503 the server is currently unable to handle the request`. The fix is k3s's own documented recovery mechanism, run once against the affected node: stop the container, then `k3s server --cluster-reset` (with the SAME flags the real service uses, including `--node-ip` set to the node's own persisted Tailscale IP - getting this wrong re-registers the single remaining member under the *wrong* address, e.g. the invoking container's own ad-hoc bridge IP, and the real service then fails to start with `"this server is a not a member of the etcd cluster"`) - this resets *etcd cluster membership* back to a single member using the *existing local data*, it does not wipe anything. It's a one-shot operation: k3s writes a guard file after a successful reset and refuses to run `--cluster-reset` again until it's manually deleted (`rm /var/lib/rancher/k3s/server/db/reset-flag`) if a second reset is genuinely needed. After a reset, restart the service normally (without the flag) and clean up the stale member's own now-orphaned Kubernetes `Node` object and `node-password` Secret before it rejoins fresh. diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index 203b74ba..d84e803c 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -97,7 +97,7 @@ A native server is configured as every Docker server is, because k3s refuses a s **etcd's peer address is the host's Tailscale address.** k3s builds etcd's peer URL from its private IP, which it takes from `--node-ip` before the `--vpn-auth` integration sets only the advertise address (`pkg/cli/server/server.go`: "Etcd IP (PrivateIP) remains the local IP"), so without it a native server's etcd would advertise a LAN address and lose its peers whenever the host changes network. The server unit runs `files/native/write-node-ip.sh` before every start, which writes `node-ip` into `server.yaml.d/00-node-ip.yaml` from the client's own address (logging the client in first with exactly the `tailscale up` k3s runs, from the same `--vpn-auth` file, when it has none yet). k3s reads `server.yaml`, then every `*.yaml` in `server.yaml.d` in name order (`dotDFiles` in `pkg/configfilearg/parser.go`). -**Two units, one role.** Both units are written on every native host, and each starts only while the role file `/var/lib/rancher/k3s/fleet-role` names its role: each unit's `ExecCondition` runs `files/native/fleet-role-is.sh`, and systemd skips a start whose condition exits 1 to 254 without marking the unit failed, and `Restart=always` never restarts a unit whose start was skipped that way (`service_shall_restart` in systemd's `src/core/service.c`), so the two never run together and a demoted server is never started again by its own restart policy. A host set up before the role file existed runs the agent unit and is recorded as an agent. For a host marked `github_runner_cluster_voter_eligible` the inventory gives only the role it starts with; once its role file names a role, a run keeps it, since the controller that moves votes changes it at run time. For any other native host the inventory decides, and a run whose inventory role differs from the role file switches the host. +**Two units, one role.** Both units are written on every native host, and each starts only while the role file `/var/lib/rancher/k3s/fleet-role` names its role: each unit's `ExecCondition` runs `files/native/fleet-role-is.sh`, and systemd skips a start whose condition exits 1 to 254 without marking the unit failed, and `Restart=always` never restarts a unit whose start was skipped that way (`service_shall_restart` in systemd's `src/core/service.c`), so the two never run together and a demoted server is never started again by its own restart policy. The server unit's condition also refuses while k3s's datastore holds a tombstone (`server/db/etcd/tombstone`, which k3s writes when its etcd member is removed and would otherwise act on at its next start by joining again as a new learner), so a server whose member was removed while its role file still named it a server, because it was down at the time, does not rejoin; moving `server/db` aside, as a demotion does, takes the tombstone with it (see [The role directory and a former member](#the-role-directory-and-a-former-member)). A host set up before the role file existed runs the agent unit and is recorded as an agent. For a host marked `github_runner_cluster_voter_eligible` the inventory gives only the role it starts with; once its role file names a role, a run keeps it, since the controller that moves votes changes it at run time. For any other native host the inventory decides, and a run whose inventory role differs from the role file switches the host. **Bootstrapping, joining and rejoining.** `server.yaml.d/10-join.yaml` holds `cluster-init: true` only when the host is the inventory's bootstrap host, has no datastore (`server/db`) and none a demotion moved aside (`server/db.demoted`), and no other server of the inventory (Docker or native, or eligible) answers as running one; a server that cannot be read refuses the run, since starting a second cluster beside the real one would split the fleet. Otherwise it holds `server:`, the bootstrap host's join URL, or for the bootstrap host itself the first other server that answered, so a host that was ever a member, or a bootstrap host that comes back after the cluster moved on, always joins the cluster that exists. k3s joins a new server as an etcd learner and the leader promotes it once it has caught up (`MemberAddAsLearner` in `pkg/etcd/etcd.go`), so a learner never costs quorum, and the turn ends only once the node's `EtcdIsVoter` condition is True. A native bootstrap host is refused under `headscale_in_cluster`, whose bootstrap server runs Headscale as a static pod under a supervisor only the Docker node has (`files/k3s/node-entrypoint.sh`). @@ -321,6 +321,18 @@ The role publishes the two host settings as Node labels, `github-runner.exadev.c Beside them the role publishes the medium of the host's link, `wired`, `wifi` or `unknown`, as `github-runner.exadev.com/link` (`github_runner_cluster_link_label`), read on the host in each run that publishes the other two (`tasks/link.yml`, with `files/node-link.sh`). A node in a container or a VM cannot read it itself: it sees a veth, or a VM's virtio NIC, whatever the host beneath it uses. On a Mac whose node's Colima profile is bridged (`network.mode: bridged` with `network.address: true` in the profile's `colima.yaml`), the medium is that of the interface the VM is bridged to (`network.interface`), which is not the interface the Mac's own traffic takes when macOS prefers another, such as an Ethernet adapter over the Wi-Fi the VM is bridged to; for any other Mac runtime it is that of the interface the Mac's default route leaves by. The medium of a Mac interface is the name of the hardware port `networksetup -listallhardwareports` lists it under: Wi-Fi is `wifi`, a port whose name says Ethernet or LAN (built-in Ethernet, Thunderbolt Ethernet, a USB LAN adapter) is `wired`, and any other is `unknown`. On Linux it is that of the interface the default route leaves by, from its `/sys/class/net` entry: a wireless phy is `wifi`, a device is `wired`, and a virtual interface (a bridge, a bond, a tunnel) is `unknown`, since the link beneath it is not read. The label is the medium as the last run of the role found it, so a host whose link changes (a cable plugged in, a Wi-Fi fallback) carries the new medium after its next run of the role and the recreation or restart that follows it, as for the other labels; a host that publishes them already is recreated or restarted by the first run that adds this one. + +### The role directory and a former member + +The hosts that publish these labels (the controller is on, or the host is marked eligible or pinned) also get the role directory: `fleet-role.d` in the Compose project directory, which the role creates empty on every Docker host and `docker-compose.fleet-role.yml` mounts read-only into both node services at `/etc/github-runner`. It is a directory and not two files because whatever writes into it replaces each file in one rename, which gives the file a new inode, and a bind mount of a single file keeps showing the old one. A host that uses none of the controller's settings keeps its Compose configuration, so no container is recreated by an upgrade. The directory holds two files, each absent until something that changes the host's role writes it, and `files/k3s/node-entrypoint.sh` reads them before it starts Tailscale or k3s: + +- `role`, `server` or `agent`: a node service whose role the file does not name logs why and idles until it is stopped, so the server and agent services never run k3s together and a host whose role changed does not start the old one again through `restart: unless-stopped`. With no file each service starts as it always has. +- `join-url`, a server's URL (`https://:`): when it is not empty, the server joins that server instead of `K3S_JOIN_SERVER_URL` from `.env`, so a promoted host joins a current voter rather than the inventory's bootstrap host. + +Whatever the directory holds, a server also idles while its datastore holds a tombstone (`server/db/etcd/tombstone`). k3s writes it when etcd reports that this member was removed from the cluster (`pkg/daemons/executor/etcd.go` at v1.34.1+k3s1), and at its next start moves the datastore aside and joins again as a new learner (`Register` in `pkg/etcd/etcd.go`), so a removed server whose container restarted would become a member again behind the back of whatever removed it. The tombstone is the only record of the removal that a host down at the time has. A native server's unit refuses to start on the same tombstone (its `ExecCondition`, `files/native/fleet-role-is.sh`). + +A server never bootstraps a second cluster from a former member's host: with no join URL from either source, it passes `--cluster-init` when it has a datastore (`server/db`, which k3s reuses, ignoring the flag) or none at all (a first bootstrap), and when its datastore was moved aside by a demotion (`server/db.demoted` and no `server/db`) it does not start k3s and exits with a message, and its restart policy repeats the refusal until `join-url` names a current server. An idle service ends on `docker stop`'s signal like any other; idling never exits by itself, since the restart policy would start it again. + ## Networking Every node's k3s uses its mesh address for everything: its node IP, etcd's peer URLs, the API server and pod traffic. Which physical path the mesh takes between two nodes is Tailscale's choice, among the endpoints each node's Tailscale client can offer its peers. On Compose's default bridge network (`github_runner_cluster_network_mode: bridge`, the default) the client sits behind the Docker host's NAT and, inside a Docker VM, behind the VM's too, so it has no address its peers can reach and every pair of nodes is relayed, even on one LAN. @@ -486,4 +498,6 @@ or read from 1Password by adding `exadev.github_runner.github_runner_secrets_one `tests/unit/test_cluster_wait_nodes.py` runs the wait for nodes to be Ready and its rescue against a stand-in for `kubectl`: a node outside the inventory that is not Ready is reported and never waited for, and an inventory node that stays not Ready still fails the run. The mesh test does not cover it, since its nodes publish no API port for the host's `kubectl` to reach. +`tests/unit/test_node_entrypoint.py` runs the entrypoint's guards in a shell with a stand-in for `sleep` ([The role directory and a former member](#the-role-directory-and-a-former-member)): the join URL's precedence, `--cluster-init` with a datastore, with one moved aside and with none, the idling of a service its role file does not name and of a server on a tombstone, a host with no role file starting as before, and the override that mounts the directory. `tests/unit/test_native_server.py` runs the native server unit's condition on a tombstone. The `rejoin` mesh scenario does the same with real k3s and etcd: it removes the bootstrap Docker server's member through another server and checks the restarted server idles on its tombstone, that with its datastore moved aside and nothing to join it refuses to start, that a `join-url` naming another server makes it join the existing cluster as a learner that is promoted, with the same etcd cluster ID before and after, and that a role file naming the agent leaves the server container running nothing but its idle wait. + `tests/unit/test_node_password.py` covers [the node password volume and the servers' turns](#node-passwords-and-recreating-servers) without Docker: the filters in `plugins/filter/cluster_recreate.py`, the Compose file's volumes, and `tasks/node_password.yml`, `tasks/server_recreate_check.yml`, `tasks/server_ready.yml` and `tasks/start_servers.yml` run through the role (`tests/node_password/run.yml`) against a stand-in for `docker` that moves a real tar stream between its `docker cp` processes and keeps containers, volumes and node readiness as files, and a stand-in for the Compose module that recreates a container whose rendered hash differs and takes its node out of Ready until the role has waited for it. It checks the seeding (mode, ownership, labels, no copy of the password anywhere on the host, a failed copy leaving no volume), that `--check` changes nothing and reports each recreate, that a server is refused while another is not Ready and that agents are not, and from the order of the calls that the turns do not overlap. The `hosted` and `in_cluster` mesh scenarios recreate a joined server in Docker (`recreate_a_joined_server` in `tests/mesh/run.sh`) and check that its node rejoins with the same password and Secret, that no other container restarts, and that the role's `--check` report names exactly that server. diff --git a/roles/github_runner_cluster/defaults/main.yml b/roles/github_runner_cluster/defaults/main.yml index 0b65d229..413fd5bd 100644 --- a/roles/github_runner_cluster/defaults/main.yml +++ b/roles/github_runner_cluster/defaults/main.yml @@ -64,7 +64,7 @@ github_runner_cluster_voter_eligible_label: "github-runner.exadev.com/voter-elig github_runner_cluster_voter_pinned_label: "github-runner.exadev.com/voter-pinned" # The Node label the role publishes the medium of this host's link under, wired, wifi or unknown, which tasks/link.yml reads on the host: the medium of the interface a Mac's Colima VM is bridged to (or of the interface the Mac's default route leaves by, for a VM that is not bridged or a runtime that is not Colima), or of the interface a Linux host's default route leaves by. The role controller's probe reports it (github_runner_arc_role_controller_link_label defaults to this), since a node in a container or a VM cannot see the link beneath its own virtual interface. github_runner_cluster_link_label: "github-runner.exadev.com/link" -# Derived, never set directly: whether this host publishes the labels above, which it does while the role controller is on or the host is marked eligible or pinned, so that a fleet that uses none of them keeps its nodes' configuration as it is (a Docker node is not recreated by an upgrade). +# Derived, never set directly: whether this host publishes the labels above, which it does while the role controller is on or the host is marked eligible or pinned, so that a fleet that uses none of them keeps its nodes' configuration as it is (a Docker node is not recreated by an upgrade). The same hosts mount the role directory into a Docker node (docker-compose.fleet-role.yml). github_runner_cluster_publishes_node_labels: >- {{ github_runner_cluster_role_controller | bool or github_runner_cluster_voter_eligible | bool or github_runner_cluster_voter_pinned | bool }} # Derived, never set directly: this host's labels, as the key=value entries every runtime passes to k3s as --node-label, empty unless github_runner_cluster_publishes_node_labels. github_runner_cluster_link is the medium tasks/link.yml read on the host in this run. @@ -176,12 +176,13 @@ github_runner_cluster_network_probe_image: "busybox:1.37" # Extra Compose files merged after docker-compose.yml for this host's k3s project, as paths relative to github_runner_cluster_dir or absolute. Nothing needs them in normal use; the mesh integration tests use one to put several nodes on one Docker host. github_runner_cluster_compose_files: [] -# Derived, never set directly: every Compose file for this host's k3s project, in merge order: docker-compose.yml, then the override the headscale_in_cluster provider writes for the bootstrap server, then docker-compose.api-port.yml when github_runner_cluster_api_port is not the default, then docker-compose.node-labels.yml when the host has node labels, then github_runner_cluster_compose_files, then, with github_runner_cluster_network_mode host, docker-compose.host-network.yml, last so that its emptied port list wins over any file that publishes one. With no files beyond docker-compose.yml the role passes none at all and Compose reads it by itself, exactly as a plain `docker compose up` in github_runner_cluster_dir would. +# Derived, never set directly: every Compose file for this host's k3s project, in merge order: docker-compose.yml, then the override the headscale_in_cluster provider writes for the bootstrap server, then docker-compose.api-port.yml when github_runner_cluster_api_port is not the default, then docker-compose.node-labels.yml when the host has node labels, then docker-compose.fleet-role.yml when the host takes part in the role controller (github_runner_cluster_publishes_node_labels, the same hosts), then github_runner_cluster_compose_files, then, with github_runner_cluster_network_mode host, docker-compose.host-network.yml, last so that its emptied port list wins over any file that publishes one. With no files beyond docker-compose.yml the role passes none at all and Compose reads it by itself, exactly as a plain `docker compose up` in github_runner_cluster_dir would. github_runner_cluster_compose_file_list: >- {{ ['docker-compose.yml'] + (['docker-compose.mesh.yml'] if github_runner_cluster_mesh == 'headscale_in_cluster' and github_runner_cluster_effective_bootstrap | default(false) | bool else []) + (['docker-compose.api-port.yml'] if github_runner_cluster_api_port | int != github_runner_cluster_k3s_default_api_port | int else []) + (['docker-compose.node-labels.yml'] if github_runner_cluster_node_labels | length > 0 else []) + + (['docker-compose.fleet-role.yml'] if github_runner_cluster_publishes_node_labels | bool else []) + github_runner_cluster_compose_files + (['docker-compose.host-network.yml'] if github_runner_cluster_network_mode == 'host' else []) }} # Derived, never set directly: the Compose files this role writes into github_runner_cluster_dir itself, by the name the list above gives them. A user's own entry in github_runner_cluster_compose_files is not among them, and a missing one of those is an error in a check run as in a real one. @@ -189,6 +190,7 @@ github_runner_cluster_compose_files_written: - docker-compose.yml - docker-compose.api-port.yml - docker-compose.node-labels.yml + - docker-compose.fleet-role.yml - docker-compose.host-network.yml - docker-compose.mesh.yml # Derived, never set directly: the Compose files a Compose command run now can name. Every file in the list above in a real run, since the run has written the ones it owns before any Compose command; under --check, which writes nothing, tasks/compose_files_in_use.yml leaves out the ones this role would write and the host does not have yet. Compose refuses a command that names a file that is not there, and a check run that fails on one fails the README's promise that --check runs before every deploy. diff --git a/roles/github_runner_cluster/files/docker-compose.fleet-role.yml b/roles/github_runner_cluster/files/docker-compose.fleet-role.yml new file mode 100644 index 00000000..f603de85 --- /dev/null +++ b/roles/github_runner_cluster/files/docker-compose.fleet-role.yml @@ -0,0 +1,8 @@ +# Merged after docker-compose.yml by the github_runner_cluster role when this host takes part in the role controller (github_runner_cluster_publishes_node_labels in defaults/main.yml: the controller is on, or this host is marked a voter candidate or pinned): mounts the host's role directory, fleet-role.d in the Compose project directory, read-only into both services at /etc/github-runner, where k3s/node-entrypoint.sh reads the role the host runs and the server a promoted server joins. A directory rather than its files, because a file replaced by a rename has a new inode, which a bind mount of the file itself does not follow. Its own file, as docker-compose.node-labels.yml is, so that a host the controller never moves keeps the Compose configuration it has and its container is not recreated by an upgrade. +services: + k3s: + volumes: + - ./fleet-role.d:/etc/github-runner:ro + k3s-agent: + volumes: + - ./fleet-role.d:/etc/github-runner:ro diff --git a/roles/github_runner_cluster/files/docker-compose.yml b/roles/github_runner_cluster/files/docker-compose.yml index 27266747..becc6d07 100644 --- a/roles/github_runner_cluster/files/docker-compose.yml +++ b/roles/github_runner_cluster/files/docker-compose.yml @@ -4,7 +4,7 @@ # # Two roles, selected via Compose profiles rather than separate compose files: # - `--profile server`: a control-plane/etcd member (`k3s` service, running -# `k3s server`). Every server host runs the SAME service - whether it bootstraps the cluster (`--cluster-init`) or joins an existing one (`--server `) is decided at runtime by whether K3S_JOIN_SERVER_URL is set, not by a separate service/profile. See the project README's Architecture section for why 3 servers is the right minimum (etcd quorum math: 2 is worse than 1, not better) and why every server needs every server's own hostname on every server's own --tls-san list, not just its own - see K3S_TLS_SAN_LIST below. +# `k3s server`). Every server host runs the SAME service - whether it bootstraps the cluster (`--cluster-init`) or joins an existing one (`--server `) is decided at runtime by whether it has a server to join (K3S_JOIN_SERVER_URL, or the role directory's join-url that docker-compose.fleet-role.yml mounts), not by a separate service/profile. See the project README's Architecture section for why 3 servers is the right minimum (etcd quorum math: 2 is worse than 1, not better) and why every server needs every server's own hostname on every server's own --tls-san list, not just its own - see K3S_TLS_SAN_LIST below. # - `--profile agent`: a pure worker (`k3s-agent` service, running # `k3s agent`) - schedulable capacity with no control-plane role at all. # diff --git a/roles/github_runner_cluster/files/k3s/node-entrypoint.sh b/roles/github_runner_cluster/files/k3s/node-entrypoint.sh index 47702c1e..3836c3cf 100755 --- a/roles/github_runner_cluster/files/k3s/node-entrypoint.sh +++ b/roles/github_runner_cluster/files/k3s/node-entrypoint.sh @@ -1,7 +1,7 @@ #!/bin/sh # Entry point for the k3s and k3s-agent services in docker-compose.yml: starts Tailscale inside the node's own container, then k3s with its --vpn-auth integration, so each node joins the mesh as its own device. Bind-mounted from the Compose project directory the role manages (github_runner_cluster_dir) rather than baked into the k3s image, so a host picks up a change to it with the Compose file that uses it, without rebuilding an image Compose would not otherwise rebuild. # -# Usage: node-entrypoint.sh server|agent. Reads, from the container's environment (docker-compose.yml, from .env): K3S_VPN_AUTH_JOIN_KEY, K3S_VPN_AUTH_CONTROL_SERVER_URL (empty for Tailscale's own control plane, a Headscale URL otherwise), K3S_JOIN_SERVER_URL, K3S_TLS_SAN_LIST, K3S_DATASTORE, K3S_API_PORT, K3S_ETCD_HEARTBEAT_INTERVAL and K3S_ETCD_ELECTION_TIMEOUT (server), and K3S_MESH_SELF_HOSTED_HEALTH_URL (the bootstrap server of a cluster that runs its own Headscale, see supervise_self_hosted below). +# Usage: node-entrypoint.sh server|agent. Reads, from the container's environment (docker-compose.yml, from .env): K3S_VPN_AUTH_JOIN_KEY, K3S_VPN_AUTH_CONTROL_SERVER_URL (empty for Tailscale's own control plane, a Headscale URL otherwise), K3S_JOIN_SERVER_URL, K3S_TLS_SAN_LIST, K3S_DATASTORE, K3S_API_PORT, K3S_ETCD_HEARTBEAT_INTERVAL and K3S_ETCD_ELECTION_TIMEOUT (server), and K3S_MESH_SELF_HOSTED_HEALTH_URL (the bootstrap server of a cluster that runs its own Headscale, see supervise_self_hosted below); and, when docker-compose.fleet-role.yml mounts it, the role directory's `role` and `join-url` (fleet_role_dir below). # No set -e: a failed `tailscale up` falls through to waiting for an address, as it always has, rather than exiting and restarting the container. set -u @@ -65,6 +65,58 @@ set_regrade_flags() { printf '%s\n' "$regrade_level" > "$regrade_started_file" } +# The host's record of the role it runs, server or agent, and of the server a promoted server joins: the directory docker-compose.fleet-role.yml bind-mounts read-only from the Compose project directory (fleet-role.d), holding the files `role` and `join-url`, each absent until something writes it. A directory and not two files, because whatever writes them replaces each in one rename, which gives the file a new inode, and a bind mount of a single file keeps showing the old one. The variables replace the paths only in tests. +fleet_role_dir=${FLEET_ROLE_DIR:-/etc/github-runner} +k3s_server_dir=${K3S_SERVER_DIR:-/var/lib/rancher/k3s/server} + +# Runs nothing until the container is stopped, after saying why. Never exits by itself, since restart: unless-stopped starts a container that exits again, and with it this check, for ever. The shell stays PID 1 and waits on a sleep in the background, rather than exec'ing sleep, because the kernel delivers PID 1 only the signals it has a handler for: an exec'd sleep would ignore `docker stop`'s SIGTERM and be killed only once Docker's stop timeout ran out. The loop runs the wait again should the sleep ever end, so nothing after a call of this ever runs. +idle() { + echo "github-runner-node: $1, so k3s is not started; the container idles until it is stopped" >&2 + trap 'exit 0' TERM INT + while :; do + sleep infinity & + wait $! + done +} + +# Idles when the host is not to run this service's k3s at all: when the role file names another role than this service's (the two services never run together, and a host whose role changed does not start the old one again by its restart policy), or, for a server, when k3s has marked its datastore with a tombstone. k3s writes server/db/etcd/tombstone when etcd tells it this member was removed from the cluster (pkg/daemons/executor/etcd.go at v1.34.1+k3s1), and at its next start moves the datastore aside and joins again as a new learner (Register in pkg/etcd/etcd.go), so a retired server whose container restarted (by its restart policy, a host reboot, a recreate) would become a member again behind the back of whatever removed it; the tombstone is the only record of the removal a host that was down at the time has. A host with no role file runs the service it was started as, as every host did before the file existed. +fleet_role_guard() { + if [ -e "$fleet_role_dir/role" ]; then + recorded_role=$(cat "$fleet_role_dir/role" 2>/dev/null) || recorded_role="" + if [ "$recorded_role" != "$1" ]; then + idle "$fleet_role_dir/role names '${recorded_role}', not $1" + fi + fi + if [ "$1" = server ] && [ -e "$k3s_server_dir/db/etcd/tombstone" ]; then + idle "$k3s_server_dir/db/etcd/tombstone says this server's etcd member was removed from the cluster, and k3s started now would join it again as a new member" + fi +} + +# Sets join_url to the URL of the server a server joins: the role directory's join-url when it is not empty, which names a current voter when a promotion has written one, otherwise K3S_JOIN_SERVER_URL from .env, the inventory's bootstrap server, empty on the bootstrap host itself. +set_join_url() { + join_url=$(cat "$fleet_role_dir/join-url" 2>/dev/null) || join_url="" + if [ -z "$join_url" ]; then + join_url=${K3S_JOIN_SERVER_URL:-} + fi +} + +# Exits, without starting k3s, a server that has no server to join and whose datastore a demotion moved aside (server/db.demoted, with no server/db). It was a member of a cluster that still exists, and with nothing to join k3s would bootstrap a second one (--cluster-init, or a fresh SQLite datastore), splitting the fleet. Exits rather than idling, because it can only be wrong: the container restarts and refuses again until a join-url names a current voter. With server/db present k3s reuses the datastore and ignores --cluster-init; with neither, this is a first bootstrap. +refuse_second_cluster() { + if [ -z "$join_url" ] && [ ! -e "$k3s_server_dir/db" ] && [ -e "$k3s_server_dir/db.demoted" ]; then + echo "github-runner-node: this host was a member of a cluster ($k3s_server_dir/db.demoted, no $k3s_server_dir/db) and has no server to join, so starting k3s would bootstrap a second cluster; not starting it. Write the URL of a current server (https://:) to join-url in the role directory ($fleet_role_dir on this container, fleet-role.d in the Compose project directory)." >&2 + exit 1 + fi +} + +# Before anything else starts, Tailscale included: a host that is not to run this service must not bring this service's Tailscale device up either. +case "$role" in + server | agent) fleet_role_guard "$role" ;; +esac +if [ "$role" = server ]; then + set_join_url + refuse_second_cluster +fi + # k3s's own vpn-auth integration only runs `tailscale up`; it does not start the daemon itself (without this, k3s fails outright with "tailscale up failed: ... it doesn't appear to be running"). Waits for the control socket before anything talks to it. mkdir -p /var/run/tailscale /var/lib/tailscale tailscaled --state=/var/lib/tailscale/tailscaled.state --socket=/var/run/tailscale/tailscaled.sock & @@ -114,7 +166,7 @@ join_mesh() { fi } -# The host a joining node dials, from its server URL (https://:): K3S_JOIN_SERVER_URL for a server, K3S_URL for an agent. Empty for the bootstrap server, which joins nothing. +# The host a joining node dials, from its server URL (https://:): join_url for a server, K3S_URL for an agent. Empty for the bootstrap server, which joins nothing. join_host_of() { join_host=${1#https://} join_host=${join_host%%:*} @@ -160,7 +212,7 @@ install_tailscale_up_wait() { # # --node-ip forces etcd's peer-advertise address onto this node's mesh IP. k3s builds etcd's --initial-cluster from config.PrivateIP, which is populated from --node-ip before --vpn-auth's own executor code rebuilds NodeIP from its Tailscale detection (traced through pkg/cli/server/server.go and pkg/executor/embed/embed.go). Without it PrivateIP stays the container's bridge IP, which no other machine can reach, and joining a second server failed with "MemberAdd request timed out". An --etcd-arg override is not a substitute: it also applies to the temporary etcd k3s starts on every restart to reconcile with its datastore, which has no TLS certificates for a 0.0.0.0:2380 listener and failed with "cannot listen on TLS for [::]:2380: KeyFile and CertFile are not presented". # -# --cluster-init (only when K3S_JOIN_SERVER_URL is unset, and K3S_DATASTORE is not sqlite) bootstraps embedded etcd, or converts an existing single-node SQLite datastore to it, the one migration k3s documents; once initialised, k3s ignores it on restart, so it stays in place permanently. A host with K3S_JOIN_SERVER_URL set passes --server instead and joins as another control-plane/etcd member. +# --cluster-init (only when join_url, set by set_join_url above, is empty, and K3S_DATASTORE is not sqlite) bootstraps embedded etcd, or converts an existing single-node SQLite datastore to it, the one migration k3s documents; once initialised, k3s ignores it on restart, so it stays in place permanently. refuse_second_cluster above has already stopped a former member with no join URL, so here it is a first bootstrap or a datastore k3s reuses. A host with a join URL passes --server instead and joins as another control-plane/etcd member. # # --https-listen-port is added when K3S_API_PORT is set (set_api_port_flags above), for a second cluster on a Docker host whose default port another cluster already publishes. # @@ -175,8 +227,8 @@ mesh_server_args() { --resolv-conf=/etc/k3s-resolv.conf \ --node-ip="$mesh_ip" for flag in $api_port_flags; do set -- "$@" "$flag"; done - if [ -n "${K3S_JOIN_SERVER_URL:-}" ]; then - set -- "$@" --server "$K3S_JOIN_SERVER_URL" + if [ -n "$join_url" ]; then + set -- "$@" --server "$join_url" elif [ "${K3S_DATASTORE:-etcd}" != sqlite ]; then set -- "$@" --cluster-init fi @@ -241,7 +293,7 @@ case "$role" in supervise_self_hosted fi join_mesh - server_join_host=$(join_host_of "${K3S_JOIN_SERVER_URL:-}") + server_join_host=$(join_host_of "$join_url") wait_for_mesh_host "$server_join_host" install_tailscale_up_wait "$server_join_host" "$tailscale_up_wait_dir" /usr/local/bin/tailscale rm -f /output/kubeconfig.yaml diff --git a/roles/github_runner_cluster/files/native/fleet-role-is.sh b/roles/github_runner_cluster/files/native/fleet-role-is.sh index 7a2d54ea..5e07313e 100755 --- a/roles/github_runner_cluster/files/native/fleet-role-is.sh +++ b/roles/github_runner_cluster/files/native/fleet-role-is.sh @@ -1,15 +1,24 @@ #!/bin/sh # The ExecCondition= of both native k3s units (templates/native/k3s.service.j2 and k3s-agent.service.j2): succeeds only when the role file names the role the unit runs, so the server and the agent unit can never run together, and Restart=always can never start a unit whose role the host no longer has (a demoted server, say). systemd skips the rest of a unit's start, without marking it failed, when its ExecCondition= exits 1 to 254 (systemd.service(5), ExecCondition=, systemd 243 and later), and Restart=always does not restart a unit whose start was skipped that way (service_shall_restart in src/core/service.c at v255, the systemd of Ubuntu 24.04: `case SERVICE_RESTART_ALWAYS: return s->result != SERVICE_SKIP_CONDITION;`). # -# Usage: fleet-role-is.sh ROLE ROLE_FILE. ROLE is server or agent; ROLE_FILE holds one of them, written by the cluster role (tasks/native/role.yml) and, later, by whatever changes the host's role. A missing or unreadable file names no role, so neither unit starts until something has decided one. +# The server unit also does not start while k3s's datastore holds a tombstone. k3s writes SERVER_DIR/db/etcd/tombstone when etcd tells it this member was removed from the cluster (pkg/daemons/executor/etcd.go at v1.34.1+k3s1), and at its next start moves the datastore aside and joins again as a new learner (Register in pkg/etcd/etcd.go), so a server removed while the role file still names it a server (removed while it was down, or before anything switched the file) would become a member again behind the back of whatever removed it. Finishing the demotion moves server/db aside, tombstone and all. +# +# Usage: fleet-role-is.sh server ROLE_FILE SERVER_DIR, or fleet-role-is.sh agent ROLE_FILE. ROLE_FILE holds server or agent, written by the cluster role (tasks/native/role.yml) and by whatever changes the host's role. A missing or unreadable file names no role, so neither unit starts until something has decided one. SERVER_DIR is k3s's server directory (its data directory's server). set -u -role=${1:?usage: fleet-role-is.sh ROLE ROLE_FILE} -file=${2:?usage: fleet-role-is.sh ROLE ROLE_FILE} +role=${1:?usage: fleet-role-is.sh server ROLE_FILE SERVER_DIR | fleet-role-is.sh agent ROLE_FILE} +file=${2:?usage: fleet-role-is.sh server ROLE_FILE SERVER_DIR | fleet-role-is.sh agent ROLE_FILE} recorded=$(cat "$file" 2>/dev/null) || recorded="" -if [ "$recorded" = "$role" ]; then - exit 0 +if [ "$recorded" != "$role" ]; then + echo "fleet-role-is: ${file} names '${recorded:-no role}', not ${role}, so this unit does not start" >&2 + exit 1 +fi +if [ "$role" = server ]; then + server_dir=${3:?usage: fleet-role-is.sh server ROLE_FILE SERVER_DIR} + if [ -e "${server_dir}/db/etcd/tombstone" ]; then + echo "fleet-role-is: ${server_dir}/db/etcd/tombstone says this server's etcd member was removed from the cluster, and k3s started now would join it again as a new member, so this unit does not start" >&2 + exit 1 + fi fi -echo "fleet-role-is: ${file} names '${recorded:-no role}', not ${role}, so this unit does not start" >&2 -exit 1 +exit 0 diff --git a/roles/github_runner_cluster/tasks/main.yml b/roles/github_runner_cluster/tasks/main.yml index 54e91fd2..0e4b0b2e 100644 --- a/roles/github_runner_cluster/tasks/main.yml +++ b/roles/github_runner_cluster/tasks/main.yml @@ -86,7 +86,7 @@ ansible.builtin.include_tasks: instance_check.yml - name: Create this host's Compose project directory - # The kubeconfig directory is the connecting user's alone: the kubeconfig k3s writes into it holds the cluster's admin credentials, and is readable by others so that this user can read it where the container writes it as root (see --write-kubeconfig-mode in k3s/node-entrypoint.sh). + # The kubeconfig directory is the connecting user's alone: the kubeconfig k3s writes into it holds the cluster's admin credentials, and is readable by others so that this user can read it where the container writes it as root (see --write-kubeconfig-mode in k3s/node-entrypoint.sh). fleet-role.d is the host's role directory, created empty, which docker-compose.fleet-role.yml mounts read-only into both services: whatever changes the host's role writes `role` and `join-url` into it, and the entrypoint reads them (an absent file changes nothing). Created on every host, so that it exists before any Compose command that mounts it, which would otherwise have Docker create it as root. ansible.builtin.file: path: "{{ github_runner_cluster_dir }}/{{ item.path }}" state: directory @@ -95,6 +95,7 @@ - {path: "", mode: "0755"} - {path: k3s, mode: "0755"} - {path: kubeconfig, mode: "0700"} + - {path: fleet-role.d, mode: "0755"} loop_control: label: "{{ item.path }}" @@ -108,6 +109,7 @@ - {src: docker-compose.yml, mode: "0644"} - {src: docker-compose.api-port.yml, mode: "0644"} - {src: docker-compose.node-labels.yml, mode: "0644"} + - {src: docker-compose.fleet-role.yml, mode: "0644"} - {src: docker-compose.host-network.yml, mode: "0644"} - {src: k3s/Dockerfile, mode: "0644"} - {src: k3s/node-entrypoint.sh, mode: "0755"} diff --git a/roles/github_runner_cluster/templates/native/k3s.service.j2 b/roles/github_runner_cluster/templates/native/k3s.service.j2 index 16fe3cbb..32e6c427 100644 --- a/roles/github_runner_cluster/templates/native/k3s.service.j2 +++ b/roles/github_runner_cluster/templates/native/k3s.service.j2 @@ -29,8 +29,8 @@ RestartSec=5s # Kept across restarts, so the file the kubelet reads is there while k3s starts again; removed when the service stops, and when a start is skipped by the condition below, so it is this unit's alone and never shared with the other k3s unit. RuntimeDirectory={{ github_runner_cluster_native_server_runtime_dir }} RuntimeDirectoryPreserve=restart -# Starts only while the role file names this host a server (files/native/fleet-role-is.sh), so this unit and the k3s agent unit never run together and Restart=always never starts a server that has been demoted, which would join the cluster again as a learner. -ExecCondition={{ github_runner_cluster_native_lib_dir }}/fleet-role-is server {{ github_runner_cluster_native_role_file }} +# Starts only while the role file names this host a server and k3s has not marked its datastore with a tombstone (files/native/fleet-role-is.sh), so this unit and the k3s agent unit never run together and Restart=always never starts a server that has been demoted or whose etcd member was removed, either of which would join the cluster again as a learner. +ExecCondition={{ github_runner_cluster_native_lib_dir }}/fleet-role-is server {{ github_runner_cluster_native_role_file }} {{ github_runner_cluster_native_data_dir }}/server ExecStartPre=-/sbin/modprobe br_netfilter ExecStartPre=-/sbin/modprobe overlay ExecStartPre={{ github_runner_cluster_native_lib_dir }}/write-resolv-conf {{ github_runner_cluster_native_run_dir }}/{{ github_runner_cluster_native_server_runtime_dir }}/resolv.conf diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index 2e3ed9a6..e5c0372d 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Mesh integration test for the github_runner_cluster role: stands up three k3s nodes in Docker on one machine, each in its own copy of the Compose project, and has the role join them through Headscale. Scenarios: hosted headscale_hosted: the role runs Headscale under Compose beside the first node. in_cluster headscale_in_cluster: the role runs Headscale inside the cluster, on the bootstrap server. existing headscale_existing: the policy check fails, with the entry to add, against a server whose policy lacks autoApprovers, and passes once it has one. Every scenario uses automatic server selection (three hosts, no overrides, so three servers). native_server three Docker servers and a native host (a privileged systemd container, tests/mesh/native/Dockerfile, managed over Ansible's docker connection) that joins as an agent, is then made a server by the inventory and promoted in place: it must join etcd as a learner, be promoted to a voter with its mesh address as its peer URL, and keep the cluster writable throughout, and a rerun must not restart it. role_controller four hosts (three servers and a standby agent, with etcd's data on memory so the machine's disk does not decide): the role controller's probe and controller run in the cluster from the image built from this checkout, with the manifests the ARC role renders; every host is reported fit from a full window of real samples, with the link the cluster role labelled its node with, a voter made slow with tc netem on its mesh interface is reported unfit with the swap named and every unfit verdict attributed to it, a paused voter makes the controller report that it would not act, and the controller changes nothing (not in the default set; needs jq, and takes a window of 100 samples before it can judge). compose_faulty the role refuses a Docker Compose release that recreates the containers it has just built, before writing anything (run it under such a release; it is not in the default set). host_network each node runs in a Docker daemon of its own (a docker:dind container standing in for a VM with an address on the LAN, on the shared test network) with github_runner_cluster_network_mode host, joined through a Headscale server the test runs (headscale_existing); the nodes' Tailscale clients must then reach each other directly at those daemons' addresses, and the role must not publish any port; then the role must refuse to move two running nodes to the bridge network in one run, and move one there and back through its guard, the node keeping its mesh address and etcd membership. Linux only, since the control node reaches each daemon at its address on the test network. The cluster scenarios assert that etcd has three members, that every node is Ready, and that pods on different nodes reach each other over the mesh. +# Mesh integration test for the github_runner_cluster role: stands up three k3s nodes in Docker on one machine, each in its own copy of the Compose project, and has the role join them through Headscale. Scenarios: hosted headscale_hosted: the role runs Headscale under Compose beside the first node. in_cluster headscale_in_cluster: the role runs Headscale inside the cluster, on the bootstrap server. existing headscale_existing: the policy check fails, with the entry to add, against a server whose policy lacks autoApprovers, and passes once it has one. Every scenario uses automatic server selection (three hosts, no overrides, so three servers). native_server three Docker servers and a native host (a privileged systemd container, tests/mesh/native/Dockerfile, managed over Ansible's docker connection) that joins as an agent, is then made a server by the inventory and promoted in place: it must join etcd as a learner, be promoted to a voter with its mesh address as its peer URL, and keep the cluster writable throughout, and a rerun must not restart it. role_controller four hosts (three servers and a standby agent, with etcd's data on memory so the machine's disk does not decide): the role controller's probe and controller run in the cluster from the image built from this checkout, with the manifests the ARC role renders; every host is reported fit from a full window of real samples, with the link the cluster role labelled its node with, a voter made slow with tc netem on its mesh interface is reported unfit with the swap named and every unfit verdict attributed to it, a paused voter makes the controller report that it would not act, and the controller changes nothing (not in the default set; needs jq, and takes a window of 100 samples before it can judge). rejoin three Docker servers whose hosts the inventory marks eligible, so the role mounts each one's role directory: the bootstrap server's etcd member is removed through another server, and restarted on its tombstone it idles instead of joining again; with its datastore moved aside and no server to join it refuses to start rather than bootstrap a second cluster; given a join URL naming another server in its role directory it joins the existing cluster as a learner and is promoted, with the same cluster ID before and after; and a role file naming the agent leaves its server container idle with no k3s. compose_faulty the role refuses a Docker Compose release that recreates the containers it has just built, before writing anything (run it under such a release; it is not in the default set). host_network each node runs in a Docker daemon of its own (a docker:dind container standing in for a VM with an address on the LAN, on the shared test network) with github_runner_cluster_network_mode host, joined through a Headscale server the test runs (headscale_existing); the nodes' Tailscale clients must then reach each other directly at those daemons' addresses, and the role must not publish any port; then the role must refuse to move two running nodes to the bridge network in one run, and move one there and back through its guard, the node keeping its mesh address and etcd membership. Linux only, since the control node reaches each daemon at its address on the test network. The cluster scenarios assert that etcd has three members, that every node is Ready, and that pods on different nodes reach each other over the mesh. # # With GRTEST_PIN_CONTEXT=1, the cluster scenarios run the role, and the in_cluster recovery playbook, with github_runner_cluster_docker_context pinned while the current Docker context they see points at nothing (tests/lib/docker_context.sh), so any docker call that ignored the pin fails the run; afterwards the current context must be unchanged. context_checks the role refuses a pinned context that does not exist, and a pinned context alongside DOCKER_HOST, before writing anything (not in the default set). # @@ -244,11 +244,11 @@ EOF echo " github_runner_cluster_container_name: grtest-node${index}-k3s" # Used only by a host the topology makes an agent (the role_controller scenario has one); the role refuses one name for both services. echo " github_runner_cluster_agent_container_name: grtest-node${index}-agent" - # The role_controller scenario marks every host a candidate for a vote, through the inventory, as an owner would, and pins the first. - if [ "${role_controller:-0}" = 1 ]; then + # The role_controller scenario marks every host a candidate for a vote, through the inventory, as an owner would, and pins the first; the rejoin scenario marks them without turning the controller on, which is enough for the role to mount each host's role directory. + if [ "${role_controller:-0}" = 1 ] || [ "${voters_eligible:-0}" = 1 ]; then echo " github_runner_cluster_voter_eligible: true" - if [ "$index" = 1 ]; then echo " github_runner_cluster_voter_pinned: true"; fi fi + if [ "${role_controller:-0}" = 1 ] && [ "$index" = 1 ]; then echo " github_runner_cluster_voter_pinned: true"; fi done if [ -n "$native_role" ]; then echo " ${native_host}:" @@ -1009,10 +1009,10 @@ docker_servers_serving() { native_node_ready() { [ "$(kubectl_on 1 get node "$native_host" --no-headers 2>/dev/null | awk '{print $2}')" = Ready ]; } native_unit_is() { [ "$(native_exec systemctl is-active "$1" /dev/null)" = "$2" ]; } -# The etcd member list, as assert_cluster reads it, through node 1. +# The etcd member list, as assert_cluster reads it, through node 1 or the node given. etcd_members() { docker rm -f grtest-etcdctl >/dev/null 2>&1 || true - docker run --rm --name grtest-etcdctl --network "container:$(container 1)" --volumes-from "$(container 1)" --entrypoint etcdctl "$etcd_image" \ + docker run --rm --name grtest-etcdctl --network "container:$(container "${1:-1}")" --volumes-from "$(container "${1:-1}")" --entrypoint etcdctl "$etcd_image" \ --endpoints https://127.0.0.1:2379 --cacert /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \ --cert /var/lib/rancher/k3s/server/tls/etcd/client.crt --key /var/lib/rancher/k3s/server/tls/etcd/client.key \ member list -w simple /dev/null; then fail "the fourth node is a server"; fi ids=$(etcd_member_ids 1) - log "A check run on hosts that lack the node-labels Compose file, as a host that ran the role before the controller was turned on does, passes and writes nothing" - local before_started + log "A check run on hosts that lack the Compose files of the role controller, as a host that ran the role before the controller was turned on does, passes and writes nothing" + local before_started file before_started="$(started_at)" - for index in $(seq 1 "$node_count"); do rm -f "${work}/$(node_name "$index")/docker-compose.node-labels.yml"; done - run_role --check || fail "the role's check run failed on hosts without docker-compose.node-labels.yml" for index in $(seq 1 "$node_count"); do - [ ! -e "${work}/$(node_name "$index")/docker-compose.node-labels.yml" ] || fail "the check run wrote docker-compose.node-labels.yml on $(node_name "$index")" + for file in docker-compose.node-labels.yml docker-compose.fleet-role.yml; do rm -f "${work}/$(node_name "$index")/${file}"; done + done + run_role --check || fail "the role's check run failed on hosts without the role controller's Compose files" + for index in $(seq 1 "$node_count"); do + for file in docker-compose.node-labels.yml docker-compose.fleet-role.yml; do + [ ! -e "${work}/$(node_name "$index")/${file}" ] || fail "the check run wrote ${file} on $(node_name "$index")" + done done [ "$before_started" = "$(started_at)" ] || fail "the check run restarted a node" - log "The real run after it writes the file first, then names it, and recreates no container" + log "The real run after it writes the files first, then names them, and recreates no container" run_role for index in $(seq 1 "$node_count"); do - [ -e "${work}/$(node_name "$index")/docker-compose.node-labels.yml" ] || fail "the real run did not write docker-compose.node-labels.yml on $(node_name "$index")" + for file in docker-compose.node-labels.yml docker-compose.fleet-role.yml; do + [ -e "${work}/$(node_name "$index")/${file}" ] || fail "the real run did not write ${file} on $(node_name "$index")" + done done [ "$before_started" = "$(started_at)" ] || fail "the real run restarted a node whose configuration it left as it was" @@ -1421,6 +1427,113 @@ EOF native_unit_is fleet-regrade inactive || fail "fleet-regrade still runs" } +# --- A former member of the cluster (the rejoin scenario) --- + +# The etcd cluster ID as node $1's own etcd reports it, in etcdctl's fields output, which prints the 64-bit ID as an integer (a JSON reader would round it to a float). +etcd_cluster_id() { + docker rm -f grtest-etcdctl >/dev/null 2>&1 || true + docker run --rm --name grtest-etcdctl --network "container:$(container "$1")" --volumes-from "$(container "$1")" --entrypoint etcdctl "$etcd_image" \ + --endpoints https://127.0.0.1:2379 --cacert /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \ + --cert /var/lib/rancher/k3s/server/tls/etcd/client.crt --key /var/lib/rancher/k3s/server/tls/etcd/client.key \ + endpoint status -w fields &1 | grep -F "$2" >/dev/null; } +removed_member_recorded() { [ -n "$(kubectl_on 2 get node "$(node_name 1)" -o 'jsonpath={.metadata.annotations.etcd\.k3s\.cattle\.io/removed-node-name}' 2>/dev/null)" ]; } +has_tombstone() { docker exec "$(container 1)" test -e /var/lib/rancher/k3s/server/db/etcd/tombstone /dev/null; } +started_members() { etcd_members 2 2>/dev/null | grep -c ', started, ' || true; } +node1_voter_started() { etcd_members 2 2>/dev/null | grep -F ", $(node_name 1)-" | grep -E ', started, .*, false$' >/dev/null; } +node1_is_voter() { [ "$(kubectl_on 2 get node "$(node_name 1)" -o 'jsonpath={.status.conditions[?(@.type=="EtcdIsVoter")].status}' 2>/dev/null)" = True ]; } +# Whether node 1's container has been started again by its restart policy since it was last started by hand. +node1_restarted() { [ "$(docker inspect --format '{{.RestartCount}}' "$(container 1)")" -gt 0 ]; } +# The command names of node 1's processes but the ones this read runs, one per line. +node1_processes() { + # shellcheck disable=SC2016 # Expanded by the container's shell, which leaves out itself and the cat it runs. + docker exec "$(container 1)" sh -c 'for p in /proc/[0-9]*; do [ "${p#/proc/}" = "$$" ] || cat "$p/comm" 2>/dev/null; done' /dev/null + wait_for "k3s to record the removal of $(node_name 1)'s member" 300 removed_member_recorded + wait_for "$(node_name 1)'s k3s to write its tombstone" 120 has_tombstone + [ "$(started_members)" = 2 ] || { etcd_members 2; fail "expected two started etcd members after the removal"; } + + log "Restarted, the removed server idles on its tombstone instead of joining again" + docker restart "$(container 1)" >/dev/null + wait_for "$(node_name 1) to say it idles on its tombstone" 120 container_logged 1 "tombstone says this server's etcd member was removed from the cluster, and k3s started now would join it again as a new member, so k3s is not started" + wait_for "$(node_name 1) to run nothing but the idle wait" 60 node1_idles + [ "$(started_members)" = 2 ] || { etcd_members 2; fail "the removed server joined etcd again"; } + [ "$(etcd_members 2 | wc -l | tr -d ' ')" = 2 ] || { etcd_members 2; fail "the removed server is a member again"; } + + log "With its datastore moved aside and no server to join, it refuses to start rather than bootstrap a second cluster" + docker stop "$(container 1)" >/dev/null + # The trap in the idle wait ends it on docker stop's SIGTERM; without it Docker would kill the container once its stop timeout ran out. + [ "$(docker inspect --format '{{.State.ExitCode}}' "$(container 1)")" = 0 ] || fail "the idle container did not exit 0 on docker stop's SIGTERM" + # The annotations the removal left are cleared as a demotion clears them, so the member that joins next is not taken for the one removed. + kubectl_on 2 annotate node "$(node_name 1)" etcd.k3s.cattle.io/remove- etcd.k3s.cattle.io/removed-node-name- >/dev/null + docker run --rm -v "$(node_name 1)_k3s-data:/data" "$busybox_image" mv /data/server/db /data/server/db.demoted + refused_since=$(date -u +%Y-%m-%dT%H:%M:%SZ) + docker start "$(container 1)" >/dev/null + wait_for "$(node_name 1) to refuse to bootstrap a second cluster" 120 container_logged 1 "has no server to join, so starting k3s would bootstrap a second cluster; not starting it" + # The refusal exits, so the restart policy keeps starting the container and each start refuses again; no start may get as far as k3s, whose first line names its version. + if docker logs --since "$refused_since" "$(container 1)" 2>&1 | grep -F 'Starting k3s v' >/dev/null; then fail "$(node_name 1) started k3s with no server to join"; fi + wait_for "$(node_name 1) to exit on its refusal and be started again by its restart policy" 120 node1_restarted + [ "$(etcd_members 2 | wc -l | tr -d ' ')" = 2 ] || { etcd_members 2; fail "the members changed while $(node_name 1) refused to start"; } + [ "$(etcd_cluster_id 2)" = "$cluster_id" ] || fail "the cluster ID changed while $(node_name 1) refused to start" + + log "Given a join URL naming $(node_name 2) in its role directory, it joins the cluster that exists" + printf 'https://%s.mesh.grtest.internal:6443\n' "$(node_name 2)" > "${work}/$(node_name 1)/fleet-role.d/join-url" + docker restart "$(container 1)" >/dev/null + wait_for "$(node_name 1) to be a started voting member of etcd again" 600 node1_voter_started + wait_for "$(node_name 1)'s Node to report EtcdIsVoter" 300 node1_is_voter + local promoted=0 index + for index in 2 3; do + if container_logged "$index" "Promoted learner $(node_name 1)-"; then promoted=1; fi + done + [ "$promoted" = 1 ] || fail "no other server's k3s logged promoting $(node_name 1)'s learner" + [ "$(started_members)" = 3 ] || { etcd_members 2; fail "expected three started etcd members after the rejoin"; } + [ "$(etcd_cluster_id 1)" = "$cluster_id" ] || fail "$(node_name 1) joined a cluster with another ID: $(etcd_cluster_id 1) (was ${cluster_id})" + [ "$(node_password_hash 1)" = "$password" ] || fail "$(node_name 1)'s node password changed" + docker run --rm -v "$(node_name 1)_k3s-data:/data" "$busybox_image" test -d /data/server/db.demoted || fail "the moved-aside datastore is gone" + assert_cluster + + log "A role file naming the agent leaves the server container idle, with no k3s" + printf 'agent\n' > "${work}/$(node_name 1)/fleet-role.d/role" + docker restart "$(container 1)" >/dev/null + wait_for "$(node_name 1) to say its role file names the agent" 120 container_logged 1 "/etc/github-runner/role names 'agent', not server, so k3s is not started" + wait_for "$(node_name 1) to run nothing but the idle wait" 60 node1_idles + started_before=$(docker inspect --format '{{.State.StartedAt}} {{.RestartCount}}' "$(container 1)") + node1_idles || { node1_processes; fail "$(node_name 1) runs more than the idle wait"; } + [ "$(docker inspect --format '{{.State.Running}}' "$(container 1)")" = true ] || fail "the idle container is not running" + [ "$(docker inspect --format '{{.State.StartedAt}} {{.RestartCount}}' "$(container 1)")" = "$started_before" ] || fail "the idle container restarted" +} + scenarios=("$@") [ "${#scenarios[@]}" -gt 0 ] || scenarios=(existing hosted in_cluster) for scenario in "${scenarios[@]}"; do @@ -1431,6 +1544,7 @@ for scenario in "${scenarios[@]}"; do regrade) scenario_regrade ;; native_server) scenario_native_server ;; role_controller) scenario_role_controller ;; + rejoin) scenario_rejoin ;; host_network) scenario_host_network ;; compose_faulty) scenario_compose_faulty ;; context_checks) scenario_context_checks ;; diff --git a/tests/unit/test_cluster_etcd_timing.py b/tests/unit/test_cluster_etcd_timing.py index c13771d7..afe7718f 100644 --- a/tests/unit/test_cluster_etcd_timing.py +++ b/tests/unit/test_cluster_etcd_timing.py @@ -185,6 +185,8 @@ def server_args(self, builder: str, **env: str) -> list[str]: 'vpn_auth="name=tailscale,joinKey=example"', 'mesh_ip="100.64.0.1"', 'regrade_flags=""', + # The server's join URL as set_join_url chooses it when the role directory names none (test_node_entrypoint.py tests the choice). + 'join_url="${K3S_JOIN_SERVER_URL:-}"', function_source("set_etcd_timing_flags"), function_source("set_api_port_flags"), function_source(builder), diff --git a/tests/unit/test_cluster_instance.py b/tests/unit/test_cluster_instance.py index 7f552157..e13407dd 100644 --- a/tests/unit/test_cluster_instance.py +++ b/tests/unit/test_cluster_instance.py @@ -244,6 +244,8 @@ def server_args(self, builder: str, **env: str) -> list[str]: 'vpn_auth="name=tailscale,joinKey=example"', 'mesh_ip="100.64.0.1"', 'regrade_flags=""', + # The server's join URL as set_join_url chooses it when the role directory names none (test_node_entrypoint.py tests the choice). + 'join_url="${K3S_JOIN_SERVER_URL:-}"', function_source("set_etcd_timing_flags"), function_source("set_api_port_flags"), function_source(builder), diff --git a/tests/unit/test_cluster_join_wait.py b/tests/unit/test_cluster_join_wait.py index f4bf7b64..23c68e6b 100644 --- a/tests/unit/test_cluster_join_wait.py +++ b/tests/unit/test_cluster_join_wait.py @@ -129,7 +129,8 @@ def case_body(self, role: str) -> str: def test_a_joining_server_waits_before_k3s_and_starts_k3s_with_the_wrapper_first_on_its_path(self) -> None: body = self.case_body("server") - self.assertIn('server_join_host=$(join_host_of "${K3S_JOIN_SERVER_URL:-}")', body) + # join_url is the role directory's join-url or K3S_JOIN_SERVER_URL (set_join_url, tested in test_node_entrypoint.py). + self.assertIn('server_join_host=$(join_host_of "$join_url")', body) self.assertLess(body.index('wait_for_mesh_host "$server_join_host"'), body.index("exec /bin/k3s server")) self.assertLess(body.index('install_tailscale_up_wait "$server_join_host" "$tailscale_up_wait_dir" /usr/local/bin/tailscale'), body.index("exec /bin/k3s server")) self.assertIn('PATH="$tailscale_up_wait_dir:$PATH" exec /bin/k3s server', body) diff --git a/tests/unit/test_cluster_kubeconfig.py b/tests/unit/test_cluster_kubeconfig.py index dde5a795..e92f0375 100644 --- a/tests/unit/test_cluster_kubeconfig.py +++ b/tests/unit/test_cluster_kubeconfig.py @@ -34,6 +34,8 @@ def kubeconfig_modes(builder: str) -> list[str]: 'regrade_flags=""', 'api_port_flags=""', 'etcd_timing_flags=""', + # The bootstrap server's: no join URL. + 'join_url=""', function_source(builder), builder, 'printf "%s\\n" "$server_args"', diff --git a/tests/unit/test_cluster_regrade.py b/tests/unit/test_cluster_regrade.py index 44a6dee4..e1e3f731 100644 --- a/tests/unit/test_cluster_regrade.py +++ b/tests/unit/test_cluster_regrade.py @@ -374,6 +374,8 @@ def run_script(self, *body: str, **env: str) -> subprocess.CompletedProcess[str] 'mesh_ip="100.64.0.1"', 'etcd_timing_flags=""', 'api_port_flags=""', + # The server's join URL as set_join_url chooses it when the role directory names none (test_node_entrypoint.py tests the choice). + 'join_url="${K3S_JOIN_SERVER_URL:-}"', assignments(["regrade_level_file", "regrade_started_file", "regrade_flags"]), function_source("set_regrade_flags"), function_source("mesh_server_args"), diff --git a/tests/unit/test_cluster_voters.py b/tests/unit/test_cluster_voters.py index d4c2b647..9f6e8e2b 100644 --- a/tests/unit/test_cluster_voters.py +++ b/tests/unit/test_cluster_voters.py @@ -147,7 +147,7 @@ def test_an_eligible_host_publishes_eligible_true_and_pinned_false_in_its_env_an expected = f"K3S_NODE_LABELS={ELIGIBLE_LABEL}=true {PINNED_LABEL}=false {LINK_LABEL}=wired" self.assertIn(expected, server) self.assertIn(expected, agent) - self.assertEqual(files, ["docker-compose.yml", "docker-compose.node-labels.yml"]) + self.assertEqual(files, ["docker-compose.yml", "docker-compose.node-labels.yml", "docker-compose.fleet-role.yml"]) def test_a_pinned_host_publishes_pinned_true(self) -> None: server, _, _ = self.render_env("-e", f"{PINNED}=true") @@ -158,6 +158,7 @@ def test_with_the_controller_on_every_host_publishes_both_as_false_so_that_a_set for lines in (server, agent): self.assertIn(f"K3S_NODE_LABELS={ELIGIBLE_LABEL}=false {PINNED_LABEL}=false {LINK_LABEL}=wired", lines) self.assertIn("docker-compose.node-labels.yml", files) + self.assertIn("docker-compose.fleet-role.yml", files) def test_the_label_names_can_be_changed(self) -> None: server, _, _ = self.render_env("-e", f"{ELIGIBLE}=true", "-e", "github_runner_cluster_voter_eligible_label=example.com/candidate") diff --git a/tests/unit/test_native_node.py b/tests/unit/test_native_node.py index fc5d1953..70e37932 100644 --- a/tests/unit/test_native_node.py +++ b/tests/unit/test_native_node.py @@ -239,7 +239,9 @@ def role_is(self, role: str, recorded: str | None) -> subprocess.CompletedProces file = self.tmp / "fleet-role" if recorded is not None: file.write_text(recorded) - return subprocess.run(["sh", str(ROLE / "files" / "native" / "fleet-role-is.sh"), role, str(file)], env=self.env(), capture_output=True, text=True, timeout=60, check=False) + # The server unit also names k3s's server directory, for its tombstone (test_native_server.py tests that part). + server = [str(self.tmp / "server")] if role == "server" else [] + return subprocess.run(["sh", str(ROLE / "files" / "native" / "fleet-role-is.sh"), role, str(file), *server], env=self.env(), capture_output=True, text=True, timeout=60, check=False) def test_the_role_condition_holds_only_for_the_role_the_file_names(self) -> None: self.assertEqual(self.role_is("server", "server").returncode, 0) diff --git a/tests/unit/test_native_server.py b/tests/unit/test_native_server.py index 4501aa98..0a49aa5d 100644 --- a/tests/unit/test_native_server.py +++ b/tests/unit/test_native_server.py @@ -4,6 +4,7 @@ import re import stat +import subprocess import unittest import yaml @@ -45,7 +46,8 @@ def test_the_server_unit_runs_k3s_server_under_its_role_condition_with_the_mesh_ text = self.file("github_runner_cluster_native_unit_dir", "k3s.service").read_text() service = [line for line in text.split("[Service]")[1].split("[Install]")[0].splitlines() if line and not line.startswith("#")] lib, config = self.paths["github_runner_cluster_native_lib_dir"], self.paths["github_runner_cluster_native_config_dir"] - self.assertIn(f"ExecCondition={lib}/fleet-role-is server {self.paths['github_runner_cluster_native_data_dir']}/fleet-role", service) + data = self.paths["github_runner_cluster_native_data_dir"] + self.assertIn(f"ExecCondition={lib}/fleet-role-is server {data}/fleet-role {data}/server", service) self.assertIn(f"ExecStartPre={lib}/write-node-ip {config}/server.yaml.d/00-node-ip.yaml {config}/vpn-auth", service) self.assertIn(f"ExecStart={self.paths['github_runner_cluster_native_bin_dir']}/k3s server --config {config}/server.yaml", service) # Each unit has a runtime directory of its own: systemd removes a unit's when a start of it is skipped by its condition, which would otherwise take the running unit's files. @@ -266,6 +268,28 @@ def test_an_eligible_agent_is_staged_as_a_standby_server_and_keeps_its_role(self self.assertEqual(self.role_file(), "agent") self.assertEqual(self.unit_calls(before), []) + def test_the_server_unit_does_not_start_while_its_datastore_holds_a_tombstone(self) -> None: + self.assertEqual(self.run_role(host_vars={"github_runner_cluster_bootstrap": True}, only_the_node=True).returncode, 0) + text = self.file("github_runner_cluster_native_unit_dir", "k3s.service").read_text() + # The server unit's ExecCondition= as systemd runs it, from the unit the role wrote, against the role file and k3s data directory the role keeps. + condition = next(line for line in text.splitlines() if line.startswith("ExecCondition=")).removeprefix("ExecCondition=").split() + + def start() -> subprocess.CompletedProcess[str]: + return subprocess.run(["sh", *condition], capture_output=True, text=True, timeout=60, check=False) + + self.assertEqual(start().returncode, 0) + tombstone = self.paths["github_runner_cluster_native_data_dir"] / "server" / "db" / "etcd" / "tombstone" + tombstone.parent.mkdir(parents=True, exist_ok=True) + tombstone.touch() + refused = start() + # 1 to 254 is a start systemd skips, which Restart=always does not repeat, rather than a failure. + self.assertEqual(refused.returncode, 1) + self.assertEqual(self.role_file(), "server") + self.assertIn(f"{tombstone} says this server's etcd member was removed from the cluster", refused.stderr) + # Moving server/db aside, as finishing a demotion does, takes the tombstone with it. + tombstone.parent.parent.rename(tombstone.parent.parent.with_name("db.demoted")) + self.assertEqual(start().returncode, 0) + def test_a_native_bootstrap_host_is_refused_under_headscale_in_cluster(self) -> None: result = self.run_role(host_vars={"github_runner_cluster_bootstrap": True, "github_runner_cluster_mesh": "headscale_in_cluster"}) self.assertNotEqual(result.returncode, 0) diff --git a/tests/unit/test_node_entrypoint.py b/tests/unit/test_node_entrypoint.py new file mode 100644 index 00000000..5236e255 --- /dev/null +++ b/tests/unit/test_node_entrypoint.py @@ -0,0 +1,239 @@ +"""Tests for what the node entrypoint does before Tailscale and k3s start: the role directory's guards that keep a host from running the service its role file does not name, or a server whose etcd member was removed, the join URL a promoted server takes from the role directory, and the refusal of a former member that has nothing to join, which would otherwise bootstrap a second cluster. The entrypoint starts tailscaled and execs k3s at top level, so the assignments, the functions and the guard block are taken from it and run in a shell, with a stand-in for sleep on PATH.""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +ROLE = ROOT / "roles" / "github_runner_cluster" +ENTRYPOINT = ROLE / "files" / "k3s" / "node-entrypoint.sh" +FLEET_ROLE_COMPOSE = ROLE / "files" / "docker-compose.fleet-role.yml" +MOUNT = "./fleet-role.d:/etc/github-runner:ro" +JOIN_URL = "https://node2.example.internal:6443" +BOOTSTRAP_URL = "https://bootstrap.example.internal:6443" + +# Stands in for sleep: records its arguments, then sends the shell that started it in the background the SIGTERM `docker stop` would, so a test of the idle loop ends, and ends as a stopped container's would. +SLEEP_STUB = """#!/bin/sh +echo "$*" >> "$STUB/sleep" +kill -TERM "$PPID" +""" + + +def function_source(name: str) -> str: + """The definition of a shell function in the entrypoint, from its first line to the closing brace at the start of a line.""" + match = re.search(rf"^{name}\(\) \{{\n.*?^\}}\n", ENTRYPOINT.read_text(), re.MULTILINE | re.DOTALL) + assert match is not None, f"{name} is not defined in node-entrypoint.sh" + return match.group(0) + + +def assignment(name: str) -> str: + """The top-level assignment of a variable in the entrypoint, on one line.""" + match = re.search(rf"^{name}=.*$", ENTRYPOINT.read_text(), re.MULTILINE) + assert match is not None, f"{name} is not assigned in node-entrypoint.sh" + return match.group(0) + + +def guard_block() -> str: + """The top-level lines that run the guards, from the comment above them to the line before tailscaled's directories are made.""" + match = re.search(r"^# Before anything else starts.*?\n(.*?)\n\n# k3s's own vpn-auth integration", ENTRYPOINT.read_text(), re.MULTILINE | re.DOTALL) + assert match is not None, "no guard block before Tailscale starts in node-entrypoint.sh" + return match.group(1) + + +class EntrypointGuardTest(unittest.TestCase): + def setUp(self) -> None: + self.work = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.work) + self.role_dir = self.work / "fleet-role.d" + self.role_dir.mkdir() + self.server_dir = self.work / "k3s" / "server" + self.stub = self.work / "stub" + self.stub.mkdir() + bin_dir = self.work / "bin" + bin_dir.mkdir() + sleep = bin_dir / "sleep" + sleep.write_text(SLEEP_STUB) + sleep.chmod(0o755) + self.path = f"{bin_dir}{os.pathsep}{os.environ['PATH']}" + + def start(self, role: str, **env: str) -> subprocess.CompletedProcess[str]: + """Runs the entrypoint's guards as the service ``role`` starts, then, if they let it through, builds a server's k3s arguments as the server case does and prints them after the word started.""" + script = "\n".join( + [ + "set -u", + f"role={role}", + assignment("fleet_role_dir"), + assignment("k3s_server_dir"), + 'vpn_auth="name=tailscale,joinKey=example"', + 'mesh_ip="100.64.0.1"', + 'api_port_flags=""', + 'etcd_timing_flags=""', + 'regrade_flags=""', + function_source("idle"), + function_source("fleet_role_guard"), + function_source("set_join_url"), + function_source("refuse_second_cluster"), + function_source("mesh_server_args"), + guard_block(), + 'if [ "$role" = server ]; then mesh_server_args; else server_args=""; fi', + 'printf "started %s\\n" "$server_args"', + ] + ) + clean = {key: value for key, value in os.environ.items() if not key.startswith(("K3S_", "FLEET_ROLE_"))} + settings = {"FLEET_ROLE_DIR": str(self.role_dir), "K3S_SERVER_DIR": str(self.server_dir), "STUB": str(self.stub), "PATH": self.path} + return subprocess.run(["/bin/sh", "-c", script], capture_output=True, text=True, check=False, env=clean | settings | env, timeout=60) + + def server_args(self, result: subprocess.CompletedProcess[str]) -> list[str]: + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(result.stdout.startswith("started"), result.stdout + result.stderr) + return result.stdout.split()[1:] + + def idled(self, result: subprocess.CompletedProcess[str]) -> None: + """Asserts the start idled: it waited on sleep infinity, ended only through the stop signal (status 0, as `docker stop` leaves it) and got no further.""" + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotIn("started", result.stdout) + self.assertEqual((self.stub / "sleep").read_text().splitlines(), ["infinity"]) + + def former_member(self) -> None: + (self.server_dir / "db.demoted" / "etcd").mkdir(parents=True) + + def test_the_join_url_file_takes_precedence_over_the_inventorys_join_url(self) -> None: + (self.role_dir / "join-url").write_text(JOIN_URL + "\n") + args = self.server_args(self.start("server", K3S_JOIN_SERVER_URL=BOOTSTRAP_URL)) + self.assertEqual(args[args.index("--server") + 1], JOIN_URL) + self.assertNotIn(BOOTSTRAP_URL, args) + self.assertNotIn("--cluster-init", args) + # An empty file names no server, so the inventory's still counts. + (self.role_dir / "join-url").write_text("") + args = self.server_args(self.start("server", K3S_JOIN_SERVER_URL=BOOTSTRAP_URL)) + self.assertEqual(args[args.index("--server") + 1], BOOTSTRAP_URL) + + def test_a_server_with_its_datastore_and_nothing_to_join_keeps_cluster_init(self) -> None: + (self.server_dir / "db" / "etcd").mkdir(parents=True) + # A copy an earlier demotion moved aside beside it changes nothing: k3s reuses the datastore it finds and ignores the flag. + self.former_member() + args = self.server_args(self.start("server")) + self.assertIn("--cluster-init", args) + self.assertNotIn("--server", args) + + def test_a_former_member_with_nothing_to_join_is_refused_and_k3s_is_not_started(self) -> None: + self.former_member() + result = self.start("server") + self.assertEqual(result.returncode, 1) + self.assertNotIn("started", result.stdout) + self.assertIn("has no server to join, so starting k3s would bootstrap a second cluster; not starting it", result.stderr) + self.assertIn(f"join-url in the role directory ({self.role_dir}", result.stderr) + # It exits rather than idling, so it never waited. + self.assertFalse((self.stub / "sleep").exists()) + # With a join URL, from either source, it joins the cluster that exists. + for env, written in (({"K3S_JOIN_SERVER_URL": BOOTSTRAP_URL}, ""), ({}, JOIN_URL)): + with self.subTest(env=env, written=written): + (self.role_dir / "join-url").write_text(written) + args = self.server_args(self.start("server", **env)) + self.assertIn("--server", args) + self.assertNotIn("--cluster-init", args) + + def test_a_first_bootstrap_with_no_datastore_at_all_passes_cluster_init(self) -> None: + args = self.server_args(self.start("server")) + self.assertIn("--cluster-init", args) + self.assertNotIn("--server", args) + + def test_a_service_whose_role_the_role_file_does_not_name_idles(self) -> None: + for role, recorded in (("server", "agent\n"), ("agent", "server"), ("server", "")): + with self.subTest(role=role, recorded=recorded): + (self.stub / "sleep").unlink(missing_ok=True) + (self.role_dir / "role").write_text(recorded) + result = self.start(role, K3S_JOIN_SERVER_URL=BOOTSTRAP_URL) + self.idled(result) + self.assertIn(f"{self.role_dir}/role names '{recorded.strip()}', not {role}, so k3s is not started", result.stderr) + + def test_a_server_whose_member_was_removed_idles_whatever_its_role_file_says(self) -> None: + (self.server_dir / "db" / "etcd").mkdir(parents=True) + (self.server_dir / "db" / "etcd" / "tombstone").touch() + for recorded in (None, "server"): + with self.subTest(recorded=recorded): + (self.stub / "sleep").unlink(missing_ok=True) + if recorded is not None: + (self.role_dir / "role").write_text(recorded) + result = self.start("server", K3S_JOIN_SERVER_URL=BOOTSTRAP_URL) + self.idled(result) + self.assertIn("db/etcd/tombstone says this server's etcd member was removed from the cluster", result.stderr) + # The tombstone is the server's: the agent service never runs etcd and starts as before. + (self.role_dir / "role").write_text("agent") + self.assertEqual(self.start("agent").returncode, 0) + + def test_with_no_role_file_each_service_starts_as_it_always_has(self) -> None: + (self.server_dir / "db" / "etcd").mkdir(parents=True) + self.assertEqual( + self.server_args(self.start("server", K3S_JOIN_SERVER_URL=BOOTSTRAP_URL)), + ["--disable=traefik", "--disable-network-policy", "--write-kubeconfig-mode=644", "--vpn-auth=name=tailscale,joinKey=example", "--resolv-conf=/etc/k3s-resolv.conf", "--node-ip=100.64.0.1", "--server", BOOTSTRAP_URL], + ) + self.assertEqual(self.start("agent").returncode, 0) + self.assertFalse((self.stub / "sleep").exists()) + # A role file naming the service's own role changes nothing either. + (self.role_dir / "role").write_text("server\n") + self.assertIn("--server", self.server_args(self.start("server", K3S_JOIN_SERVER_URL=BOOTSTRAP_URL))) + + def test_the_guards_run_before_tailscale_starts_and_the_server_joins_the_url_they_chose(self) -> None: + text = ENTRYPOINT.read_text() + self.assertLess(text.index(guard_block()), text.index("tailscaled --state=")) + match = re.search(r"^ server\)\n(.*?)^ ;;\n", text, re.MULTILINE | re.DOTALL) + assert match is not None, "no server case in node-entrypoint.sh" + self.assertIn('server_join_host=$(join_host_of "$join_url")', match.group(1)) + self.assertLess(match.group(1).index('wait_for_mesh_host "$server_join_host"'), match.group(1).index("exec /bin/k3s server")) + + +def has_compose() -> bool: + if shutil.which("docker") is None: + return False + return subprocess.run(["docker", "compose", "version"], capture_output=True, check=False).returncode == 0 + + +class RoleDirectoryTest(unittest.TestCase): + """The role directory reaches both services through an override of its own, which the role places and lists only for a host that takes part in the role controller (test_cluster_voters.py checks the list), and the directory the role creates for it.""" + + def test_the_override_mounts_the_directory_read_only_into_both_services_and_nothing_else(self) -> None: + services = yaml.safe_load(FLEET_ROLE_COMPOSE.read_text())["services"] + self.assertEqual(services, {"k3s": {"volumes": [MOUNT]}, "k3s-agent": {"volumes": [MOUNT]}}) + self.assertEqual(assignment("fleet_role_dir"), "fleet_role_dir=${FLEET_ROLE_DIR:-/etc/github-runner}") + # The base file does not mount it, so a host the controller never moves is not recreated by an upgrade. + self.assertNotIn("fleet-role.d", (ROLE / "files" / "docker-compose.yml").read_text()) + + def test_the_role_places_the_override_and_creates_the_directory_before_any_compose_command(self) -> None: + tasks = yaml.safe_load((ROLE / "tasks" / "main.yml").read_text()) + names = [task["name"] for task in tasks] + create = next(task for task in tasks if task.get("ansible.builtin.file", {}).get("state") == "directory") + self.assertIn({"path": "fleet-role.d", "mode": "0755"}, create["loop"]) + place = next(task for task in tasks if "loop" in task and any(entry.get("src") == "docker-compose.yml" for entry in task["loop"])) + self.assertIn({"src": "docker-compose.fleet-role.yml", "mode": "0644"}, place["loop"]) + self.assertLess(names.index(create["name"]), names.index("Set up the mesh the nodes join, and publish its join key and control server")) + defaults = yaml.safe_load((ROLE / "defaults" / "main.yml").read_text()) + self.assertIn("docker-compose.fleet-role.yml", defaults["github_runner_cluster_compose_files_written"]) + + @unittest.skipUnless(has_compose(), "docker compose is not installed") + def test_merged_both_containers_mount_the_projects_directory_read_only(self) -> None: + project = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, project) + clean = {key: value for key, value in os.environ.items() if not key.startswith(("DOCKER_", "COMPOSE_", "K3S_", "TAILSCALE_"))} + base = {"K3S_TOKEN": "t", "K3S_TLS_SAN_LIST": "n", "K3S_VERSION": "v1.34.1-k3s1", "TAILSCALE_VERSION": "v1.102.4"} + args = ["docker", "compose", "--project-directory", str(project), "--project-name", "example", "-f", str(ROLE / "files" / "docker-compose.yml"), "-f", str(FLEET_ROLE_COMPOSE), "--profile", "server", "--profile", "agent", "config", "--format", "json"] + result = subprocess.run(args, capture_output=True, text=True, check=False, env=clean | base) + self.assertEqual(result.returncode, 0, result.stderr) + services = json.loads(result.stdout)["services"] + for name in ("k3s", "k3s-agent"): + with self.subTest(service=name): + mounts = [volume for volume in services[name]["volumes"] if volume["target"] == "/etc/github-runner"] + self.assertEqual([(mount["type"], mount["source"], mount["read_only"]) for mount in mounts], [("bind", str(project / "fleet-role.d"), True)]) + + +if __name__ == "__main__": + unittest.main()