diff --git a/plugins/filter/cluster_guard.py b/plugins/filter/cluster_guard.py index e55f75f7..e2a43362 100644 --- a/plugins/filter/cluster_guard.py +++ b/plugins/filter/cluster_guard.py @@ -1,4 +1,4 @@ -"""Filters the github_runner_cluster role uses before it restarts a node (moving its container between networks, resizing its Colima VM): whether the restart would be safe for the cluster, whether the fleet-regrade Lease that serialises such steps may be taken and how it is given back, and whether the Docker host has an address on the LAN.""" +"""Filters the github_runner_cluster role uses before it restarts a node (moving its container between networks, resizing its Colima VM): whether the restart would be safe for the cluster, whether the fleet-regrade Lease that serialises such steps may be taken and how it is given back, whether the role controller holds it for a voter swap, during which no run of the role may change a node, and whether the Docker host has an address on the LAN.""" from __future__ import annotations @@ -41,6 +41,12 @@ def _etcd_healthy(result: Any) -> bool: return result.get("rc") == 0 and str(result.get("stdout", "")).strip() == "ok" +def _reason(result: Mapping[str, Any]) -> str: + """The last line of what a failed command or task said, or its exit status when it said nothing.""" + lines = [line.strip() for line in str(result.get("stderr", "") or result.get("msg", "") or "").splitlines() if line.strip()] + return lines[-1] if lines else f"exit status {result.get('rc')}" + + def cluster_restart_blockers(nodes: Any, pods: Any, node_name: Any, servers: Any, etcd_results: Any) -> list[str]: """Say why restarting (or recreating) this host's node now would be unsafe, or return an empty list when it would not be. @@ -133,8 +139,7 @@ def cluster_guard_state(results: Any) -> dict[str, Any]: reasons.append(f"{host} did not answer within github_runner_cluster_cluster_read_timeout") continue if result.get("rc") != 0: - lines = [line.strip() for line in str(result.get("stderr", "") or message).splitlines() if line.strip()] - reasons.append(f"{host} could not be read ({lines[-1] if lines else 'exit status ' + str(result.get('rc'))})") + reasons.append(f"{host} could not be read ({_reason(result)})") continue try: state = json.loads(str(result.get("stdout", ""))) @@ -181,6 +186,14 @@ def _epoch(stamp: str) -> float: return datetime.strptime(re.sub(r"\.[0-9]+Z$", "Z", stamp), "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc).timestamp() +def _lease_expires(spec: Mapping[str, Any]) -> float | None: + """When a Lease's holder's time runs out, as epoch seconds: its renew time (its acquire time when it was never renewed) plus its duration, the rule lease_state applies in files/regrade/fleet-regrade-core.sh; None when it records neither time.""" + renewed = spec.get("renewTime") or spec.get("acquireTime") + if not renewed: + return None + return _epoch(str(renewed)) + int(spec.get("leaseDurationSeconds") or 0) + + def _lease_read(read: Any) -> dict[str, Any] | None | str: """The Lease a `kubectl get lease --output json` result holds, None when the API server answered that there is none, or the reason the read failed.""" if not isinstance(read, Mapping): @@ -219,10 +232,10 @@ def cluster_lease_claim(read: Any, holder: Any, now: Any, duration: Any, name: A return {"state": "absent", "holder": "", "verb": "create", "manifest": manifest, "problem": ""} spec = current.get("spec") or {} current_holder = str(spec.get("holderIdentity") or "") - renewed = spec.get("renewTime") or spec.get("acquireTime") + expires = _lease_expires(spec) if not current_holder: state = "free" - elif not renewed or _epoch(str(renewed)) + int(spec.get("leaseDurationSeconds") or 0) <= float(now): + elif expires is None or expires <= float(now): state = "expired" elif current_holder == holder: state = "mine" @@ -245,6 +258,53 @@ def cluster_lease_release(read: Any, holder: Any) -> dict[str, Any]: return {"verb": "replace", "manifest": released} +def cluster_role_lease_guard(reads: Any, holder: Any, swap_id_annotation: Any) -> dict[str, str]: + """Say whether the role controller holds the regrade Lease, which it holds for the whole of a voter swap: while it does, a run of the role must change no node, so that it never undoes or races the swap. + + :param reads: the loop results of reading the Lease through each server of the inventory (tasks/role_lease_guard.yml), each with its host as ``item`` and ``stdout`` either "stopped" or "running", the reading host's clock as epoch seconds, then the Lease as ``kubectl get lease -o json`` prints it (nothing when there is none). :param holder: the holderIdentity the role controller takes the Lease as. :param swap_id_annotation: the annotation on the Lease that names the swap. :returns: ``state``, one of ``held`` (by the controller, and unexpired), ``free`` (no server runs, or the Lease is absent, has run out, is unheld or held by anything else, such as a regrading step) and ``unknown`` (no server answered and one could not be read), with ``message``, a sentence saying why. :raises AnsibleFilterError: if a server answered with something that is not a Lease. + + One answer decides, since every server's API reads the same Lease. Its expiry (_lease_expires, as cluster_lease_claim judges it) is compared with the clock of the host that read it, the clock the controller renews by. + """ + if not isinstance(reads, Sequence) or isinstance(reads, str): + raise AnsibleFilterError("The reads must be a list of loop results.") + unread: list[str] = [] + for result in reads: + if not isinstance(result, Mapping): + raise AnsibleFilterError(f"{result!r} is not a loop result.") + host = str(result.get("item", "?")) + if result.get("unreachable") or result.get("rc") != 0: + unread.append(f"{host} could not be read ({_reason(result)})") + continue + state, _, rest = str(result.get("stdout", "")).partition("\n") + if state.strip() == "stopped": + continue + clock, _, text = rest.partition("\n") + if state.strip() != "running" or not clock.strip().isdigit(): + raise AnsibleFilterError(f"{host} gave an answer that is not a read of the Lease: {result.get('stdout')!r}") + if not text.strip(): + return {"state": "free", "message": f"there is no Lease (read through {host})"} + try: + lease = json.loads(text) + except ValueError as exc: + raise AnsibleFilterError(f"{host} gave a Lease that is not JSON: {exc}") from exc + if not isinstance(lease, dict): + raise AnsibleFilterError(f"{host} gave a Lease that is not an object: {text!r}") + metadata = lease.get("metadata") or {} + spec = lease.get("spec") or {} + name = f"{metadata.get('namespace', '?')}/{metadata.get('name', '?')}" + expires = _lease_expires(spec) + if str(spec.get("holderIdentity") or "") != str(holder) or expires is None: + return {"state": "free", "message": f"the Lease {name} is not held by {holder} (read through {host})"} + if expires <= int(clock): + return {"state": "free", "message": f"the Lease {name} that {holder} held has run out (read through {host})"} + swap = str((metadata.get("annotations") or {}).get(str(swap_id_annotation)) or "") + until = datetime.fromtimestamp(expires, tz=timezone.utc).isoformat(timespec="seconds") + return {"state": "held", "message": f"the role controller ({holder}) holds the Lease {name} for the voter swap {swap or '(unnamed)'}, read through {host}, until {until} unless it renews it first"} + if unread: + return {"state": "unknown", "message": "whether the role controller holds the Lease cannot be told: " + "; ".join(unread)} + return {"state": "free", "message": "no server of the inventory runs, so there is no Lease"} + + class FilterModule: """Registers the cluster guard filters with Ansible.""" @@ -256,4 +316,5 @@ def filters(self) -> dict[str, Any]: "addresses_in_network": addresses_in_network, "cluster_lease_claim": cluster_lease_claim, "cluster_lease_release": cluster_lease_release, + "cluster_role_lease_guard": cluster_role_lease_guard, } diff --git a/plugins/filter/cluster_native.py b/plugins/filter/cluster_native.py index 3cf8177a..44de6c41 100644 --- a/plugins/filter/cluster_native.py +++ b/plugins/filter/cluster_native.py @@ -1,4 +1,4 @@ -"""Filters the github_runner_cluster role's native node uses: which k3s release asset a host's architecture takes, the release and package versions the node's pins name, which tailnet the host's Tailscale client is logged in to, which role the host runs and whether it bootstraps a cluster, and whether the cluster's other voters are Ready enough for a server to start, restart or join.""" +"""Filters the github_runner_cluster role's native node uses: which k3s release asset a host's architecture takes, the release and package versions the node's pins name, which tailnet the host's Tailscale client is logged in to, whether a server bootstraps a cluster, and whether the cluster's other voters are Ready enough for a server to start, restart or join.""" from __future__ import annotations @@ -82,38 +82,12 @@ def cluster_native_tailnet_login(status: Any, tailnet_domain: Any) -> dict[str, return {"state": state, "backend": backend, "domain": domain} -NODE_ROLES = ("server", "agent") - - def _reason(result: dict[str, Any]) -> str: """The last line of what a failed command or task said, or its exit status when it said nothing.""" lines = [line.strip() for line in str(result.get("stderr") or result.get("msg") or "").splitlines() if line.strip()] return lines[-1] if lines else f"exit status {result.get('rc')}" -def cluster_native_role(recorded: Any, agent_installed: Any, eligible: Any, inventory_role: Any, running: Any = ()) -> dict[str, str]: - """Decide the role a native host runs now and the role it is to run after this run. - - :param recorded: what the host's role file holds (empty when it has none). :param agent_installed: whether the host has the k3s-agent unit, which a host set up before the role file existed has. :param eligible: whether the host is marked github_runner_cluster_voter_eligible. :param inventory_role: the role the topology gives the host. :param running: the roles whose unit is active now. :returns: ``current`` (``server``, ``agent``, or empty on a first install) and ``desired``. :raises AnsibleFilterError: if the role file holds anything but a role, or the inventory role or a running role is not one. - - The role file is the host's own record of its role and outlives restarts, reboots and runs of the role. A switch writes the new role to it before it stops the old unit, so while exactly one unit runs, that unit's role is the one the host runs now, and a switch that stopped part-way (a member removal that did not finish) is taken up again by the next run. With neither unit running, the file says; a host from before the file existed ran only the agent, so its unit counts as the agent role. For a host marked eligible the inventory gives only the role it starts with: once the file names a role, that role is kept, since whatever moves votes between hosts changes it at run time and a run of the role must not undo that. For any other host the inventory decides. - """ - text = "" if recorded is None else str(recorded).strip() - if text and text not in NODE_ROLES: - raise AnsibleFilterError(f"The role file holds {text!r}; it must be 'server' or 'agent'. Write the role the host runs into it, or remove it on a host that runs neither unit.") - if inventory_role not in NODE_ROLES: - raise AnsibleFilterError(f"{inventory_role!r} is not a node role (server or agent).") - active = [str(role) for role in running] - if any(role not in NODE_ROLES for role in active): - raise AnsibleFilterError(f"{active!r} are not node roles (server or agent).") - if len(active) == 1: - current = active[0] - else: - current = text or ("agent" if bool(agent_installed) else "") - desired = text if text and bool(eligible) else str(inventory_role) - return {"current": current, "desired": desired} - - def cluster_native_cluster_init(reads: Any, this_host: Any) -> dict[str, Any]: """Say whether a native bootstrap host with no datastore of its own may start a new cluster: only when no other server of the inventory runs one. @@ -174,7 +148,6 @@ def filters(self) -> dict[str, Any]: "cluster_native_k3s_release": cluster_native_k3s_release, "cluster_native_tailscale_package_version": cluster_native_tailscale_package_version, "cluster_native_tailnet_login": cluster_native_tailnet_login, - "cluster_native_role": cluster_native_role, "cluster_native_cluster_init": cluster_native_cluster_init, "cluster_native_voter_blockers": cluster_native_voter_blockers, } diff --git a/plugins/filter/cluster_role.py b/plugins/filter/cluster_role.py new file mode 100644 index 00000000..db2b980c --- /dev/null +++ b/plugins/filter/cluster_role.py @@ -0,0 +1,42 @@ +"""Filters the github_runner_cluster role uses to decide which role, server or agent, a host runs, whether it runs natively or under Docker Compose.""" + +from __future__ import annotations + +from typing import Any + +from ansible.errors import AnsibleFilterError + +NODE_ROLES = ("server", "agent") + + +def cluster_host_role(recorded: Any, legacy_agent: Any, eligible: Any, inventory_role: Any, running: Any = ()) -> dict[str, str]: + """Decide the role a host runs now and the role it is to run after this run. + + :param recorded: what the host's role file holds (empty when it has none). :param legacy_agent: whether the host runs, or is set up to run, the agent from before the role file existed, which counts as the agent role when the file is absent and nothing runs (a native host's k3s-agent unit; never for a Docker host, whose role file is new). :param eligible: whether the host is marked github_runner_cluster_voter_eligible. :param inventory_role: the role the topology gives the host. :param running: the roles whose unit or container runs now. :returns: ``current`` (``server``, ``agent``, or empty on a first install) and ``desired``. :raises AnsibleFilterError: if the role file holds anything but a role, or the inventory role or a running role is not one. + + The role file is the host's own record of its role and outlives restarts, reboots and runs of the role. A switch writes the new role to it before it stops the old unit or container, so while exactly one of them runs, its role is the one the host runs now, and a switch that stopped part-way (a member removal that did not finish) is taken up again by the next run. With neither running, the file says. For a host marked eligible the inventory gives only the role it starts with: once the file names a role, that role is kept, since whatever moves votes between hosts changes it at run time and a run of the role must not undo that. For any other host the inventory decides. + """ + text = "" if recorded is None else str(recorded).strip() + if text and text not in NODE_ROLES: + raise AnsibleFilterError(f"The role file holds {text!r}; it must be 'server' or 'agent'. Write the role the host runs into it, or remove it on a host that runs neither.") + if inventory_role not in NODE_ROLES: + raise AnsibleFilterError(f"{inventory_role!r} is not a node role (server or agent).") + active = [str(role) for role in running] + if any(role not in NODE_ROLES for role in active): + raise AnsibleFilterError(f"{active!r} are not node roles (server or agent).") + if len(active) == 1: + current = active[0] + else: + current = text or ("agent" if bool(legacy_agent) else "") + desired = text if text and bool(eligible) else str(inventory_role) + return {"current": current, "desired": desired} + + +class FilterModule: + """Registers the role filters with Ansible.""" + + def filters(self) -> dict[str, Any]: + """Return the filters this plugin provides.""" + return { + "cluster_host_role": cluster_host_role, + } diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index 31f076d9..dab16c21 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -417,6 +417,19 @@ The two nodes share the Docker host's memory, CPU and kernel limits. Each k3s se Each Compose project runs its own `docker-prune` service against the same daemon. Two of them do the same work and are harmless. + +### A Docker host's recorded role + +A Docker host marked `github_runner_cluster_voter_eligible` keeps its role in its role directory, as a native host keeps its role in its role file: the first run records the inventory's role in `fleet-role.d/role` (`github_runner_cluster_docker_role_file`) when nothing is recorded there, and never writes over a recorded role, since the controller that moves votes changes it at run time and a run of the role must not undo that (`tasks/docker_role.yml` decides, with `cluster_host_role` in `plugins/filter/cluster_role.py`, which the native node shares; `tasks/docker_role_record.yml` records). The recorded role picks the Compose profile, the container the run starts and checks, whether the host takes a server's turn (`tasks/start_servers.yml` gives a turn to every host of the play whose role is `server`, not to the inventory's list of servers), and whether the regrade service runs: it runs beside a server only, so an eligible host may keep `github_runner_cluster_regrade: true` across a swap and has the service removed while it runs as an agent. The quorum guard a planned restart passes (`tasks/restart_guard.yml`, for a move between networks or a Colima VM resize) counts the host by its recorded role too: one recorded as an agent is read through the other servers and has no etcd vote of its own to keep. A host that is not marked eligible gets no record and runs the inventory's role. + +Every Docker host gets one `.env` (`templates/env.j2`), holding both node services' variables whichever role it runs. Each service interpolates only the variables it names, so a service's configuration is the same as from the `.env` its role was given before there was one, and no running container is recreated by the change. + +A run refuses, before it changes anything, a host whose node containers run in another role than the one it is to run (a host not marked eligible whose inventory role changed, an eligible host whose container no longer matches its record), a host running both containers, and a host not marked eligible that has a role recorded, which the entrypoint would obey once the directory is mounted: starting the other profile beside a running node would register the node twice under one name, and switching a running Docker node's role is not something a run of the role does. The message says what to stop or set. + +### Runs during a voter swap + +The role controller holds the regrade Lease (`kube-system/fleet-regrade`, the Lease regrading takes for a step, with the holder `fleet-role`) for the whole of a voter swap, its annotation `github-runner.exadev.com/swap-id` naming the swap. While it holds it unexpired, every run of the role, Docker or native, refuses before it changes anything (`tasks/role_lease_guard.yml`, with `cluster_role_lease_guard` in `plugins/filter/cluster_guard.py`), so a run never recreates a container, restarts a unit or rewrites a node's configuration in the middle of a swap. The Lease is read through each server of the inventory and each host marked eligible, a Docker server's through its container and a native server's through `k3s kubectl`, and its expiry (renew time plus duration, as regrading judges it) is compared with the clock of the host it was read on. A Lease regrading holds, or one that has run out, does not stop the run; no server answering when one could not be read does, since the run cannot tell. `--check` reports the same refusals. The names are the role's constants (the Lease's own, `github_runner_cluster_regrade_lease_namespace` and `github_runner_cluster_regrade_lease_name`, which a Colima VM resize takes too, and `github_runner_cluster_role_lease_holder` and `github_runner_cluster_role_lease_swap_id_annotation`, derived, not settings). + ## Variables - `github_runner_cluster_dir` (default `~/.github-runner` on the target host): the directory the role manages as the host's Compose project. It copies the collection's `docker-compose.yml` and `k3s/` build context there, templates `.env` there, and k3s writes the kubeconfig to its `kubeconfig/` subdirectory, so the host needs no checkout of this repository. The kubeconfig holds the cluster's admin credentials. k3s writes it with mode 644, since on a host whose Docker runs containers as root the file is root's and the connecting user reads it through the bits for others, and the role makes the `kubeconfig/` directory 0700 for the connecting user, which keeps every other local user out. @@ -432,7 +445,7 @@ Each Compose project runs its own `docker-prune` service against the same daemon - `github_runner_cluster_node_taints`: Kubernetes taints (`key=value:Effect`, no spaces) the host's node registers with, empty by default. `NoSchedule` keeps pods without a matching toleration off it and `NoExecute` also evicts the ones already there. k3s applies them on every registration, so they come back if the node object is deleted, which a taint set with `kubectl` does not. - `github_runner_cluster_etcd_heartbeat_interval`, `github_runner_cluster_etcd_election_timeout`: etcd's timing in milliseconds for every server, empty by default (k3s's own). Set together, the same on every server, the election timeout at least five times the heartbeat interval; see [etcd timing](#etcd-timing). - `github_runner_cluster_role_controller` (default false), `github_runner_cluster_voters` (default 3), `github_runner_cluster_voter_eligible` and `github_runner_cluster_voter_pinned` (per host, default false): the role controller's inventory (see [Role controller](#role-controller)). `github_runner_cluster_voter_eligible_label` and `github_runner_cluster_voter_pinned_label` are the Node labels they are published under, derived, never set directly. -- `github_runner_cluster_regrade` (default false): run the `regrade` service, or on a native server the `fleet-regrade` unit, which steps this server down under memory pressure while it keeps its etcd vote (see [Memory-pressure regrading](#memory-pressure-regrading)). Servers only. `github_runner_cluster_regrade_pressure_threshold` (percent, default 10) and `github_runner_cluster_regrade_available_floor_percent` (default 15) are the limit; `github_runner_cluster_regrade_step_up` (`automatic`, the default, or `manual`), `github_runner_cluster_regrade_sample_seconds`, `github_runner_cluster_regrade_sustain_seconds`, `github_runner_cluster_regrade_cooldown_seconds`, `github_runner_cluster_regrade_operation_timeout_seconds` and `github_runner_cluster_regrade_lease_duration_seconds` are the timing, all derived from `github_runner_cluster_regrade_pressure_window_seconds` (the kernel's `avg300`, 300 seconds, not a setting) unless set. `github_runner_cluster_regrade_level_label` and `github_runner_cluster_regrade_taint` are derived, never set directly. +- `github_runner_cluster_regrade` (default false): run the `regrade` service, or on a native server the `fleet-regrade` unit, which steps this server down under memory pressure while it keeps its etcd vote (see [Memory-pressure regrading](#memory-pressure-regrading)). Servers only, and a Docker host marked eligible, whose service runs only while its recorded role is server (see [A Docker host's recorded role](#a-docker-hosts-recorded-role)). `github_runner_cluster_regrade_pressure_threshold` (percent, default 10) and `github_runner_cluster_regrade_available_floor_percent` (default 15) are the limit; `github_runner_cluster_regrade_step_up` (`automatic`, the default, or `manual`), `github_runner_cluster_regrade_sample_seconds`, `github_runner_cluster_regrade_sustain_seconds`, `github_runner_cluster_regrade_cooldown_seconds`, `github_runner_cluster_regrade_operation_timeout_seconds` and `github_runner_cluster_regrade_lease_duration_seconds` are the timing, all derived from `github_runner_cluster_regrade_pressure_window_seconds` (the kernel's `avg300`, 300 seconds, not a setting) unless set. `github_runner_cluster_regrade_level_label` and `github_runner_cluster_regrade_taint` are derived, never set directly. - Per-host overrides: `github_runner_cluster_node_role` (`server` or `agent`), `github_runner_cluster_bootstrap` (true picks the bootstrap host, false excludes a host), `github_runner_cluster_server_url` (join this server instead, for a cluster whose servers are outside the inventory), `github_runner_cluster_tls_sans`, `github_runner_cluster_node_address`, and `github_runner_cluster_voter_eligible` (true stages a native host as a standby server, its server configuration written and its name on every server's certificate, without making it a server; see [Native servers](#native-servers)). The run fails if the result has an even number of servers, no servers, or more than one bootstrap host. - `github_runner_cluster_docker_context` (default empty): pins the Docker context this host's node runs in (see [Docker contexts](#docker-contexts)). - `github_runner_cluster_network_mode` (default `bridge`, or `host`): the network the node's container runs on, and with `host`, `github_runner_cluster_lan_cidr` (required: the LAN the nodes share) and `github_runner_cluster_network_probe_image` (default `busybox:1.37`, for reading the host's addresses) (see [Networking](#networking)). @@ -453,6 +466,7 @@ Each Compose project runs its own `docker-prune` service against the same daemon - `github_runner_cluster_native_tailscale_install` (default `true`): whether the role installs the pinned Tailscale package from Tailscale's apt repository (Debian and Ubuntu). `false` leaves the host's client as it is and only checks its login. - `github_runner_cluster_native_tailscale_repository_url` (default `https://pkgs.tailscale.com/stable`): where that repository's signing key and list file are fetched from, when the host has none. - `github_runner_cluster_native_tailscale_leave_other_tailnet` (default `false`): whether the role logs a client out of another tailnet so that k3s can join the cluster's; `false` stops the run instead. +- `github_runner_cluster_docker_role_file`, `github_runner_cluster_role_lease_holder` and `github_runner_cluster_role_lease_swap_id_annotation` are derived: the file that records an eligible Docker host's role, and the role controller's holder name on the regrade Lease and the annotation naming its swap (see [A Docker host's recorded role](#a-docker-hosts-recorded-role) and [Runs during a voter swap](#runs-during-a-voter-swap)). - `github_runner_cluster_native_agent_service`, `github_runner_cluster_native_server_service`, `github_runner_cluster_native_service`, `github_runner_cluster_native_join_url`, `github_runner_cluster_native_join_host`, `github_runner_cluster_native_role_file` and `github_runner_cluster_native_restart_marker` are derived: the units' names (`k3s-agent` and `k3s`), the unit of the host's role, the URL the node joins and the host in it, which the Tailscale wrapper waits for, the role file both units' conditions read, and the prefix of the per-unit file that says a restart is waiting; `github_runner_cluster_native_regrade_level_file`, `github_runner_cluster_native_regrade_started_file`, `github_runner_cluster_native_regrade_dir`, `github_runner_cluster_native_regrade_service`, `github_runner_cluster_native_regrade_commands`, `github_runner_cluster_native_agent_runtime_dir` and `github_runner_cluster_native_server_runtime_dir` are derived too: a native server's requested and started regrading levels, where its regrade scripts and settings go, the unit that runs them, the commands regrading needs with the package of each, and each unit's own runtime directory under `/run` (one per unit, since systemd removes a unit's runtime directory when its condition skips a start, which on a shared one would take the running unit's resolver file). `github_runner_cluster_native_drain_for` is derived as well: what `tasks/native/drain.yml` says was not done when it cannot empty the node, a role switch's by default, which a Colima VM resize passes as its own. `github_runner_cluster_native_bin_dir`, `github_runner_cluster_native_lib_dir`, `github_runner_cluster_native_config_dir`, `github_runner_cluster_native_unit_dir`, `github_runner_cluster_native_data_dir`, `github_runner_cluster_native_tailscale_cli`, `github_runner_cluster_native_apt_keyring`, `github_runner_cluster_native_apt_source`, `github_runner_cluster_native_run_dir` and `github_runner_cluster_native_psi_file` are where the native node's files live, which are not settings, and are variables only so the tests can point them at a stand-in. ## Examples @@ -505,4 +519,6 @@ or read from 1Password by adding `exadev.github_runner.github_runner_secrets_one `tests/unit/test_node_entrypoint.py` runs the entrypoint's guards in a shell with a stand-in for `sleep` ([The role directory and a former member](#the-role-directory-and-a-former-member)): the join URL's precedence, `--cluster-init` with a datastore, with one moved aside and with none, the idling of a service its role file does not name and of a server on a tombstone, a host with no role file starting as before, and the override that mounts the directory. `tests/unit/test_native_server.py` runs the native server unit's condition on a tombstone. The `rejoin` mesh scenario does the same with real k3s and etcd: it removes the bootstrap Docker server's member through another server and checks the restarted server idles on its tombstone, that with its datastore moved aside and nothing to join it refuses to start, that a `join-url` naming another server makes it join the existing cluster as a learner that is promoted, with the same etcd cluster ID before and after, and that a role file naming the agent leaves the server container running nothing but its idle wait. +`tests/unit/test_docker_role.py` runs the Docker host's role decision, its record and the guard for a voter swap through the role against the stand-ins for `docker` and the Compose module ([A Docker host's recorded role](#a-docker-hosts-recorded-role), [Runs during a voter swap](#runs-during-a-voter-swap)): an eligible host keeps its recorded role over the inventory's, the role is recorded only when absent and only for an eligible host, the servers' turns and the regrade service follow it, a host whose containers or record differ from the role it is to run is refused, a run is refused while the controller holds the Lease and goes on while regrading holds it or it has run out, `--check` reports the same refusals, and the one `.env` gives each service the configuration hash it had before. `tests/unit/test_native_node.py` runs the same guard on a native host. + `tests/unit/test_node_password.py` covers [the node password volume and the servers' turns](#node-passwords-and-recreating-servers) without Docker: the filters in `plugins/filter/cluster_recreate.py`, the Compose file's volumes, and `tasks/node_password.yml`, `tasks/server_recreate_check.yml`, `tasks/server_ready.yml` and `tasks/start_servers.yml` run through the role (`tests/node_password/run.yml`) against a stand-in for `docker` that moves a real tar stream between its `docker cp` processes and keeps containers, volumes and node readiness as files, and a stand-in for the Compose module that recreates a container whose rendered hash differs and takes its node out of Ready until the role has waited for it. It checks the seeding (mode, ownership, labels, no copy of the password anywhere on the host, a failed copy leaving no volume), that `--check` changes nothing and reports each recreate, that a server is refused while another is not Ready and that agents are not, and from the order of the calls that the turns do not overlap. The `hosted` and `in_cluster` mesh scenarios recreate a joined server in Docker (`recreate_a_joined_server` in `tests/mesh/run.sh`) and check that its node rejoins with the same password and Secret, that no other container restarts, and that the role's `--check` report names exactly that server. diff --git a/roles/github_runner_cluster/defaults/main.yml b/roles/github_runner_cluster/defaults/main.yml index 070622b9..dee4056a 100644 --- a/roles/github_runner_cluster/defaults/main.yml +++ b/roles/github_runner_cluster/defaults/main.yml @@ -202,11 +202,16 @@ github_runner_cluster_docker_context: "" # The names of this host's k3s server container and k3s agent container, which docker-compose.yml takes from .env. Container names are unique on a Docker daemon, so a second cluster on the same Docker hosts needs both set to names of its own (see github_runner_cluster_api_port); the defaults are the names every existing node runs under, so leaving them unset changes nothing. Changing one on a running host makes Compose recreate that container under the new name, on the same volumes, which a server's turn treats as any other recreate. github_runner_cluster_container_name: github-runner-k3s github_runner_cluster_agent_container_name: github-runner-k3s-agent -# The node's k3s container as this host runs it: the server's, or the agent's. Derived, never set directly. -github_runner_cluster_node_container: "{{ github_runner_cluster_container_name if github_runner_cluster_effective_node_role == 'server' else github_runner_cluster_agent_container_name }}" +# Derived, never set directly: the file in the Compose project directory's role directory (fleet-role.d, which docker-compose.fleet-role.yml mounts into both services) that records the role a Docker host marked github_runner_cluster_voter_eligible runs, server or agent, so that the role outlives a run of the role. tasks/docker_role.yml reads it and tasks/docker_role_record.yml writes it only when it is absent; k3s/node-entrypoint.sh idles the service of the other role. +github_runner_cluster_docker_role_file: "{{ github_runner_cluster_dir }}/fleet-role.d/role" +# Derived, never set directly: the holderIdentity the role controller takes the regrade Lease (github_runner_cluster_regrade_lease_namespace and github_runner_cluster_regrade_lease_name) as for the whole of a voter swap, and the annotation on it that names the swap. While the controller holds it unexpired, every run of the role refuses before it changes a node (tasks/role_lease_guard.yml). +github_runner_cluster_role_lease_holder: fleet-role +github_runner_cluster_role_lease_swap_id_annotation: github-runner.exadev.com/swap-id +# The node's k3s container as this host runs it: the server's, or the agent's, by the role tasks/docker_role.yml decided this host runs. Derived, never set directly. +github_runner_cluster_node_container: "{{ github_runner_cluster_container_name if github_runner_cluster_docker_role == 'server' else github_runner_cluster_agent_container_name }}" # Derived, never set directly: the Compose service and the named volume that holds /etc/rancher/node (the node's password) for this host's role, as docker-compose.yml names them, and the volume's name in Docker, which Compose forms from the project and the volume's key. The role seeds the volume from the running container before the first recreate that mounts it (tasks/node_password.yml). -github_runner_cluster_node_service: "{{ 'k3s' if github_runner_cluster_effective_node_role == 'server' else 'k3s-agent' }}" -github_runner_cluster_node_password_volume_key: "{{ 'k3s-node' if github_runner_cluster_effective_node_role == 'server' else 'k3s-agent-node' }}" +github_runner_cluster_node_service: "{{ 'k3s' if github_runner_cluster_docker_role == 'server' else 'k3s-agent' }}" +github_runner_cluster_node_password_volume_key: "{{ 'k3s-node' if github_runner_cluster_docker_role == 'server' else 'k3s-agent-node' }}" github_runner_cluster_node_password_volume: "{{ github_runner_cluster_compose_project }}_{{ github_runner_cluster_node_password_volume_key }}" # The most seconds the role waits, after it has recreated a server's container, for that server's node to report Ready again before it recreates the next server's. A recreated server restarts k3s, rejoins etcd, and registers its node and starts its kubelet, which can take minutes on a busy host; the wait ends as soon as the node is Ready on a heartbeat from the new container, so this only bounds a server that does not come back, which then fails the run with the other servers still untouched. github_runner_cluster_server_ready_timeout: 600 diff --git a/roles/github_runner_cluster/tasks/colima_size_turn.yml b/roles/github_runner_cluster/tasks/colima_size_turn.yml index c2d93f44..6031321d 100644 --- a/roles/github_runner_cluster/tasks/colima_size_turn.yml +++ b/roles/github_runner_cluster/tasks/colima_size_turn.yml @@ -37,7 +37,7 @@ # This host's own when it is a server, since its API answers until the VM stops and again before the node is uncordoned; otherwise another server of the inventory's. Worked out whether or not the node runs, though only a running one is read through it, since Ansible templates a task's delegate_to before its condition, and api.yml reads nothing. ansible.builtin.include_tasks: native/api.yml vars: - github_runner_cluster_native_api_self: "{{ github_runner_cluster_effective_node_role == 'server' }}" + github_runner_cluster_native_api_self: "{{ github_runner_cluster_docker_role == 'server' }}" - name: Read the fleet-regrade Lease # A read that fails is recorded rather than failing here, so the refusal below says why. Bounded by github_runner_cluster_cluster_read_timeout, as the API call's --request-timeout. @@ -143,7 +143,7 @@ - name: Wait for the node to come back ansible.builtin.include_tasks: native/server_ready.yml vars: - github_runner_cluster_native_role: "{{ github_runner_cluster_effective_node_role }}" + github_runner_cluster_native_role: "{{ github_runner_cluster_docker_role }}" github_runner_cluster_native_started: "{{ github_runner_cluster_colima_size_stopped }}" when: github_runner_cluster_colima_size_live | bool diff --git a/roles/github_runner_cluster/tasks/docker_role.yml b/roles/github_runner_cluster/tasks/docker_role.yml new file mode 100644 index 00000000..9e6ef529 --- /dev/null +++ b/roles/github_runner_cluster/tasks/docker_role.yml @@ -0,0 +1,83 @@ +--- +# Decides which role this Docker host runs, server or agent, and publishes it as github_runner_cluster_docker_role (the profile, the container, the server turns and the regrade service follow it), with github_runner_cluster_docker_current_role (what the host runs now, or empty when nothing runs and nothing is recorded). plugins/filter/cluster_role.py's cluster_host_role decides, as it does for a native host: a host marked github_runner_cluster_voter_eligible keeps the role its role file records (github_runner_cluster_docker_role_file), since the role controller changes it at run time, and starts from the inventory's while it has none; any other host runs the inventory's role. Refuses, before anything is changed, a host whose node containers run in another role than the one it is to run, or both at once, since starting the other profile beside a running node would register a second node under the same name: switching a Docker host's role is not something a run of the role does. Needs topology.yml. Every task only reads, and runs under --check too. + +- name: Read the role this host's role file records + ansible.builtin.slurp: + src: "{{ github_runner_cluster_docker_role_file }}" + register: github_runner_cluster_docker_role_recorded + failed_when: + - github_runner_cluster_docker_role_recorded is failed + - "'not found' not in github_runner_cluster_docker_role_recorded.msg | default('')" + +- name: Read which of this host's node containers run + # docker ps rather than docker inspect, so that a container that does not exist reads as absent while a daemon that cannot be reached fails the run. A container that runs is counted whatever its k3s does: one that idles because the role file names the other role (k3s/node-entrypoint.sh) still holds the node's name and volumes until it is removed. + ansible.builtin.shell: | + set -eu + for role in server agent; do + if [ "$role" = server ]; then container=$server; else container=$agent; fi + state=$(docker ps --all --filter "name=^/${container}\$" --format '{{ '{{' }}.State{{ '}}' }}') + echo "${role}=${state:-absent}" + done + environment: >- + {{ github_runner_cluster_docker_cli_environment + | combine({'server': github_runner_cluster_container_name, 'agent': github_runner_cluster_agent_container_name}) }} + register: github_runner_cluster_docker_role_containers + changed_when: false + check_mode: false + +- name: Work out the role this host runs now and the role it is to run + vars: + github_runner_cluster_docker_role_running: >- + {{ github_runner_cluster_docker_role_containers.stdout_lines | select('match', '^(server|agent)=running$') + | map('regex_replace', '=running$', '') | list }} + github_runner_cluster_docker_roles: >- + {{ (github_runner_cluster_docker_role_recorded.content | b64decode if github_runner_cluster_docker_role_recorded.content is defined else '') + | exadev.github_runner.cluster_host_role(false, github_runner_cluster_topology.hosts[inventory_hostname].eligible, + github_runner_cluster_effective_node_role, github_runner_cluster_docker_role_running) }} + ansible.builtin.set_fact: + github_runner_cluster_docker_role: "{{ github_runner_cluster_docker_roles.desired }}" + github_runner_cluster_docker_current_role: "{{ github_runner_cluster_docker_roles.current }}" + github_runner_cluster_docker_role_running: "{{ github_runner_cluster_docker_role_running }}" + github_runner_cluster_docker_role_file_text: >- + {{ github_runner_cluster_docker_role_recorded.content | b64decode | trim if github_runner_cluster_docker_role_recorded.content is defined else '' }} + +- name: Say which role this host runs + ansible.builtin.debug: + msg: >- + {{ inventory_hostname }} is to run as {{ github_runner_cluster_docker_role }} + ({{ 'recorded in ' ~ github_runner_cluster_docker_role_file if github_runner_cluster_docker_role_file_text | length > 0 and github_runner_cluster_topology.hosts[inventory_hostname].eligible else 'from the inventory' }}). + +- name: Fail if both of this host's node containers run + ansible.builtin.fail: + msg: >- + Not running the role on {{ inventory_hostname }}: both of its node containers run ({{ github_runner_cluster_container_name }} and + {{ github_runner_cluster_agent_container_name }}), which register the same node twice. Stop the one whose role the host is not to run + ({{ github_runner_cluster_docker_role }}) with `docker stop` and run again. Nothing was changed. + when: github_runner_cluster_docker_role_running | length > 1 + +- name: Fail if this host runs, or records, another role than the one it is to run + # The role this host runs now is its running container's, or, with none running, the role its role file records. It differs from the role it is to run when the inventory changed a host that is not eligible, when an eligible host's container no longer matches its record, or when a role file is left on a host that is no longer eligible, whose record the entrypoint would still obey once the role directory is mounted. + vars: + github_runner_cluster_docker_role_now: >- + {{ ('its ' ~ github_runner_cluster_docker_current_role ~ ' container (' + ~ (github_runner_cluster_container_name if github_runner_cluster_docker_current_role == 'server' else github_runner_cluster_agent_container_name) + ~ ') runs') + if github_runner_cluster_docker_role_running | length == 1 + else ('its role directory records ' ~ github_runner_cluster_docker_current_role ~ ' (' ~ github_runner_cluster_docker_role_file ~ ')') }} + github_runner_cluster_docker_role_source: >- + {{ ('its role directory records ' ~ github_runner_cluster_docker_role ~ ' (' ~ github_runner_cluster_docker_role_file ~ ')') + if github_runner_cluster_topology.hosts[inventory_hostname].eligible and github_runner_cluster_docker_role_file_text | length > 0 + else 'the inventory gives it the role ' ~ github_runner_cluster_docker_role }} + ansible.builtin.fail: + msg: >- + Not running the role on {{ inventory_hostname }}: {{ github_runner_cluster_docker_role_now }}, but {{ github_runner_cluster_docker_role_source }}. + {% if github_runner_cluster_topology.hosts[inventory_hostname].eligible %}The host is marked github_runner_cluster_voter_eligible, so its + role is the one its role directory records, and a run of the role does not switch a running node to another role: finish or undo the switch + (stop the container of the role it is not to run, or write the role it runs into {{ github_runner_cluster_docker_role_file }}) and run + again.{% else %}The host is not marked github_runner_cluster_voter_eligible, so the inventory decides its role and nothing may record one + of its own: set github_runner_cluster_node_role to the role it runs, or stop its container and remove {{ github_runner_cluster_docker_role_file }} + to move it, and run again.{% endif %} Nothing was changed. + when: + - github_runner_cluster_docker_role_running | length <= 1 + - github_runner_cluster_docker_current_role | length > 0 + - github_runner_cluster_docker_current_role != github_runner_cluster_docker_role diff --git a/roles/github_runner_cluster/tasks/docker_role_record.yml b/roles/github_runner_cluster/tasks/docker_role_record.yml new file mode 100644 index 00000000..49c80631 --- /dev/null +++ b/roles/github_runner_cluster/tasks/docker_role_record.yml @@ -0,0 +1,10 @@ +--- +# Records the role an eligible Docker host runs in its role directory (github_runner_cluster_docker_role_file), only when nothing is recorded there yet: once a role is recorded, changing it is a role switch, which the role controller makes at run time, never a run's side effect. A host that is not marked github_runner_cluster_voter_eligible gets no record, since the inventory decides its role and the entrypoint runs the service it was started as. Needs tasks/docker_role.yml, and the role directory, which tasks/main.yml creates first. + +- name: Record the role this eligible host runs, when nothing is recorded yet + ansible.builtin.copy: + content: "{{ github_runner_cluster_docker_role }}\n" + dest: "{{ github_runner_cluster_docker_role_file }}" + mode: "0644" + force: false + when: github_runner_cluster_topology.hosts[inventory_hostname].eligible | bool diff --git a/roles/github_runner_cluster/tasks/host_network.yml b/roles/github_runner_cluster/tasks/host_network.yml index c81590c5..8a6ba49e 100644 --- a/roles/github_runner_cluster/tasks/host_network.yml +++ b/roles/github_runner_cluster/tasks/host_network.yml @@ -64,7 +64,7 @@ environment: "{{ github_runner_cluster_docker_cli_environment }}" vars: github_runner_cluster_network_container: >- - {{ github_runner_cluster_container_name if github_runner_cluster_effective_node_role == 'server' else 'github-runner-k3s-agent' }} + {{ github_runner_cluster_container_name if github_runner_cluster_docker_role == 'server' else github_runner_cluster_agent_container_name }} register: github_runner_cluster_network_node_container changed_when: false check_mode: false diff --git a/roles/github_runner_cluster/tasks/instance_check.yml b/roles/github_runner_cluster/tasks/instance_check.yml index fd0c927e..ddc3a190 100644 --- a/roles/github_runner_cluster/tasks/instance_check.yml +++ b/roles/github_runner_cluster/tasks/instance_check.yml @@ -35,7 +35,7 @@ {{ github_runner_cluster_instance_containers.stdout_lines | exadev.github_runner.cluster_instance_conflicts( github_runner_cluster_compose_project, [github_runner_cluster_container_name, github_runner_cluster_agent_container_name], - github_runner_cluster_api_port if github_runner_cluster_effective_node_role == 'server' else '', + github_runner_cluster_api_port if github_runner_cluster_docker_role == 'server' else '', github_runner_cluster_dir) }} ansible.builtin.fail: msg: >- diff --git a/roles/github_runner_cluster/tasks/main.yml b/roles/github_runner_cluster/tasks/main.yml index e0a7a6bd..4bdfc6a6 100644 --- a/roles/github_runner_cluster/tasks/main.yml +++ b/roles/github_runner_cluster/tasks/main.yml @@ -1,5 +1,5 @@ --- -# Brings this host up as a k3s node, natively under systemd (github_runner_cluster_node_mode: native, tasks/native/main.yml) or, by default, under Docker Compose: a server (bootstrapping the cluster or joining it) or an agent, as tasks/topology.yml decides from the cluster group. Safe to rerun: the .env is templated and docker compose converges to it. +# Brings this host up as a k3s node, natively under systemd (github_runner_cluster_node_mode: native, tasks/native/main.yml) or, by default, under Docker Compose: a server (bootstrapping the cluster or joining it) or an agent, as tasks/topology.yml decides from the cluster group, or, for a host marked eligible, as its role directory records (tasks/docker_role.yml). Safe to rerun: the .env is templated and docker compose converges to it. - name: Bring this host up as a native k3s server or agent, with no Docker # github_runner_cluster_node_mode: native. Everything below drives Docker, which a native node does not have, so the role ends after this; see tasks/native/main.yml. @@ -52,6 +52,14 @@ - name: Decide this host's part in the cluster ansible.builtin.include_tasks: topology.yml +- name: Decide which role this host runs, from its role directory when it is eligible + # Right after the topology, so that every check and task below follows the role the host runs, which for an eligible host is the one recorded on it rather than the inventory's. See tasks/docker_role.yml. + ansible.builtin.include_tasks: docker_role.yml + +- name: Refuse the run while the role controller holds the regrade Lease for a voter swap + # Before anything is written to the host or any container is touched. See tasks/role_lease_guard.yml. + ansible.builtin.include_tasks: role_lease_guard.yml + - name: Check the versions the node's image is built from ansible.builtin.include_tasks: versions.yml @@ -103,6 +111,10 @@ loop_control: label: "{{ item.path }}" +- name: Record the role an eligible host runs in its role directory + # Right after the directory exists, so that the role is on the host before any Compose command starts a node that reads it. + ansible.builtin.include_tasks: docker_role_record.yml + - name: Place the collection's Compose file and k3s build context in it # Before the mesh step, since the headscale_in_cluster provider starts the bootstrap server's Compose project itself. Copied on every run, so a collection upgrade reaches the host. The entrypoint is bind-mounted read-only and run through /bin/sh, so its mode only matters to someone running it by hand. ansible.builtin.copy: @@ -127,7 +139,7 @@ ansible.builtin.include_tasks: node_password.yml - name: Set up the mesh the nodes join, and publish its join key and control server - # Must complete before either branch below templates .env, since both env.j2 and env-agent.j2 read the mesh facts it publishes. See tasks/mesh/main.yml for the provider interface. + # Must complete before either branch below templates .env, since env.j2 reads the mesh facts it publishes. See tasks/mesh/main.yml for the provider interface. ansible.builtin.include_tasks: mesh/main.yml - name: Resolve this host's Docker context again for the Compose tasks below @@ -135,7 +147,8 @@ ansible.builtin.include_tasks: docker_context.yml - name: Set up and start this host as a k3s server, bootstrapping or joining - when: github_runner_cluster_effective_node_role == "server" + # By the role tasks/docker_role.yml decided, which for an eligible host is the one recorded on the host. + when: github_runner_cluster_docker_role == "server" block: - name: Fail if this host isn't included in the fleet's own TLS SAN list # Only an overridden list can miss it, since the automatic one names every server. Checks the EFFECTIVE node name (github_runner_cluster_node_name if set, else inventory_hostname), not raw inventory_hostname: a host may deliberately override its own node name, and checking the wrong string here would incorrectly fail that host. @@ -149,7 +162,8 @@ automatic list. when: github_runner_cluster_effective_node_name not in (github_runner_cluster_effective_tls_sans | join(' ')) - - name: Template .env for this k3s server + - name: Template .env for this node + # One .env for either role, holding both services' variables, so that a host whose role changes at run time has what the other service reads already; each service reads only its own, so the other's never changes its configuration. ansible.builtin.template: src: env.j2 dest: "{{ github_runner_cluster_dir }}/.env" @@ -168,11 +182,11 @@ when: ansible_facts['system'] == 'Darwin' - name: Join as a k3s agent - when: github_runner_cluster_effective_node_role == "agent" + when: github_runner_cluster_docker_role == "agent" block: - - name: Template .env for the k3s agent + - name: Template .env for this node ansible.builtin.template: - src: env-agent.j2 + src: env.j2 dest: "{{ github_runner_cluster_dir }}/.env" mode: "0600" no_log: true @@ -180,6 +194,10 @@ - name: Start this agent with Docker Compose ansible.builtin.include_tasks: start_agent.yml + - name: Remove the regrade service, which runs beside a server only + # A host whose recorded role became agent may still run the service it ran as a server. + ansible.builtin.include_tasks: regrade_service.yml + - name: Check the runtime and this node's container are up ansible.builtin.include_tasks: end_checks.yml when: ansible_facts['system'] == 'Darwin' diff --git a/roles/github_runner_cluster/tasks/native/main.yml b/roles/github_runner_cluster/tasks/native/main.yml index 7d18132a..b71c0318 100644 --- a/roles/github_runner_cluster/tasks/native/main.yml +++ b/roles/github_runner_cluster/tasks/native/main.yml @@ -61,6 +61,10 @@ - name: Decide this host's part in the cluster ansible.builtin.include_tasks: topology.yml +- name: Refuse the run while the role controller holds the regrade Lease for a voter swap + # Before anything on the host is changed. See tasks/role_lease_guard.yml. + ansible.builtin.include_tasks: role_lease_guard.yml + - name: Check etcd's heartbeat interval and election timeout, which a native server passes to its etcd as every server does ansible.builtin.include_tasks: etcd_timing.yml diff --git a/roles/github_runner_cluster/tasks/native/role.yml b/roles/github_runner_cluster/tasks/native/role.yml index 9b18a9d9..23c633a2 100644 --- a/roles/github_runner_cluster/tasks/native/role.yml +++ b/roles/github_runner_cluster/tasks/native/role.yml @@ -1,5 +1,5 @@ --- -# Decides which role this native host runs, server or agent, and, for a server, whether it bootstraps a new cluster or joins one. Publishes github_runner_cluster_native_current_role (what the host runs now: its role file, or agent for a host set up before the file existed, or empty on a first install), github_runner_cluster_native_role (what it is to run after this run; plugins/filter/cluster_native.py's cluster_native_role says how the two are decided), github_runner_cluster_native_stage_server (whether the server's files are written: a server, or a host marked eligible, staged as a standby) and github_runner_cluster_native_cluster_init. Needs topology.yml. Every task only reads, and runs under --check too. +# Decides which role this native host runs, server or agent, and, for a server, whether it bootstraps a new cluster or joins one. Publishes github_runner_cluster_native_current_role (what the host runs now: its role file, or agent for a host set up before the file existed, or empty on a first install), github_runner_cluster_native_role (what it is to run after this run; plugins/filter/cluster_role.py's cluster_host_role says how the two are decided), github_runner_cluster_native_stage_server (whether the server's files are written: a server, or a host marked eligible, staged as a standby) and github_runner_cluster_native_cluster_init. Needs topology.yml. Every task only reads, and runs under --check too. - name: Read the role file, the agent unit and the datastores on the host # The server's directory under k3s's data directory is root's alone (k3s makes it 0700), so the reads take become. @@ -35,7 +35,7 @@ vars: github_runner_cluster_native_roles: >- {{ (github_runner_cluster_native_role_recorded.content | b64decode if github_runner_cluster_native_role_recorded is not skipped else '') - | exadev.github_runner.cluster_native_role(github_runner_cluster_native_role_paths.results[1].stat.exists, + | exadev.github_runner.cluster_host_role(github_runner_cluster_native_role_paths.results[1].stat.exists, github_runner_cluster_topology.hosts[inventory_hostname].eligible, github_runner_cluster_effective_node_role, (['server'] if github_runner_cluster_native_role_units.results[0].rc == 0 else []) diff --git a/roles/github_runner_cluster/tasks/regrade.yml b/roles/github_runner_cluster/tasks/regrade.yml index 91549d7c..89df00a2 100644 --- a/roles/github_runner_cluster/tasks/regrade.yml +++ b/roles/github_runner_cluster/tasks/regrade.yml @@ -11,9 +11,11 @@ when: github_runner_cluster_regrade | bool block: - name: Fail if this host is not a server - # An agent has no control plane to shed and no etcd vote to keep. A native host is judged by the role it is to run after this run, which for a host marked github_runner_cluster_voter_eligible is the one recorded on the host, not the inventory's. + # An agent has no control plane to shed and no etcd vote to keep. A native host is judged by the role it is to run after this run, which for a host marked github_runner_cluster_voter_eligible is the one recorded on the host, not the inventory's. A Docker host marked eligible passes whatever role it runs: tasks/regrade_service.yml runs the service only while its recorded role is server, so the setting holds across a swap. The other Docker hosts run the inventory's role (tasks/docker_role.yml refuses any other). ansible.builtin.assert: - that: (github_runner_cluster_native_role if github_runner_cluster_node_mode == 'native' else github_runner_cluster_effective_node_role) == 'server' + that: >- + (github_runner_cluster_native_role == 'server') if github_runner_cluster_node_mode == 'native' + else (github_runner_cluster_effective_node_role == 'server' or github_runner_cluster_topology.hosts[inventory_hostname].eligible) quiet: true fail_msg: >- github_runner_cluster_regrade is true, but {{ inventory_hostname }} is an agent. Regrading steps a server down while it diff --git a/roles/github_runner_cluster/tasks/regrade_service.yml b/roles/github_runner_cluster/tasks/regrade_service.yml index b1edb6d2..bbe4aec5 100644 --- a/roles/github_runner_cluster/tasks/regrade_service.yml +++ b/roles/github_runner_cluster/tasks/regrade_service.yml @@ -1,8 +1,8 @@ --- -# Runs the regrade service beside this server's k3s container when github_runner_cluster_regrade is true, and removes it when it is not. Run for a server host, after the servers have been started; tasks/regrade.yml has already checked the settings. +# Runs the regrade service beside this server's k3s container when github_runner_cluster_regrade is true and the role this host runs (tasks/docker_role.yml) is server, and removes it otherwise. Run for every Docker host, after its node has been started: an agent has no control plane to shed and no etcd vote to keep, so a host marked eligible that runs as an agent keeps its regrading setting for when it runs as a server again, and has no service meanwhile. tasks/regrade.yml has already checked the settings. - name: Run the regrade service - when: github_runner_cluster_regrade | bool + when: github_runner_cluster_regrade | bool and github_runner_cluster_docker_role == 'server' block: - name: Create the regrade service's directory ansible.builtin.file: @@ -44,7 +44,7 @@ state: present - name: Stop regrading - when: not (github_runner_cluster_regrade | bool) + when: not (github_runner_cluster_regrade | bool and github_runner_cluster_docker_role == 'server') block: - name: Read the level this server's k3s started at # Read-only, so it runs under --check too. Absent while the container is not running, or has never been regraded and so has an older entrypoint. @@ -55,6 +55,8 @@ changed_when: false failed_when: false check_mode: false + # An agent's container has no level: only a server is ever stepped down. + when: github_runner_cluster_docker_role == 'server' - name: Fail if the server is stepped down # Without the service nothing steps it back up, and the entrypoint would keep starting it at the level it was left at. Fatal for the whole play, so that no other host removes its service while one server is still stepped down: the refusal is then the same on every host. @@ -66,6 +68,7 @@ `docker compose --profile regrade exec regrade fleet-regrade up` in {{ github_runner_cluster_dir }} until it reports full (status shows the level), then run again. Nothing was changed. when: + - github_runner_cluster_regrade_level is not skipped - github_runner_cluster_regrade_level.rc == 0 - github_runner_cluster_regrade_level.stdout | trim != 'full' diff --git a/roles/github_runner_cluster/tasks/restart_guard.yml b/roles/github_runner_cluster/tasks/restart_guard.yml index 00be5961..e5806792 100644 --- a/roles/github_runner_cluster/tasks/restart_guard.yml +++ b/roles/github_runner_cluster/tasks/restart_guard.yml @@ -14,7 +14,7 @@ 'timeout': github_runner_cluster_cluster_read_timeout | string}) }} delegate_to: "{{ item }}" loop: >- - {{ ([inventory_hostname] if github_runner_cluster_effective_node_role == 'server' else []) + {{ ([inventory_hostname] if github_runner_cluster_docker_role == 'server' else []) + (github_runner_cluster_topology.hosts | dict2items | selectattr('value.role', 'equalto', 'server') | map(attribute='key') | reject('equalto', inventory_hostname) | list) }} vars: @@ -58,19 +58,22 @@ ignore_unreachable: true check_mode: false timeout: "{{ github_runner_cluster_cluster_read_timeout | int }}" - when: github_runner_cluster_effective_node_role == 'server' + when: github_runner_cluster_docker_role == 'server' - name: Refuse to restart the node while the cluster needs it vars: github_runner_cluster_guard: "{{ (github_runner_cluster_guard_reads.results | exadev.github_runner.cluster_guard_state).state }}" # A step that cordons the node and waits for its runner pods itself checks the cluster before it does, with the pods left out, and again once they have gone. + # The servers are the inventory's, with this host counted by the role it runs (tasks/docker_role.yml), which for an eligible host is the one recorded on it: one recorded as an agent has no etcd vote to keep. Another host's recorded role is not read here, so a server the inventory does not name counts against the restart, as the filter says. + github_runner_cluster_guard_servers: >- + {{ (github_runner_cluster_topology.hosts | dict2items | selectattr('value.role', 'equalto', 'server') + | rejectattr('key', 'equalto', inventory_hostname) | map(attribute='value.node_name') | list) + + ([github_runner_cluster_effective_node_name] if github_runner_cluster_docker_role == 'server' else []) }} github_runner_cluster_guard_blockers: >- {{ github_runner_cluster_guard.nodes | exadev.github_runner.cluster_restart_blockers(github_runner_cluster_guard.pods if github_runner_cluster_restart_guard_runners | bool else [], github_runner_cluster_effective_node_name, - github_runner_cluster_topology.hosts | dict2items - | selectattr('value.role', 'equalto', 'server') - | map(attribute='value.node_name') | list, + github_runner_cluster_guard_servers, github_runner_cluster_guard_etcd.results | default([])) }} ansible.builtin.fail: msg: >- diff --git a/roles/github_runner_cluster/tasks/role_lease_guard.yml b/roles/github_runner_cluster/tasks/role_lease_guard.yml new file mode 100644 index 00000000..717e570e --- /dev/null +++ b/roles/github_runner_cluster/tasks/role_lease_guard.yml @@ -0,0 +1,82 @@ +--- +# Refuses the run, before it changes anything, while the role controller holds the regrade Lease (github_runner_cluster_regrade_lease_namespace/github_runner_cluster_regrade_lease_name held by github_runner_cluster_role_lease_holder, unexpired), which it holds for the whole of a voter swap: a run that recreated a container, restarted a unit or rewrote a node's configuration in the middle of a swap could undo it or take a second vote away. A Lease held by regrading, or one that has run out, does not stop the run. The Lease is read through every server of the inventory and every host marked eligible, since any one may be stopped or being rebuilt and the servers can change at run time, a Docker server's through its container and a native server's through `k3s kubectl` on its host; one answer decides (plugins/filter/cluster_guard.py's cluster_role_lease_guard), and the run is refused too when no server answered and one could not be read, since the run cannot tell. Every task only reads, and runs under --check too, which so reports the refusal a real run would make. Needs topology.yml; each read is bounded by github_runner_cluster_cluster_read_timeout, as a whole and as the API call's --request-timeout. + +- name: Work out which hosts of the inventory to read the Lease through + ansible.builtin.set_fact: + github_runner_cluster_role_lease_servers: >- + {{ ((github_runner_cluster_topology.hosts | dict2items | selectattr('value.role', 'equalto', 'server') | map(attribute='key') | list) + + (github_runner_cluster_topology.hosts | dict2items | selectattr('value.eligible') | map(attribute='key') | list)) | unique }} + github_runner_cluster_role_lease_native: >- + {{ github_runner_cluster_topology.hosts | dict2items | selectattr('value.native') | map(attribute='key') | list }} + +- name: Read the Lease through each Docker server of the inventory + # Each on its own host with its own pinned Docker context, through the image's kubectl inside the server's container, which reads that server's kubeconfig. Prints "stopped" when the container does not run, or "running", the host's clock as epoch seconds (the clock the Lease's expiry is judged by) and the Lease, nothing when there is none. A read that fails is recorded rather than failing here, so the filter can say which servers could not be read. + ansible.builtin.shell: | + set -eu + state=$(docker ps --all --filter "name=^/${container}\$" --format '{{ '{{' }}.State{{ '}}' }}') + if [ "$state" != running ]; then + echo stopped + exit 0 + fi + echo running + date +%s + docker exec "$container" kubectl get lease "$name" --namespace "$namespace" --ignore-not-found --output json --request-timeout="${timeout}s" + environment: >- + {{ ({'DOCKER_CONTEXT': github_runner_cluster_role_lease_context} if github_runner_cluster_role_lease_context | length > 0 else {}) + | combine({'container': github_runner_cluster_role_lease_container, 'name': github_runner_cluster_regrade_lease_name, + 'namespace': github_runner_cluster_regrade_lease_namespace, 'timeout': github_runner_cluster_cluster_read_timeout | string}) }} + delegate_to: "{{ item }}" + loop: "{{ github_runner_cluster_role_lease_servers | reject('in', github_runner_cluster_role_lease_native) | list }}" + vars: + github_runner_cluster_role_lease_container: >- + {{ github_runner_cluster_container_name if item == inventory_hostname + else hostvars[item].github_runner_cluster_container_name | default('github-runner-k3s') }} + github_runner_cluster_role_lease_context: >- + {{ (github_runner_cluster_docker_context if item == inventory_hostname + else hostvars[item].github_runner_cluster_docker_context | default('')) | trim }} + register: github_runner_cluster_role_lease_reads_docker + changed_when: false + failed_when: false + ignore_unreachable: true + check_mode: false + timeout: "{{ github_runner_cluster_cluster_read_timeout | int }}" + +- name: Read the Lease through each native server of the inventory + # The same read, through `k3s kubectl` on a native server's host, which reads the admin kubeconfig k3s writes there for root only (hence become), and "stopped" when the server unit is not active. The unit's name and the binary's place are the role's constants, read from the inventory where a host sets them (the tests do). + become: true + ansible.builtin.shell: | + set -eu + if ! systemctl is-active --quiet "$service"; then + echo stopped + exit 0 + fi + echo running + date +%s + "$k3s" kubectl get lease "$name" --namespace "$namespace" --ignore-not-found --output json --request-timeout="${timeout}s" + environment: + service: "{{ github_runner_cluster_native_server_service }}" + k3s: "{{ hostvars[item].github_runner_cluster_native_bin_dir | default(github_runner_cluster_native_bin_dir) }}/k3s" + name: "{{ github_runner_cluster_regrade_lease_name }}" + namespace: "{{ github_runner_cluster_regrade_lease_namespace }}" + timeout: "{{ github_runner_cluster_cluster_read_timeout | string }}" + delegate_to: "{{ item }}" + loop: "{{ github_runner_cluster_role_lease_servers | select('in', github_runner_cluster_role_lease_native) | list }}" + register: github_runner_cluster_role_lease_reads_native + changed_when: false + failed_when: false + ignore_unreachable: true + check_mode: false + timeout: "{{ github_runner_cluster_cluster_read_timeout | int }}" + +- name: Fail while the role controller holds the Lease for a voter swap, or when whether it does cannot be told + vars: + github_runner_cluster_role_lease_verdict: >- + {{ (github_runner_cluster_role_lease_reads_docker.results + github_runner_cluster_role_lease_reads_native.results) + | exadev.github_runner.cluster_role_lease_guard(github_runner_cluster_role_lease_holder, github_runner_cluster_role_lease_swap_id_annotation) }} + ansible.builtin.fail: + msg: >- + Not changing {{ inventory_hostname }}: {{ github_runner_cluster_role_lease_verdict.message }}. + {% if github_runner_cluster_role_lease_verdict.state == 'held' %}A run of the role during a voter swap could undo it or take a second vote + away. Run again once the swap has ended and the controller has released the Lease (the role controller's status ConfigMap reports the swap).{% else %}Bring + a server back within reach and run again.{% endif %} Nothing was changed. + when: github_runner_cluster_role_lease_verdict.state != 'free' diff --git a/roles/github_runner_cluster/tasks/running_version.yml b/roles/github_runner_cluster/tasks/running_version.yml index 925679a9..808d5569 100644 --- a/roles/github_runner_cluster/tasks/running_version.yml +++ b/roles/github_runner_cluster/tasks/running_version.yml @@ -30,7 +30,7 @@ timeout: "{{ github_runner_cluster_cluster_read_timeout | int }}" when: - not github_runner_cluster_allow_k3s_version_jump | bool - - github_runner_cluster_effective_node_role == 'agent' + - github_runner_cluster_docker_role == 'agent' - name: Read every node's k3s version through each server of the inventory ansible.builtin.include_tasks: cluster_nodes_read.yml @@ -41,7 +41,7 @@ github_runner_cluster_version_change_blockers: >- {{ github_runner_cluster_k3s_version | exadev.github_runner.cluster_k3s_version_change_blockers( github_runner_cluster_effective_node_name, github_runner_cluster_running_own.stdout, - github_runner_cluster_running_reads.results | default([]), github_runner_cluster_effective_node_role, + github_runner_cluster_running_reads.results | default([]), github_runner_cluster_docker_role, github_runner_cluster_running_agent_image.stdout | default('') | trim | length == 0) }} ansible.builtin.fail: msg: >- diff --git a/roles/github_runner_cluster/tasks/server_recreate_check.yml b/roles/github_runner_cluster/tasks/server_recreate_check.yml index db6362a9..e1078508 100644 --- a/roles/github_runner_cluster/tasks/server_recreate_check.yml +++ b/roles/github_runner_cluster/tasks/server_recreate_check.yml @@ -21,7 +21,7 @@ argv: >- {{ ['docker', 'compose', '--project-name', github_runner_cluster_compose_project, '--project-directory', github_runner_cluster_dir] + (github_runner_cluster_compose_files_in_use | map('regex_replace', '^(?!/)', github_runner_cluster_dir ~ '/') | map('regex_replace', '^', '--file=') | list) - + ['--profile', github_runner_cluster_effective_node_role, 'config', '--hash', '*'] }} + + ['--profile', github_runner_cluster_docker_role, 'config', '--hash', '*'] }} environment: "{{ github_runner_cluster_docker_cli_environment }}" register: github_runner_cluster_recreate_rendered changed_when: false diff --git a/roles/github_runner_cluster/tasks/start_servers.yml b/roles/github_runner_cluster/tasks/start_servers.yml index 54f11ff4..3f05b7ae 100644 --- a/roles/github_runner_cluster/tasks/start_servers.yml +++ b/roles/github_runner_cluster/tasks/start_servers.yml @@ -1,9 +1,14 @@ --- -# Starts this play's k3s servers one at a time, from the server hosts' own runs, in no fixed order. A server's container is recreated when its Compose configuration changed, which stops its etcd member, so the servers take turns: each turn (tasks/start_server.yml) refuses to recreate a running server unless every other server reports Ready, and waits for a recreated server to be Ready again before the next one starts. This relies on Ansible's linear strategy, the default, where every host takes the same task together, except that tasks a host includes dynamically are its own copies: the tasks one turn includes run for that host alone while the others wait, so no two servers are ever away together. Which server goes first is the order the hosts' results arrive in, not the inventory's, and nothing here depends on it: a joining server's Compose start does not wait for the bootstrap server. Hosts outside the play (--limit) take no turn. +# Starts this play's k3s servers (the hosts whose role is server, tasks/docker_role.yml) one at a time, from the server hosts' own runs, in no fixed order. A server's container is recreated when its Compose configuration changed, which stops its etcd member, so the servers take turns: each turn (tasks/start_server.yml) refuses to recreate a running server unless every other server reports Ready, and waits for a recreated server to be Ready again before the next one starts. This relies on Ansible's linear strategy, the default, where every host takes the same task together, except that tasks a host includes dynamically are its own copies: the tasks one turn includes run for that host alone while the others wait, so no two servers are ever away together. Which server goes first is the order the hosts' results arrive in, not the inventory's, and nothing here depends on it: a joining server's Compose start does not wait for the bootstrap server. Hosts outside the play (--limit) take no turn. - name: Give each server its turn to start ansible.builtin.include_tasks: start_server.yml - loop: "{{ github_runner_cluster_topology.servers | select('in', ansible_play_hosts_all) | list }}" + # Every host of the play whose role is server by tasks/docker_role.yml, which for an eligible host is the role recorded on it rather than the inventory's list of servers. A host that failed before deciding its role has none, and takes no turn, as it would not have run one. + loop: >- + {{ ansible_play_hosts_all + | select('in', hostvars | dict2items | selectattr('value.github_runner_cluster_docker_role', 'defined') + | selectattr('value.github_runner_cluster_docker_role', 'equalto', 'server') | map(attribute='key') | list) + | list }} loop_control: loop_var: github_runner_cluster_turn when: github_runner_cluster_turn == inventory_hostname diff --git a/roles/github_runner_cluster/templates/env-agent.j2 b/roles/github_runner_cluster/templates/env-agent.j2 deleted file mode 100644 index b9b2626d..00000000 --- a/roles/github_runner_cluster/templates/env-agent.j2 +++ /dev/null @@ -1,25 +0,0 @@ -# Generated by the exadev.github_runner collection's github_runner_cluster role on every run. Do not edit by hand: change this host's ansible/host_vars file and rerun ansible-playbook, or, on a host brought up with bootstrap.sh, change .env.bootstrap and rerun ./bootstrap.sh. -# The Compose files, as absolute paths, and the project the role runs this node with, so that a docker compose command run by hand in this directory, or with --project-directory from another, acts on the same project with the same files (each command still needs its --profile, and the host's pinned Docker context when it has one). -COMPOSE_FILE={{ github_runner_cluster_compose_file_list | map('regex_replace', '^(?!/)', github_runner_cluster_dir ~ '/') | join(':') }} -COMPOSE_PROJECT_NAME={{ github_runner_cluster_compose_project }} -K3S_AGENT_CONTAINER_NAME={{ github_runner_cluster_agent_container_name }} -# The server's container name and API port, which docker-compose.yml interpolates for the k3s service: unused by the agent, but set so that the agent's .env renders the same names for both services as the role uses. -K3S_CONTAINER_NAME={{ github_runner_cluster_container_name }} -K3S_API_PORT={{ github_runner_cluster_api_port }} -K3S_TOKEN={{ github_runner_cluster_k3s_token }} -K3S_VERSION={{ github_runner_cluster_k3s_version }} -TAILSCALE_VERSION={{ github_runner_cluster_tailscale_version }} -K3S_AGENT_SERVER_URL={{ github_runner_cluster_effective_server_url }} -K3S_AGENT_NODE_NAME={{ github_runner_cluster_effective_node_name }} -K3S_VPN_AUTH_JOIN_KEY={{ github_runner_cluster_mesh_join_key }} -{% if github_runner_cluster_mesh_control_server_url | length > 0 %} -K3S_VPN_AUTH_CONTROL_SERVER_URL={{ github_runner_cluster_mesh_control_server_url }} -{% endif %} -# Unused by the k3s-agent service itself, but docker compose validates every service's env interpolation up front regardless of which --profile is active - the k3s service requires K3S_TLS_SAN_LIST, and an agent host's .env needs to set SOMETHING for that check to pass under `docker compose --profile agent`. -K3S_TLS_SAN_LIST={{ github_runner_cluster_effective_tls_sans | join(' ') }} -{% if github_runner_cluster_node_labels | length > 0 %} -K3S_NODE_LABELS={{ github_runner_cluster_node_labels | join(' ') }} -{% endif %} -{% if github_runner_cluster_node_taints | length > 0 %} -K3S_NODE_TAINTS={{ github_runner_cluster_node_taints | join(' ') }} -{% endif %} diff --git a/roles/github_runner_cluster/templates/env.j2 b/roles/github_runner_cluster/templates/env.j2 index dffffbf3..b16495c0 100644 --- a/roles/github_runner_cluster/templates/env.j2 +++ b/roles/github_runner_cluster/templates/env.j2 @@ -2,7 +2,9 @@ # The Compose files, as absolute paths, and the project the role runs this node with, so that a docker compose command run by hand in this directory, or with --project-directory from another, acts on the same project with the same files (each command still needs its --profile, and the host's pinned Docker context when it has one). COMPOSE_FILE={{ github_runner_cluster_compose_file_list | map('regex_replace', '^(?!/)', github_runner_cluster_dir ~ '/') | join(':') }} COMPOSE_PROJECT_NAME={{ github_runner_cluster_compose_project }} +# Both node services' variables, whichever role the host runs (tasks/docker_role.yml): each service of docker-compose.yml interpolates only the variables it names, so its configuration, and the hash Compose compares to decide whether to recreate its container, is the same as from a file holding only those, and a host whose role changes at run time already has what the other service reads. K3S_CONTAINER_NAME={{ github_runner_cluster_container_name }} +K3S_AGENT_CONTAINER_NAME={{ github_runner_cluster_agent_container_name }} K3S_API_PORT={{ github_runner_cluster_api_port }} K3S_TOKEN={{ github_runner_cluster_k3s_token }} K3S_VERSION={{ github_runner_cluster_k3s_version }} @@ -13,10 +15,12 @@ K3S_VPN_AUTH_JOIN_KEY={{ github_runner_cluster_mesh_join_key }} K3S_VPN_AUTH_CONTROL_SERVER_URL={{ github_runner_cluster_mesh_control_server_url }} {% endif %} K3S_JOIN_SERVER_URL={{ github_runner_cluster_effective_server_url }} +K3S_AGENT_SERVER_URL={{ github_runner_cluster_effective_server_url }} {% if github_runner_cluster_datastore == 'sqlite' %} K3S_DATASTORE=sqlite {% endif %} K3S_NODE_NAME={{ github_runner_cluster_effective_node_name }} +K3S_AGENT_NODE_NAME={{ github_runner_cluster_effective_node_name }} {% if github_runner_cluster_node_labels | length > 0 %} K3S_NODE_LABELS={{ github_runner_cluster_node_labels | join(' ') }} {% endif %} diff --git a/tests/cluster_versions/render_env.yml b/tests/cluster_versions/render_env.yml index 823af481..b18ff404 100644 --- a/tests/cluster_versions/render_env.yml +++ b/tests/cluster_versions/render_env.yml @@ -1,6 +1,6 @@ --- -# Renders the cluster role's env.j2 and env-agent.j2 with the role's defaults and placeholder values for the settings this test does not look at, writing them to env_output and env_agent_output, and, when compose_files_output is set, the Compose file list the node's project would use, for tests/unit/test_cluster_versions.py, test_cluster_etcd_timing.py and test_cluster_instance.py. Role variables are passed as extra variables, which outrank the defaults loaded here. -- name: Render the node .env templates +# Renders the cluster role's env.j2, the one .env a Docker node gets whichever role it runs, with the role's defaults and placeholder values for the settings this test does not look at, writing it to env_output, and, when compose_files_output is set, the Compose file list the node's project would use, for tests/unit/test_cluster_versions.py, test_cluster_etcd_timing.py, test_cluster_instance.py, test_cluster_voters.py and test_docker_role.py. Role variables are passed as extra variables, which outrank the defaults loaded here. +- name: Render the node .env hosts: localhost connection: local gather_facts: false @@ -22,12 +22,6 @@ dest: "{{ env_output }}" mode: "0600" - - name: Render env-agent.j2 - ansible.builtin.copy: - content: "{{ lookup('ansible.builtin.template', playbook_dir ~ '/../../roles/github_runner_cluster/templates/env-agent.j2') }}" - dest: "{{ env_agent_output }}" - mode: "0600" - - name: Write the Compose files the node's project would use ansible.builtin.copy: content: "{{ github_runner_cluster_compose_file_list | to_json }}" diff --git a/tests/host_network/play.yml b/tests/host_network/play.yml index 17fd5216..9cf1c6b6 100644 --- a/tests/host_network/play.yml +++ b/tests/host_network/play.yml @@ -7,6 +7,7 @@ vars: github_runner_cluster_docker_cli_environment: {} github_runner_cluster_container_name: github-runner-k3s + github_runner_cluster_agent_container_name: github-runner-k3s-agent github_runner_cluster_compose_project: github-runner github_runner_cluster_network_probe_image: "busybox:1.37" github_runner_cluster_cluster_read_timeout: 30 diff --git a/tests/mesh/recreate_check.yml b/tests/mesh/recreate_check.yml index deec9419..8b3483a5 100644 --- a/tests/mesh/recreate_check.yml +++ b/tests/mesh/recreate_check.yml @@ -1,5 +1,5 @@ --- -# Reports which servers' containers the cluster role would recreate, for tests/mesh/run.sh: the role's own check before its Compose step (tasks/server_recreate_check.yml), run on its own against the cluster the test brought up. Run with --check, which is when the check says so; it changes nothing. +# Reports which servers' containers the cluster role would recreate, for tests/mesh/run.sh: the role's own check before its Compose step (tasks/server_recreate_check.yml), run on its own against the cluster the test brought up, after the role each host runs is decided (tasks/docker_role.yml), which the check's profile and container follow. Run with --check, which is when the check says so; it changes nothing. - name: Report which k3s containers the role would recreate hosts: github_runner_cluster gather_facts: false @@ -8,4 +8,4 @@ ansible.builtin.include_role: name: exadev.github_runner.github_runner_cluster tasks_from: "{{ item }}" - loop: [docker_context, topology, server_recreate_check] + loop: [docker_context, topology, docker_role, server_recreate_check] diff --git a/tests/regrade/render_env.yml b/tests/regrade/render_env.yml index cb6991f0..0372a040 100644 --- a/tests/regrade/render_env.yml +++ b/tests/regrade/render_env.yml @@ -9,6 +9,7 @@ vars: github_runner_cluster_effective_node_name: node2 github_runner_cluster_effective_node_role: server + github_runner_cluster_docker_role: server github_runner_cluster_topology: servers: [node1, node2, node3] hosts: diff --git a/tests/regrade/service.yml b/tests/regrade/service.yml index 65834561..39795b09 100644 --- a/tests/regrade/service.yml +++ b/tests/regrade/service.yml @@ -6,6 +6,7 @@ gather_facts: false vars: github_runner_cluster_effective_node_role: server + github_runner_cluster_docker_role: server github_runner_cluster_dir: /srv/github-runner github_runner_cluster_docker_cli_context: "" github_runner_cluster_docker_cli_environment: {} diff --git a/tests/unit/cluster_stand_ins.py b/tests/unit/cluster_stand_ins.py index bcece27b..f8beac6e 100644 --- a/tests/unit/cluster_stand_ins.py +++ b/tests/unit/cluster_stand_ins.py @@ -54,7 +54,7 @@ def flatten(tasks: list[dict[str, Any]]) -> list[dict[str, Any]]: return found -# Stands in for the docker CLI, with its state in files under $DOCKER_STUB: containers/.state (what docker ps prints), .image, .hash (the Compose configuration hash label), .node/ (the files of the container's /etc/rancher/node), .volume (the volume mounted there) and .cp-fails; volumes// and labels/; rendered/.txt (what docker compose config --hash prints); cluster.nodes (what kubectl get nodes prints, " " per line); and containers/.kubectl-fails and never-ready-; leases/ counts reads of a node's lease, whose renewal is older than its container's start on the first read and later after that, which is also when the node reports Ready (kubelet confirming itself) unless never-ready- exists. Every call is appended to calls.log with its DOCKER_CONTEXT, and the owner and mode of each file docker cp extracts to owners.log. docker cp moves a real tar stream between processes, so a pipeline that is wired wrongly moves nothing. +# Stands in for the docker CLI, with its state in files under $DOCKER_STUB: containers/.state (what docker ps prints), .image, .hash (the Compose configuration hash label), .node/ (the files of the container's /etc/rancher/node), .volume (the volume mounted there) and .cp-fails; volumes// and labels/; rendered/.txt (what docker compose config --hash prints); cluster.nodes (what kubectl get nodes prints, " " per line); fleet-regrade.lease (what kubectl get lease fleet-regrade prints, nothing when absent); and containers/.kubectl-fails and never-ready-; leases/ counts reads of a node's lease, whose renewal is older than its container's start on the first read and later after that, which is also when the node reports Ready (kubelet confirming itself) unless never-ready- exists. Every call is appended to calls.log with its DOCKER_CONTEXT, and the owner and mode of each file docker cp extracts to owners.log. docker cp moves a real tar stream between processes, so a pipeline that is wired wrongly moves nothing. DOCKER_STUB = """#!{python} import os, re, sys, tarfile stub = os.environ["DOCKER_STUB"] @@ -176,7 +176,9 @@ def set_node(node, status): if rest[:2] == ["kubectl", "get"]: if read("containers", container + ".kubectl-fails") is not None: die("The connection to the server 127.0.0.1:6443 was refused") - if rest[2] == "nodes": + if rest[2:4] == ["lease", "fleet-regrade"]: + print(read("fleet-regrade.lease") or "", end="") + elif rest[2] == "nodes": print(read("cluster.nodes") or "", end="") elif rest[2] == "node": print(dict(line.split() for line in (read("cluster.nodes") or "").splitlines() if len(line.split()) == 2).get(rest[3], ""), end="") @@ -288,9 +290,9 @@ def tearDown(self) -> None: # Whatever a test did, no real tool was reached. self.assertFalse((self.stub / "forbidden.log").exists(), (self.stub / "forbidden.log").read_text() if (self.stub / "forbidden.log").exists() else "") - def hosts(self, servers: tuple[str, ...], agents: tuple[str, ...] = (), **extra: Any) -> tuple[dict[str, Any], dict[str, Any]]: - """The inventory's hosts and the group variables for a cluster whose servers and agents are named, each host's container named for it.""" - topology = {"servers": list(servers), "hosts": {name: {"role": "server" if name in servers else "agent", "bootstrap": name == servers[0], "node_name": name} for name in (*servers, *agents)}} + def hosts(self, servers: tuple[str, ...], agents: tuple[str, ...] = (), eligible: tuple[str, ...] = (), **extra: Any) -> tuple[dict[str, Any], dict[str, Any]]: + """The inventory's hosts and the group variables for a cluster whose servers and agents are named, those in ``eligible`` marked github_runner_cluster_voter_eligible, each host's containers named for it.""" + topology = {"servers": list(servers), "hosts": {name: {"role": "server" if name in servers else "agent", "bootstrap": name == servers[0], "node_name": name, "address": name, "native": False, "eligible": name in eligible} for name in (*servers, *agents)}} hosts: dict[str, Any] = {} for name in (*servers, *agents): server = name in servers @@ -299,8 +301,11 @@ def hosts(self, servers: tuple[str, ...], agents: tuple[str, ...] = (), **extra: (self.work / "nodes" / name / "docker-compose.yml").write_text("services: {}\n") hosts[name] = { "github_runner_cluster_effective_node_role": "server" if server else "agent", + # The role tasks/docker_role.yml decides, which a test that runs it sets again. + "github_runner_cluster_docker_role": "server" if server else "agent", "github_runner_cluster_effective_node_name": name, "github_runner_cluster_container_name": f"{name}-k3s", + "github_runner_cluster_agent_container_name": f"{name}-k3s-agent", "github_runner_cluster_dir": str(self.work / "nodes" / name), "github_runner_cluster_compose_project": name, "github_runner_cluster_compose_file_list": ["docker-compose.yml"], diff --git a/tests/unit/native_stand_ins.py b/tests/unit/native_stand_ins.py index 7441e8a9..a76d48e7 100644 --- a/tests/unit/native_stand_ins.py +++ b/tests/unit/native_stand_ins.py @@ -213,6 +213,8 @@ def make_stand_ins(self) -> None: "get node "*removed-node-name*) if [ -e "$state/member-removed" ]; then printf 'laptop-0a1b2c3d,'; else printf ',laptop-0a1b2c3d'; fi ;; "annotate --overwrite node "*"etcd.k3s.cattle.io/remove=true"*) [ -e "$state/member-stays" ] || touch "$state/member-removed" ;; "get node "*) if [ -e "$state/node-ready" ]; then cat "$state/node-ready"; else printf 'True,True'; fi ;; + # The regrade Lease, as JSON, and nothing when there is none; any other Lease is a node's, whose renew time is asked for. + "get lease fleet-regrade "*) [ ! -e "$state/fleet-regrade-lease" ] || cat "$state/fleet-regrade-lease" ;; "get lease "*) if [ -e "$state/lease" ]; then cat "$state/lease"; else echo 2099-01-01T00:00:00Z; fi ;; "get pods "*) [ -e "$state/runner-pods" ] && cat "$state/runner-pods"; exit 0 ;; cordon*|uncordon*|drain*|label*|annotate*) ;; diff --git a/tests/unit/test_cluster_colima_size.py b/tests/unit/test_cluster_colima_size.py index d44f740f..a97240eb 100644 --- a/tests/unit/test_cluster_colima_size.py +++ b/tests/unit/test_cluster_colima_size.py @@ -391,6 +391,7 @@ def run_role(self, *extra: str, limit: str | None = None, **variables: Any) -> s hosts = { name: { "github_runner_cluster_effective_node_role": "server" if name in self.SERVERS else "agent", + "github_runner_cluster_docker_role": "server" if name in self.SERVERS else "agent", "github_runner_cluster_effective_node_name": name, "github_runner_cluster_container_name": f"{name}-k3s", "github_runner_cluster_agent_container_name": f"{name}-k3s", diff --git a/tests/unit/test_cluster_compose_files_in_use.py b/tests/unit/test_cluster_compose_files_in_use.py index 1a81d94e..fbd5ca25 100644 --- a/tests/unit/test_cluster_compose_files_in_use.py +++ b/tests/unit/test_cluster_compose_files_in_use.py @@ -142,7 +142,7 @@ def test_the_files_are_placed_then_the_list_is_worked_out_then_the_mesh_and_ever self.assertGreater(matching[0], worked_out, later) def test_no_task_names_the_whole_list_to_a_compose_command_except_where_it_is_defined_or_checked(self) -> None: - allowed = {"defaults/main.yml", "tasks/compose_files.yml", "tasks/compose_files_in_use.yml", "templates/env.j2", "templates/env-agent.j2"} + allowed = {"defaults/main.yml", "tasks/compose_files.yml", "tasks/compose_files_in_use.yml", "templates/env.j2"} offenders = [] for path in sorted(ROLE.rglob("*")): relative = str(path.relative_to(ROLE)) diff --git a/tests/unit/test_cluster_etcd_timing.py b/tests/unit/test_cluster_etcd_timing.py index afe7718f..43e2dc77 100644 --- a/tests/unit/test_cluster_etcd_timing.py +++ b/tests/unit/test_cluster_etcd_timing.py @@ -107,26 +107,21 @@ def test_an_agent_with_other_timing_is_not_compared_with_the_servers(self) -> No result = self.check(hosts) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - def render_env(self, *extra: str) -> tuple[list[str], list[str]]: - env, agent = self.work / "env", self.work / "env-agent" - args = [ANSIBLE_PLAYBOOK, str(ROOT / "tests" / "cluster_versions" / "render_env.yml"), "-e", f"env_output={env}", "-e", f"env_agent_output={agent}", "-e", "github_runner_cluster_dir=/srv/github-runner", *extra] + def render_env(self, *extra: str) -> list[str]: + env = self.work / "env" + args = [ANSIBLE_PLAYBOOK, str(ROOT / "tests" / "cluster_versions" / "render_env.yml"), "-e", f"env_output={env}", "-e", "github_runner_cluster_dir=/srv/github-runner", *extra] result = subprocess.run(args, capture_output=True, text=True, check=False, env=ansible_env(), cwd=self.work) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - return env.read_text().splitlines(), agent.read_text().splitlines() + return env.read_text().splitlines() - def test_the_server_env_carries_both_values_when_set(self) -> None: - server, _ = self.render_env("-e", f"{HEARTBEAT}=500", "-e", f"{ELECTION}=5000") - self.assertIn("K3S_ETCD_HEARTBEAT_INTERVAL=500", server) - self.assertIn("K3S_ETCD_ELECTION_TIMEOUT=5000", server) + def test_the_env_carries_both_values_when_set(self) -> None: + # The agent service takes neither from it: see ComposeFileTest. + lines = self.render_env("-e", f"{HEARTBEAT}=500", "-e", f"{ELECTION}=5000") + self.assertIn("K3S_ETCD_HEARTBEAT_INTERVAL=500", lines) + self.assertIn("K3S_ETCD_ELECTION_TIMEOUT=5000", lines) - def test_the_server_env_names_neither_by_default(self) -> None: - server, agent = self.render_env() - for lines in (server, agent): - self.assertEqual([line for line in lines if "ETCD" in line], []) - - def test_the_agent_env_never_carries_them(self) -> None: - _, agent = self.render_env("-e", f"{HEARTBEAT}=500", "-e", f"{ELECTION}=5000") - self.assertEqual([line for line in agent if "ETCD" in line], []) + def test_the_env_names_neither_by_default(self) -> None: + self.assertEqual([line for line in self.render_env() if "ETCD" in line], []) class ComposeFileTest(unittest.TestCase): diff --git a/tests/unit/test_cluster_instance.py b/tests/unit/test_cluster_instance.py index e13407dd..778b63f0 100644 --- a/tests/unit/test_cluster_instance.py +++ b/tests/unit/test_cluster_instance.py @@ -140,34 +140,32 @@ def setUp(self) -> None: self.work = Path(tempfile.mkdtemp()) self.addCleanup(shutil.rmtree, self.work) - def render(self, *extra: str) -> tuple[list[str], list[str], list[str]]: - env, agent, files = self.work / "env", self.work / "env-agent", self.work / "files.json" - args = [ANSIBLE_PLAYBOOK, str(ROOT / "tests" / "cluster_versions" / "render_env.yml"), "-e", f"env_output={env}", "-e", f"env_agent_output={agent}", "-e", f"compose_files_output={files}", "-e", "github_runner_cluster_dir=/srv/github-runner", *extra] + def render(self, *extra: str) -> tuple[list[str], list[str]]: + env, files = self.work / "env", self.work / "files.json" + args = [ANSIBLE_PLAYBOOK, str(ROOT / "tests" / "cluster_versions" / "render_env.yml"), "-e", f"env_output={env}", "-e", f"compose_files_output={files}", "-e", "github_runner_cluster_dir=/srv/github-runner", *extra] result = subprocess.run(args, capture_output=True, text=True, check=False, env=ansible_env(), cwd=self.work) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) compose_files: list[str] = json.loads(files.read_text()) - return env.read_text().splitlines(), agent.read_text().splitlines(), compose_files + return env.read_text().splitlines(), compose_files def test_the_defaults_write_the_existing_names_and_port_and_add_no_compose_file(self) -> None: - server, agent, files = self.render() - self.assertIn(f"K3S_CONTAINER_NAME={SERVER_CONTAINER}", server) - self.assertIn(f"K3S_API_PORT={K3S_PORT}", server) - self.assertIn(f"K3S_AGENT_CONTAINER_NAME={AGENT_CONTAINER}", agent) - self.assertIn(f"K3S_CONTAINER_NAME={SERVER_CONTAINER}", agent) - self.assertIn(f"K3S_API_PORT={K3S_PORT}", agent) + lines, files = self.render() + self.assertIn(f"K3S_CONTAINER_NAME={SERVER_CONTAINER}", lines) + self.assertIn(f"K3S_AGENT_CONTAINER_NAME={AGENT_CONTAINER}", lines) + self.assertIn(f"K3S_API_PORT={K3S_PORT}", lines) self.assertEqual(files, ["docker-compose.yml"]) - self.assertIn("COMPOSE_FILE=/srv/github-runner/docker-compose.yml", server) + self.assertIn("COMPOSE_FILE=/srv/github-runner/docker-compose.yml", lines) - def test_a_second_cluster_s_values_reach_both_templates_and_the_port_adds_the_override(self) -> None: - server, agent, files = self.render("-e", "github_runner_cluster_container_name=second-k3s", "-e", "github_runner_cluster_agent_container_name=second-k3s-agent", "-e", "github_runner_cluster_api_port=6543") - self.assertIn("K3S_CONTAINER_NAME=second-k3s", server) - self.assertIn("K3S_API_PORT=6543", server) - self.assertIn("K3S_AGENT_CONTAINER_NAME=second-k3s-agent", agent) + def test_a_second_cluster_s_values_reach_the_env_and_the_port_adds_the_override(self) -> None: + lines, files = self.render("-e", "github_runner_cluster_container_name=second-k3s", "-e", "github_runner_cluster_agent_container_name=second-k3s-agent", "-e", "github_runner_cluster_api_port=6543") + self.assertIn("K3S_CONTAINER_NAME=second-k3s", lines) + self.assertIn("K3S_API_PORT=6543", lines) + self.assertIn("K3S_AGENT_CONTAINER_NAME=second-k3s-agent", lines) self.assertEqual(files, ["docker-compose.yml", "docker-compose.api-port.yml"]) - self.assertIn("COMPOSE_FILE=/srv/github-runner/docker-compose.yml:/srv/github-runner/docker-compose.api-port.yml", server) + self.assertIn("COMPOSE_FILE=/srv/github-runner/docker-compose.yml:/srv/github-runner/docker-compose.api-port.yml", lines) def test_the_default_port_written_as_a_string_adds_no_override(self) -> None: - _, _, files = self.render("-e", '{"github_runner_cluster_api_port": "6443"}') + _, files = self.render("-e", '{"github_runner_cluster_api_port": "6443"}') self.assertEqual(files, ["docker-compose.yml"]) @@ -319,7 +317,8 @@ def containers(self, *lines: str) -> None: (self.stub / "containers").write_text("\n".join(lines) + "\n") def run_check(self, *extra: str, role: str = "server", group_vars: dict[str, Any] | None = None, hosts: dict[str, dict[str, Any]] | None = None) -> subprocess.CompletedProcess[str]: - node = {"github_runner_cluster_effective_node_role": role, "github_runner_cluster_docker_cli_environment": {}, "github_runner_cluster_dir": "/srv/second"} + # The role tasks/docker_role.yml would decide is the inventory's, for a host not marked eligible. + node = {"github_runner_cluster_effective_node_role": role, "github_runner_cluster_docker_role": role, "github_runner_cluster_docker_cli_environment": {}, "github_runner_cluster_dir": "/srv/second"} group = {"vars": {"github_runner_cluster_compose_project": "second", **(group_vars or {})}, "hosts": {name: node | values for name, values in (hosts or {"node1": {}}).items()}} inventory = self.work / "inventory.yml" inventory.write_text(yaml.safe_dump({"all": {"children": {"github_runner_cluster": group}}})) diff --git a/tests/unit/test_cluster_regrade.py b/tests/unit/test_cluster_regrade.py index e1e3f731..eba4a59a 100644 --- a/tests/unit/test_cluster_regrade.py +++ b/tests/unit/test_cluster_regrade.py @@ -131,6 +131,12 @@ def test_an_agent_is_refused(self) -> None: result = self.refused("node4 is an agent", {name: {} for name in ("node1", "node2", "node3", "node4")}, **{ENABLED: True}) self.assertEqual(re.findall(r"fatal: \[(node\d)\]", result.stdout), ["node4"]) + def test_a_docker_agent_that_may_become_a_server_may_regrade(self) -> None: + # The service runs on such a host only while the role it records is server (tasks/regrade_service.yml), so the setting it keeps across a swap is not a mistake. + hosts = {name: {ENABLED: True} for name in ("node1", "node2", "node3")} | {"node4": {ENABLED: True, "github_runner_cluster_voter_eligible": True}} + result = self.check(hosts) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + def test_regrading_only_the_servers_of_a_group_with_an_agent_passes(self) -> None: hosts = {"node1": {ENABLED: True}, "node2": {ENABLED: True}, "node3": {ENABLED: True}, "node4": {}} result = self.check(hosts) @@ -466,14 +472,16 @@ def test_the_settings_are_checked_before_the_role_changes_anything(self) -> None self.assertLess(order.index("topology.yml"), order.index("regrade.yml")) self.assertLess(order.index("regrade.yml"), order.index("Create this host's Compose project directory")) - def test_the_service_runs_on_servers_after_the_node_is_started(self) -> None: + def test_the_service_runs_on_servers_after_the_node_is_started_and_is_removed_from_agents(self) -> None: tasks = yaml.safe_load((ROLE / "tasks" / "main.yml").read_text()) - [block] = [task for task in tasks if "block" in task and task.get("when") == 'github_runner_cluster_effective_node_role == "server"'] + [block] = [task for task in tasks if "block" in task and task.get("when") == 'github_runner_cluster_docker_role == "server"'] steps = [step.get("ansible.builtin.include_tasks") for step in block["block"]] self.assertLess(steps.index("start_servers.yml"), steps.index("regrade_service.yml")) self.assertLess(steps.index("regrade_service.yml"), steps.index("end_checks.yml")) - [agent] = [task for task in tasks if "block" in task and task.get("when") == 'github_runner_cluster_effective_node_role == "agent"'] - self.assertNotIn("regrade_service.yml", [step.get("ansible.builtin.include_tasks") for step in agent["block"]]) + # On an agent it only removes the service, which a host that ran as a server before its recorded role changed may still run. + [agent] = [task for task in tasks if "block" in task and task.get("when") == 'github_runner_cluster_docker_role == "agent"'] + agent_steps = [step.get("ansible.builtin.include_tasks") for step in agent["block"]] + self.assertLess(agent_steps.index("start_agent.yml"), agent_steps.index("regrade_service.yml")) def test_the_wait_for_ready_leaves_out_a_server_regrading_has_stepped_down(self) -> None: text = (ROLE / "tasks" / "wait_for_ready.yml").read_text() diff --git a/tests/unit/test_cluster_versions.py b/tests/unit/test_cluster_versions.py index 5db04e50..fcbf0f28 100644 --- a/tests/unit/test_cluster_versions.py +++ b/tests/unit/test_cluster_versions.py @@ -378,6 +378,8 @@ def running(self, *extra: str, pin: str | None = None, run_on: str = "node2", ro hosts = { name: { "github_runner_cluster_effective_node_role": role, + # The role tasks/docker_role.yml would decide: the inventory's, for a host not marked eligible. + "github_runner_cluster_docker_role": role, "github_runner_cluster_effective_node_name": name, "github_runner_cluster_container_name": f"{name}-k3s", "github_runner_cluster_docker_context": f"ctx-{name}", @@ -584,6 +586,8 @@ def bootstrap_shaped(self, role: str) -> subprocess.CompletedProcess[str]: "github_runner_cluster_server_url": "https://server.example.internal:6443", "github_runner_cluster_tls_sans": [], "github_runner_cluster_docker_cli_environment": {}, + # The role tasks/docker_role.yml decides for a host not marked eligible: the inventory's. + "github_runner_cluster_docker_role": role, "check_tasks": "topology,running_version", "check_hosts": "all", })) @@ -620,28 +624,28 @@ def test_a_compose_file_path_with_a_colon_is_refused(self) -> None: result = self.run_checks({"node1": {"github_runner_cluster_dir": "/srv/node1"}}, "-e", '{"github_runner_cluster_compose_files": ["/srv/extra.yml"]}', check_tasks="compose_files") self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - def render_env(self, *extra: str) -> tuple[list[str], list[str]]: - env, agent = self.work / "env", self.work / "env-agent" - args = [ANSIBLE_PLAYBOOK, str(ROOT / "tests" / "cluster_versions" / "render_env.yml"), "-e", f"env_output={env}", "-e", f"env_agent_output={agent}", "-e", "github_runner_cluster_dir=/srv/github-runner", *extra] + def render_env(self, *extra: str) -> list[str]: + env = self.work / "env" + args = [ANSIBLE_PLAYBOOK, str(ROOT / "tests" / "cluster_versions" / "render_env.yml"), "-e", f"env_output={env}", "-e", "github_runner_cluster_dir=/srv/github-runner", *extra] result = subprocess.run(args, capture_output=True, text=True, check=False, env=ansible_env(), cwd=self.work) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - return env.read_text().splitlines(), agent.read_text().splitlines() + return env.read_text().splitlines() - def test_the_env_files_carry_both_versions(self) -> None: - for lines in self.render_env("-e", "github_runner_cluster_tailscale_version=v1.90.0"): - self.assertIn(f"K3S_VERSION={TESTED}", lines) - self.assertIn("TAILSCALE_VERSION=v1.90.0", lines) + def test_the_env_file_carries_both_versions(self) -> None: + lines = self.render_env("-e", "github_runner_cluster_tailscale_version=v1.90.0") + self.assertIn(f"K3S_VERSION={TESTED}", lines) + self.assertIn("TAILSCALE_VERSION=v1.90.0", lines) - def test_the_env_files_name_the_compose_files_and_project(self) -> None: - for lines in self.render_env("-e", '{"github_runner_cluster_compose_files": ["compose.extra.yml", "/elsewhere/absolute.yml"], "github_runner_cluster_compose_project": "example"}'): - self.assertIn("COMPOSE_FILE=/srv/github-runner/docker-compose.yml:/srv/github-runner/compose.extra.yml:/elsewhere/absolute.yml", lines) - self.assertIn("COMPOSE_PROJECT_NAME=example", lines) + def test_the_env_file_names_the_compose_files_and_project(self) -> None: + lines = self.render_env("-e", '{"github_runner_cluster_compose_files": ["compose.extra.yml", "/elsewhere/absolute.yml"], "github_runner_cluster_compose_project": "example"}') + self.assertIn("COMPOSE_FILE=/srv/github-runner/docker-compose.yml:/srv/github-runner/compose.extra.yml:/elsewhere/absolute.yml", lines) + self.assertIn("COMPOSE_PROJECT_NAME=example", lines) @unittest.skipUnless(has_compose(), "docker compose is not installed") def test_a_compose_command_run_by_hand_uses_the_role_s_files_and_project(self) -> None: project = self.work / "project" project.mkdir() - server_env, _ = self.render_env("-e", json.dumps({"github_runner_cluster_compose_files": ["compose.extra.yml"], "github_runner_cluster_compose_project": "example", "github_runner_cluster_dir": str(project)})) + server_env = self.render_env("-e", json.dumps({"github_runner_cluster_compose_files": ["compose.extra.yml"], "github_runner_cluster_compose_project": "example", "github_runner_cluster_dir": str(project)})) shutil.copy(COMPOSE, project / "docker-compose.yml") (project / "compose.extra.yml").write_text("services:\n k3s:\n labels:\n example.extra: applied\n") (project / ".env").write_text("\n".join(server_env) + "\n") diff --git a/tests/unit/test_cluster_voters.py b/tests/unit/test_cluster_voters.py index 9f6e8e2b..55041acd 100644 --- a/tests/unit/test_cluster_voters.py +++ b/tests/unit/test_cluster_voters.py @@ -130,46 +130,43 @@ def test_a_pinned_host_does_not_count_as_eligible(self) -> None: def test_a_datastore_with_no_etcd_is_refused(self) -> None: self.refused("has no etcd votes to move", **{CONTROLLER: True, "github_runner_cluster_datastore": "sqlite"}) - def render_env(self, *extra: str) -> tuple[list[str], list[str], list[str]]: - env, agent, files = self.work / "env", self.work / "env-agent", self.work / "files.json" - args = [ANSIBLE_PLAYBOOK, str(ROOT / "tests" / "cluster_versions" / "render_env.yml"), "-e", f"env_output={env}", "-e", f"env_agent_output={agent}", "-e", f"compose_files_output={files}", "-e", "github_runner_cluster_dir=/srv/github-runner", *extra] + def render_env(self, *extra: str) -> tuple[list[str], list[str]]: + env, files = self.work / "env", self.work / "files.json" + args = [ANSIBLE_PLAYBOOK, str(ROOT / "tests" / "cluster_versions" / "render_env.yml"), "-e", f"env_output={env}", "-e", f"compose_files_output={files}", "-e", "github_runner_cluster_dir=/srv/github-runner", *extra] result = subprocess.run(args, capture_output=True, text=True, check=False, env=ansible_env(), cwd=self.work) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - return env.read_text().splitlines(), agent.read_text().splitlines(), json.loads(files.read_text()) + return env.read_text().splitlines(), json.loads(files.read_text()) def test_a_host_that_uses_none_of_it_keeps_its_configuration_exactly(self) -> None: - server, agent, files = self.render_env() - self.assertEqual([line for line in server + agent if "LABEL" in line], []) + lines, files = self.render_env() + self.assertEqual([line for line in lines if "LABEL" in line], []) self.assertEqual(files, ["docker-compose.yml"], "no override file, so an upgrade recreates no container") def test_an_eligible_host_publishes_eligible_true_and_pinned_false_in_its_env_and_its_compose_files(self) -> None: - server, agent, files = self.render_env("-e", f"{ELIGIBLE}=true") - expected = f"K3S_NODE_LABELS={ELIGIBLE_LABEL}=true {PINNED_LABEL}=false {LINK_LABEL}=wired" - self.assertIn(expected, server) - self.assertIn(expected, agent) + # One variable, which both services read through docker-compose.node-labels.yml. + lines, files = self.render_env("-e", f"{ELIGIBLE}=true") + self.assertIn(f"K3S_NODE_LABELS={ELIGIBLE_LABEL}=true {PINNED_LABEL}=false {LINK_LABEL}=wired", lines) self.assertEqual(files, ["docker-compose.yml", "docker-compose.node-labels.yml", "docker-compose.fleet-role.yml"]) def test_a_pinned_host_publishes_pinned_true(self) -> None: - server, _, _ = self.render_env("-e", f"{PINNED}=true") - self.assertIn(f"K3S_NODE_LABELS={ELIGIBLE_LABEL}=false {PINNED_LABEL}=true {LINK_LABEL}=wired", server) + lines, _ = self.render_env("-e", f"{PINNED}=true") + self.assertIn(f"K3S_NODE_LABELS={ELIGIBLE_LABEL}=false {PINNED_LABEL}=true {LINK_LABEL}=wired", lines) def test_with_the_controller_on_every_host_publishes_both_as_false_so_that_a_setting_turned_off_is_written_over(self) -> None: - server, agent, files = self.render_env("-e", f"{CONTROLLER}=true") - for lines in (server, agent): - self.assertIn(f"K3S_NODE_LABELS={ELIGIBLE_LABEL}=false {PINNED_LABEL}=false {LINK_LABEL}=wired", lines) + lines, files = self.render_env("-e", f"{CONTROLLER}=true") + self.assertIn(f"K3S_NODE_LABELS={ELIGIBLE_LABEL}=false {PINNED_LABEL}=false {LINK_LABEL}=wired", lines) self.assertIn("docker-compose.node-labels.yml", files) self.assertIn("docker-compose.fleet-role.yml", files) def test_the_label_names_can_be_changed(self) -> None: - server, _, _ = self.render_env("-e", f"{ELIGIBLE}=true", "-e", "github_runner_cluster_voter_eligible_label=example.com/candidate") - self.assertIn(f"K3S_NODE_LABELS=example.com/candidate=true {PINNED_LABEL}=false {LINK_LABEL}=wired", server) + lines, _ = self.render_env("-e", f"{ELIGIBLE}=true", "-e", "github_runner_cluster_voter_eligible_label=example.com/candidate") + self.assertIn(f"K3S_NODE_LABELS=example.com/candidate=true {PINNED_LABEL}=false {LINK_LABEL}=wired", lines) def test_the_link_label_carries_the_medium_read_on_the_host(self) -> None: for medium in ("wifi", "unknown"): with self.subTest(medium): - server, agent, _ = self.render_env("-e", f"{CONTROLLER}=true", "-e", f"github_runner_cluster_link={medium}", "-e", "github_runner_cluster_link_label=example.com/link") - for lines in (server, agent): - self.assertIn(f"K3S_NODE_LABELS={ELIGIBLE_LABEL}=false {PINNED_LABEL}=false example.com/link={medium}", lines) + lines, _ = self.render_env("-e", f"{CONTROLLER}=true", "-e", f"github_runner_cluster_link={medium}", "-e", "github_runner_cluster_link_label=example.com/link") + self.assertIn(f"K3S_NODE_LABELS={ELIGIBLE_LABEL}=false {PINNED_LABEL}=false example.com/link={medium}", lines) class ComposeOverrideTest(unittest.TestCase): diff --git a/tests/unit/test_docker_role.py b/tests/unit/test_docker_role.py new file mode 100644 index 00000000..2de31bdf --- /dev/null +++ b/tests/unit/test_docker_role.py @@ -0,0 +1,336 @@ +"""Tests for how the cluster role records a Docker host's role on the host and keeps it across runs: the filters (plugins/filter/cluster_role.py's cluster_host_role and plugins/filter/cluster_guard.py's cluster_role_lease_guard), the decision and its refusals (tasks/docker_role.yml), the record (tasks/docker_role_record.yml), the servers' turns and the regrade service following the recorded role (tasks/start_servers.yml, tasks/regrade_service.yml), the refusal of every run while the role controller holds the regrade Lease (tasks/role_lease_guard.yml), and the one .env both services read (templates/env.j2), run through the role (tests/node_password/run.yml, tests/regrade/service.yml, tests/cluster_versions/render_env.yml) against the stand-ins for docker and the Compose module, so that no real Docker daemon or cluster is reached.""" + +from __future__ import annotations + +import importlib.util +import json +import os +import re +import subprocess +import tempfile +import time +import unittest +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import yaml + +from cluster_stand_ins import ANSIBLE_PLAYBOOK, HASH_NEW, ROLE, ROOT, RoleTestCase, ansible_env, flatten, has_compose, recap, role_tasks + + +def _filters(name: str) -> Any: + """The filter plugin plugins/filter/.py, loaded from its file.""" + spec = importlib.util.spec_from_file_location(name, ROOT / "plugins" / "filter" / f"{name}.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +cluster_role = _filters("cluster_role") +cluster_guard = _filters("cluster_guard") + +DEFAULTS: dict[str, Any] = yaml.safe_load((ROLE / "defaults" / "main.yml").read_text()) +HOLDER = DEFAULTS["github_runner_cluster_role_lease_holder"] +LEASE_NAME = DEFAULTS["github_runner_cluster_regrade_lease_name"] +LEASE_NAMESPACE = DEFAULTS["github_runner_cluster_regrade_lease_namespace"] +SWAP_ID = DEFAULTS["github_runner_cluster_role_lease_swap_id_annotation"] +COMPOSE = ROLE / "files" / "docker-compose.yml" +# Long enough that a Lease renewed this long ago is still unexpired however slowly a test runs, and the time it has run out by. +DURATION = 3600 + + +def lease(holder: str, renewed_ago: int, duration: int = DURATION, **annotations: str) -> str: + """A Lease as `kubectl get lease -o json` prints it, held by ``holder`` and last renewed ``renewed_ago`` seconds ago.""" + renewed = datetime.fromtimestamp(time.time() - renewed_ago, UTC).strftime("%Y-%m-%dT%H:%M:%S.000000Z") + return json.dumps({"apiVersion": "coordination.k8s.io/v1", "kind": "Lease", "metadata": {"name": LEASE_NAME, "namespace": LEASE_NAMESPACE, "annotations": annotations}, "spec": {"holderIdentity": holder, "leaseDurationSeconds": duration, "acquireTime": renewed, "renewTime": renewed}}, indent=2) + + +def read(host: str, stdout: str, rc: int = 0, **extra: Any) -> dict[str, Any]: + """One server's read of the Lease, as the guard registers it: ``stopped``, or ``running``, the reading host's clock as epoch seconds and the Lease as kubectl prints it (nothing when there is none).""" + return {"item": host, "rc": rc, "stdout": stdout, "stderr": "", **extra} + + +def running(host: str, text: str = "", now: float | None = None) -> dict[str, Any]: + return read(host, f"running\n{int(time.time() if now is None else now)}\n{text}") + + +class LeaseGuardFilterTest(unittest.TestCase): + @staticmethod + def guard(reads: Any, holder: str) -> dict[str, str]: + verdict: dict[str, str] = cluster_guard.cluster_role_lease_guard(reads, holder, SWAP_ID) + return verdict + + def test_a_lease_the_controller_holds_and_has_renewed_in_time_is_held(self) -> None: + verdict = self.guard([running("server1", lease(HOLDER, 60, **{SWAP_ID: "20990101T000000Z-server1-laptop"}))], HOLDER) + self.assertEqual(verdict["state"], "held") + self.assertIn("20990101T000000Z-server1-laptop", verdict["message"]) + self.assertIn("server1", verdict["message"]) + + def test_a_lease_that_has_run_out_another_holder_and_no_holder_are_free(self) -> None: + for text in (lease(HOLDER, DURATION + 60), lease("server2", 60), lease("", 60), ""): + with self.subTest(text=text[:80]): + self.assertEqual(self.guard([running("server1", text)], HOLDER)["state"], "free") + + def test_a_lease_with_no_renewal_counts_from_when_it_was_taken(self) -> None: + taken = json.loads(lease(HOLDER, 60)) + del taken["spec"]["renewTime"] + self.assertEqual(self.guard([running("server1", json.dumps(taken))], HOLDER)["state"], "held") + + def test_expiry_is_judged_by_the_reading_hosts_clock(self) -> None: + text = lease(HOLDER, 60) + self.assertEqual(self.guard([running("server1", text, now=time.time() + DURATION)], HOLDER)["state"], "free") + + def test_no_server_running_is_no_cluster_and_no_lease(self) -> None: + self.assertEqual(self.guard([read("server1", "stopped"), read("server2", "stopped")], HOLDER)["state"], "free") + self.assertEqual(self.guard([], HOLDER)["state"], "free") + + def test_one_answer_decides_and_a_server_that_could_not_be_read_does_not_matter_then(self) -> None: + unread = read("server1", "", rc=1, stderr="Cannot connect to the Docker daemon") + self.assertEqual(self.guard([unread, running("server2", lease(HOLDER, 60))], HOLDER)["state"], "held") + self.assertEqual(self.guard([unread, running("server2", "")], HOLDER)["state"], "free") + + def test_no_answer_with_a_server_that_could_not_be_read_cannot_be_told(self) -> None: + verdict = self.guard([read("server1", "", rc=1, stderr="Cannot connect to the Docker daemon"), read("server2", "stopped"), read("server3", "", unreachable=True, msg="ssh: connect timed out")], HOLDER) + self.assertEqual(verdict["state"], "unknown") + self.assertIn("server1 could not be read (Cannot connect to the Docker daemon)", verdict["message"]) + self.assertIn("server3 could not be read (ssh: connect timed out)", verdict["message"]) + + def test_an_answer_that_is_not_a_lease_is_an_error(self) -> None: + for stdout in ("running\nnot-a-time\n", "running\n0\nnot json", "running\n0\n[]", "something else"): + with self.subTest(stdout=stdout), self.assertRaises(cluster_guard.AnsibleFilterError): + self.guard([read("server1", stdout)], HOLDER) + + def test_the_filters_are_registered(self) -> None: + self.assertEqual(sorted(cluster_role.FilterModule().filters()), ["cluster_host_role"]) + self.assertIn("cluster_role_lease_guard", cluster_guard.FilterModule().filters()) + + +class DockerRoleTest(RoleTestCase): + """docker_role.yml, docker_role_record.yml, start_servers.yml and role_lease_guard.yml, through the role against the stand-ins.""" + + def setUp(self) -> None: + super().setUp() + self.inventory = self.hosts(("node1", "node2", "node3"), ("node4",), eligible=("node3", "node4")) + # Every run of the role creates the role directory before it records a role (tasks/main.yml). + for host in self.inventory[0]: + self.role_file(host).parent.mkdir() + + def role_file(self, host: str) -> Path: + return self.work / "nodes" / host / "fleet-role.d" / "role" + + def record(self, host: str, role: str) -> None: + self.role_file(host).write_text(f"{role}\n") + + def tasks(self, tasks: str, check: bool = False, limit: str | None = None) -> subprocess.CompletedProcess[str]: + hosts, group_vars = self.inventory + return self.run_tasks(tasks, hosts, group_vars, check=check, limit=limit) + + def roles_reported(self, result: subprocess.CompletedProcess[str]) -> dict[str, str]: + """The role each host's run decided on, as docker_role.yml reports it.""" + return dict(re.findall(r'"msg": "(node\d) is to run as (server|agent)\b', result.stdout)) + + def test_an_eligible_host_keeps_the_role_it_records_over_the_inventory(self) -> None: + self.record("node3", "agent") + self.record("node4", "server") + result = self.tasks("docker_role") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.roles_reported(result), {"node1": "server", "node2": "server", "node3": "agent", "node4": "server"}) + + def test_a_host_that_is_not_eligible_follows_the_inventory_and_a_new_eligible_host_starts_from_it(self) -> None: + result = self.tasks("docker_role") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.roles_reported(result), {"node1": "server", "node2": "server", "node3": "server", "node4": "agent"}) + + def test_the_role_is_recorded_for_an_eligible_host_only_and_only_when_absent(self) -> None: + # Written as the role controller may write it, without the newline a run of the role ends its own record with, so that writing over it would change its bytes. + self.role_file("node4").write_text("server") + result = self.tasks("docker_role,docker_role_record") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.role_file("node3").read_text(), "server\n") + self.assertEqual(self.role_file("node4").read_text(), "server", "a recorded role is never written over") + for host in ("node1", "node2"): + self.assertFalse(self.role_file(host).exists(), host) + self.assertEqual((recap(result.stdout, "node3")["changed"], recap(result.stdout, "node4")["changed"]), (1, 0)) + + def test_the_servers_turns_follow_the_recorded_role(self) -> None: + self.record("node3", "agent") + self.record("node4", "server") + for name in ("node1", "node2", "node4"): + self.rendered(name, HASH_NEW) + result = self.tasks("docker_role,start_servers") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertCountEqual([call for call in self.calls() if call.startswith("module up")], [f"module up node{n} profiles=server check=False" for n in (1, 2, 4)]) + + def refused(self, host: str, *needles: str, tasks: str = "docker_role", check: bool = False) -> subprocess.CompletedProcess[str]: + result = self.tasks(tasks, check=check) + self.assertNotEqual(result.returncode, 0) + self.assertIn(f"fatal: [{host}]", result.stdout) + flat = " ".join(result.stdout.split()) + for needle in needles: + self.assertIn(needle, flat) + self.assertEqual(self.mutating(), [], "nothing was changed") + return result + + def test_a_host_that_is_not_eligible_and_runs_the_other_role_is_refused(self) -> None: + self.container("node2-k3s-agent") + result = self.refused("node2", "its agent container (node2-k3s-agent) runs, but the inventory gives it the role server", "Nothing was changed.") + self.assertNotIn("fatal: [node1]", result.stdout) + + def test_an_eligible_host_whose_container_differs_from_its_record_is_refused(self) -> None: + self.record("node3", "agent") + self.container("node3-k3s") + self.refused("node3", "its server container (node3-k3s) runs, but its role directory records agent") + + def test_an_eligible_host_with_no_record_whose_container_differs_from_the_inventory_is_refused_before_anything_is_recorded(self) -> None: + self.container("node4-k3s") + self.refused("node4", "its server container (node4-k3s) runs, but the inventory gives it the role agent", tasks="docker_role,docker_role_record") + self.assertFalse(self.role_file("node4").exists()) + + def test_both_containers_running_is_refused(self) -> None: + self.container("node1-k3s") + self.container("node1-k3s-agent") + self.refused("node1", "both of its node containers run (node1-k3s and node1-k3s-agent)") + + def test_a_record_left_on_a_host_that_is_no_longer_eligible_is_refused(self) -> None: + self.record("node2", "agent") + self.refused("node2", "its role directory records agent", "the inventory gives it the role server", str(self.role_file("node2"))) + + def test_every_run_is_refused_while_the_role_controller_holds_the_lease(self) -> None: + self.container("node1-k3s") + self.write("fleet-regrade.lease", text=lease(HOLDER, 60, **{SWAP_ID: "20990101T000000Z-node2-node4"})) + result = self.tasks("role_lease_guard") + self.assertNotEqual(result.returncode, 0) + for n in (1, 2, 3, 4): + self.assertIn(f"fatal: [node{n}]", result.stdout) + self.assertIn("20990101T000000Z-node2-node4", result.stdout) + self.assertIn(f"exec node1-k3s kubectl get lease {LEASE_NAME} --namespace {LEASE_NAMESPACE} --ignore-not-found", "\n".join(self.calls())) + + def test_a_lease_regrading_holds_or_one_that_has_run_out_lets_the_run_go_on(self) -> None: + self.container("node1-k3s") + for text in (lease("node2", 60), lease(HOLDER, DURATION + 60), ""): + with self.subTest(text=text[:60]): + self.write("fleet-regrade.lease", text=text) + result = self.tasks("role_lease_guard") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_a_cluster_that_cannot_be_read_refuses_and_one_that_is_not_running_does_not(self) -> None: + result = self.tasks("role_lease_guard") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.write("daemon-down", text="") + result = self.tasks("role_lease_guard") + self.assertNotEqual(result.returncode, 0) + self.assertIn("node1 could not be read", result.stdout) + + def test_check_mode_reports_the_refusals(self) -> None: + self.container("node2-k3s-agent") + self.refused("node2", "the inventory gives it the role server", check=True) + self.write("fleet-regrade.lease", text=lease(HOLDER, 60)) + self.container("node1-k3s") + result = self.tasks("role_lease_guard", check=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("fatal: [node3]", result.stdout) + self.assertIn(f"holds the Lease {LEASE_NAMESPACE}/{LEASE_NAME}", " ".join(result.stdout.split())) + + +class RegradeServiceTest(unittest.TestCase): + """regrade_service.yml for a host whose recorded role is agent, with regrading on, against a stand-in docker that records its calls.""" + + def test_the_regrade_service_is_removed_from_a_host_recorded_as_an_agent_and_never_started(self) -> None: + with tempfile.TemporaryDirectory() as work: + stub = Path(work) / "bin" / "docker" + stub.parent.mkdir() + stub.write_text('#!/bin/sh\necho "$*" >> "$STUB_CALLS"\ncase "$1" in rm) echo "$3" ;; *) echo "unexpected docker call: $*" >&2; exit 2 ;; esac\n') + stub.chmod(0o755) + calls = Path(work) / "calls" + env = {**ansible_env(), "PATH": f"{stub.parent}:{os.environ['PATH']}", "STUB_CALLS": str(calls)} + args = [ANSIBLE_PLAYBOOK, str(ROOT / "tests" / "regrade" / "service.yml"), "-e", "github_runner_cluster_regrade=true", "-e", "github_runner_cluster_docker_role=agent"] + result = subprocess.run(args, capture_output=True, text=True, check=False, env=env, cwd=work) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(calls.read_text().splitlines(), ["rm --force github-runner-regrade-1"]) + + +class EnvTest(unittest.TestCase): + """The one .env every Docker host gets (templates/env.j2), rendered by tests/cluster_versions/render_env.yml.""" + + def render(self, work: Path, *extra: str) -> str: + output = work / "env" + args = [ANSIBLE_PLAYBOOK, str(ROOT / "tests" / "cluster_versions" / "render_env.yml"), "-e", f"env_output={output}", "-e", "github_runner_cluster_dir=/srv/github-runner", "-e", "github_runner_cluster_effective_server_url=https://node1.example.internal:6443", *extra] + result = subprocess.run(args, capture_output=True, text=True, check=False, env=ansible_env(), cwd=work) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + return output.read_text() + + def test_there_is_one_template_and_it_is_the_same_for_either_role(self) -> None: + self.assertFalse((ROLE / "templates" / "env-agent.j2").exists()) + with tempfile.TemporaryDirectory() as work: + server = self.render(Path(work), "-e", "github_runner_cluster_effective_node_role=server", "-e", "github_runner_cluster_docker_role=server") + agent = self.render(Path(work), "-e", "github_runner_cluster_effective_node_role=agent", "-e", "github_runner_cluster_docker_role=agent") + self.assertEqual(server, agent) + lines = server.splitlines() + names = {line.split("=", 1)[1] for line in lines if line.startswith(("K3S_NODE_NAME=", "K3S_AGENT_NODE_NAME="))} + self.assertEqual(len(names), 1, "both services join under the host's one node name") + for line in ("K3S_JOIN_SERVER_URL=https://node1.example.internal:6443", "K3S_AGENT_SERVER_URL=https://node1.example.internal:6443", "K3S_CONTAINER_NAME=github-runner-k3s", "K3S_AGENT_CONTAINER_NAME=github-runner-k3s-agent"): + self.assertIn(line, lines) + + @unittest.skipUnless(has_compose(), "docker compose is not installed") + def test_each_service_renders_as_it_did_from_the_env_its_role_got_before(self) -> None: + # Each service's configuration hash, which Compose compares with the running container's to decide whether to recreate it, is the same from the one .env as from the .env a host of its role got before there was one: a server's held every variable but the agent's own, and an agent's held its own (its server URL and node name, from the same settings as the server's) beside the shared ones and the variables Compose requires of every service whichever profile is active. So an upgrade recreates no running node. + agent_own = ("K3S_AGENT_CONTAINER_NAME", "K3S_AGENT_SERVER_URL", "K3S_AGENT_NODE_NAME") + server_only = ("K3S_JOIN_SERVER_URL", "K3S_DATASTORE", "K3S_NODE_NAME", "K3S_ETCD_HEARTBEAT_INTERVAL", "K3S_ETCD_ELECTION_TIMEOUT") + clean = {key: value for key, value in os.environ.items() if not key.startswith(("DOCKER_", "COMPOSE_", "K3S_", "TAILSCALE_"))} + with tempfile.TemporaryDirectory() as work: + settings = {"github_runner_cluster_etcd_heartbeat_interval": 500, "github_runner_cluster_etcd_election_timeout": 5000, "github_runner_cluster_datastore": "sqlite", "github_runner_cluster_node_taints": ["example.com/taint=true:NoSchedule"], "github_runner_cluster_mesh_control_server_url": "https://headscale.example.internal"} + text = self.render(Path(work), "-e", json.dumps(settings)) + full = {line.split("=", 1)[0]: line.split("=", 1)[1] for line in text.splitlines() if "=" in line and not line.startswith(("#", "COMPOSE_"))} + before = { + "k3s": {key: value for key, value in full.items() if key not in agent_own}, + "k3s-agent": {key: value for key, value in full.items() if key not in server_only and key not in agent_own} + | {"K3S_AGENT_CONTAINER_NAME": full["K3S_AGENT_CONTAINER_NAME"], "K3S_AGENT_SERVER_URL": full["K3S_JOIN_SERVER_URL"], "K3S_AGENT_NODE_NAME": full["K3S_NODE_NAME"]}, + } + + def config_hash(service: str, variables: dict[str, str], name: str) -> str: + env_file = Path(work) / f"{service}-{name}.env" + env_file.write_text("".join(f"{key}={value}\n" for key, value in variables.items())) + result = subprocess.run(["docker", "compose", "--project-directory", work, "--project-name", "example", "--env-file", str(env_file), "-f", str(COMPOSE), "--profile", "server", "--profile", "agent", "config", "--hash", service], capture_output=True, text=True, check=False, env=clean) + self.assertEqual(result.returncode, 0, result.stderr) + return result.stdout.split()[1] + + for service, variables in before.items(): + with self.subTest(service=service): + self.assertEqual(config_hash(service, full, "now"), config_hash(service, variables, "before")) + + +class TaskStructureTest(unittest.TestCase): + def test_the_docker_path_decides_the_role_and_checks_the_lease_before_it_writes_anything(self) -> None: + tasks = role_tasks("main.yml") + includes = [task.get("ansible.builtin.include_tasks") for task in tasks] + decided, guarded = includes.index("docker_role.yml"), includes.index("role_lease_guard.yml") + self.assertEqual((includes.index("topology.yml") + 1, decided + 1), (decided, guarded)) + first_write = next(index for index, task in enumerate(tasks) if "ansible.builtin.file" in task) + self.assertLess(guarded, first_write) + self.assertEqual(includes.index("docker_role_record.yml"), first_write + 1) + + def test_the_native_path_checks_the_lease_right_after_its_topology(self) -> None: + includes = [task.get("ansible.builtin.include_tasks") for task in role_tasks("native/main.yml")] + self.assertEqual(includes.index("role_lease_guard.yml"), includes.index("topology.yml") + 1) + + def test_the_profile_the_container_and_the_regrade_service_follow_the_recorded_role(self) -> None: + tasks = role_tasks("main.yml") + conditions = [str(task.get("when", "")) for task in tasks if "block" in task] + self.assertIn('github_runner_cluster_docker_role == "server"', conditions) + self.assertIn('github_runner_cluster_docker_role == "agent"', conditions) + self.assertNotIn("effective_node_role", (ROLE / "tasks" / "main.yml").read_text()) + for name in ("github_runner_cluster_node_container", "github_runner_cluster_node_service"): + self.assertIn("github_runner_cluster_docker_role", DEFAULTS[name]) + self.assertIn("regrade_service.yml", [task.get("ansible.builtin.include_tasks") for task in flatten(tasks) if "block" not in task]) + self.assertIn("github_runner_cluster_docker_role", (ROLE / "tasks" / "regrade_service.yml").read_text()) + + def test_the_recorded_role_file_is_the_one_the_compose_override_mounts(self) -> None: + override = yaml.safe_load((ROLE / "files" / "docker-compose.fleet-role.yml").read_text()) + mounted = override["services"]["k3s"]["volumes"][0].split(":")[0] + self.assertEqual(DEFAULTS["github_runner_cluster_docker_role_file"], "{{ github_runner_cluster_dir }}/" + mounted.removeprefix("./") + "/role") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_host_network_role.py b/tests/unit/test_host_network_role.py index e85f333d..8cd1f25b 100644 --- a/tests/unit/test_host_network_role.py +++ b/tests/unit/test_host_network_role.py @@ -105,6 +105,8 @@ def run_tasks(self, *args: str, limit: str = "node1", timeout: int | None = None "github_runner_cluster_topology": TOPOLOGY, **role_vars, } + # The role tasks/docker_role.yml would decide: the inventory's, for a host not marked eligible, unless a test gives the role an eligible host records. + extra.setdefault("github_runner_cluster_docker_role", extra["github_runner_cluster_effective_node_role"]) extra_file = self.work / "extra.json" extra_file.write_text(json.dumps(extra)) env = { @@ -188,6 +190,14 @@ def test_an_agent_in_a_group_with_no_bootstrap_host_reads_the_servers(self) -> N self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertIn("docker - exec node1-k3s kubectl get nodes --request-timeout=30s -o json", self.calls()) + def test_an_eligible_host_recorded_as_an_agent_is_read_through_the_other_servers_and_not_itself(self) -> None: + result = self.run_tasks(github_runner_cluster_docker_role="agent") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + calls = self.calls() + self.assertNotIn("docker - exec github-runner-k3s kubectl get nodes --request-timeout=30s -o json", calls) + self.assertIn("docker ctx2 exec node2-k3s kubectl get nodes --request-timeout=30s -o json", calls) + self.assertFalse([call for call in calls if call.endswith("--raw /readyz/etcd")], "an agent has no etcd vote for the others to keep") + def test_an_agent_whose_servers_are_outside_the_inventory_says_why_it_cannot_move(self) -> None: topology = {"hosts": {"node2": {"role": "agent", "bootstrap": False, "node_name": "node2"}}} result = self.run_tasks(limit="node2", github_runner_cluster_effective_node_role="agent", github_runner_cluster_effective_node_name="node2", github_runner_cluster_topology=topology) diff --git a/tests/unit/test_native_node.py b/tests/unit/test_native_node.py index 319b84c6..e5fef792 100644 --- a/tests/unit/test_native_node.py +++ b/tests/unit/test_native_node.py @@ -10,7 +10,9 @@ import stat import subprocess import tempfile +import time import unittest +from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -27,6 +29,10 @@ assert _spec is not None and _spec.loader is not None cluster_native = importlib.util.module_from_spec(_spec) _spec.loader.exec_module(cluster_native) +_role_spec = importlib.util.spec_from_file_location("cluster_role", ROOT / "plugins" / "filter" / "cluster_role.py") +assert _role_spec is not None and _role_spec.loader is not None +cluster_role = importlib.util.module_from_spec(_role_spec) +_role_spec.loader.exec_module(cluster_role) class FilterTest(unittest.TestCase): @@ -87,21 +93,21 @@ def test_a_status_that_is_not_the_clients_is_refused(self) -> None: cluster_native.cluster_native_tailnet_login(tailscale_status("Running", DOMAIN), "") def test_a_role_file_names_the_role_now_and_a_legacy_agent_counts_as_one(self) -> None: - role = cluster_native.cluster_native_role + role = cluster_role.cluster_host_role self.assertEqual(role("", False, False, "server"), {"current": "", "desired": "server"}) self.assertEqual(role("", True, False, "agent"), {"current": "agent", "desired": "agent"}) self.assertEqual(role("agent\n", True, False, "server"), {"current": "agent", "desired": "server"}) self.assertEqual(role("server", False, False, "agent"), {"current": "server", "desired": "agent"}) def test_an_eligible_host_keeps_the_role_its_file_names(self) -> None: - role = cluster_native.cluster_native_role + role = cluster_role.cluster_host_role self.assertEqual(role("agent", True, True, "server"), {"current": "agent", "desired": "agent"}) self.assertEqual(role("server", False, True, "agent"), {"current": "server", "desired": "server"}) # The inventory gives an eligible host the role it starts with. self.assertEqual(role("", False, True, "server"), {"current": "", "desired": "server"}) def test_while_exactly_one_unit_runs_its_role_is_the_role_now(self) -> None: - role = cluster_native.cluster_native_role + role = cluster_role.cluster_host_role # A demotion that stopped after the role file and before the server unit: the next run takes it up again. self.assertEqual(role("agent", True, False, "agent", ["server"]), {"current": "server", "desired": "agent"}) # A promotion that stopped after the role file and before the agent unit. @@ -113,7 +119,7 @@ def test_while_exactly_one_unit_runs_its_role_is_the_role_now(self) -> None: def test_a_role_file_holding_anything_else_is_refused(self) -> None: with self.assertRaisesRegex(Exception, "must be 'server' or 'agent'"): - cluster_native.cluster_native_role("controller", False, False, "agent") + cluster_role.cluster_host_role("controller", False, False, "agent") def test_a_cluster_is_started_only_when_no_other_server_runs_one(self) -> None: init = cluster_native.cluster_native_cluster_init @@ -665,6 +671,18 @@ def test_a_downgrade_is_refused_before_anything_is_changed(self) -> None: self.assertEqual(self.mutations(), []) self.assertIn("v1.35.0", installed.read_text()) + def test_a_run_is_refused_before_anything_is_changed_while_the_role_controller_holds_the_lease(self) -> None: + renewed = datetime.fromtimestamp(time.time() - 60, UTC).strftime("%Y-%m-%dT%H:%M:%S.000000Z") + held = {"metadata": {"name": "fleet-regrade", "namespace": "kube-system", "annotations": {"github-runner.exadev.com/swap-id": "20990101T000000Z-server1-laptop"}}, "spec": {"holderIdentity": "fleet-role", "leaseDurationSeconds": 3600, "renewTime": renewed}} + (self.state / "fleet-regrade-lease").write_text(json.dumps(held)) + for check in (False, True): + with self.subTest(check=check): + result = self.run_role(check=check) + self.assertNotEqual(result.returncode, 0) + self.assertIn("holds the Lease kube-system/fleet-regrade for the voter swap 20990101T000000Z-server1-laptop", re.sub(r"\s+", " ", result.stdout)) + self.assertEqual(self.mutations(), []) + self.assertFalse(self.file("github_runner_cluster_native_bin_dir", "k3s").exists()) + def test_a_deliberate_jump_is_allowed_when_asked_for(self) -> None: installed = self.file("github_runner_cluster_native_bin_dir", "k3s") installed.write_text('#!/bin/sh\necho "k3s version v1.35.0+k3s1 (stand-in)"\n') @@ -675,7 +693,8 @@ def test_a_deliberate_jump_is_allowed_when_asked_for(self) -> None: def test_the_run_ends_with_a_node_that_a_server_sees_ready_and_a_node_that_is_not_is_reported(self) -> None: self.assertEqual(self.run_role().returncode, 0) - reads = [call for call in self.calls() if call.startswith("docker exec") and (" kubectl get node " in call or " kubectl get lease " in call)] + # The node's own reads, not the regrade Lease's, which the run reads once before it changes anything (tasks/role_lease_guard.yml). + reads = [call for call in self.calls() if call.startswith("docker exec") and (" kubectl get node " in call or " kubectl get lease " in call) and " lease fleet-regrade " not in call] self.assertEqual(len(reads), 2) (self.state / "node-ready").write_text("False,") (self.state / "lease").write_text("2000-01-01T00:00:00Z") diff --git a/tests/unit/test_node_password.py b/tests/unit/test_node_password.py index 57897465..3d4c6ef4 100644 --- a/tests/unit/test_node_password.py +++ b/tests/unit/test_node_password.py @@ -147,7 +147,7 @@ def test_check_mode_with_nothing_to_seed_reports_nothing(self) -> None: def test_an_agent_is_seeded_into_its_own_volume(self) -> None: hosts, group_vars = self.hosts(("node1",), ("agent1",)) - self.container("github-runner-k3s-agent", password=self.SECRET) + self.container("agent1-k3s-agent", password=self.SECRET) result = self.run_tasks("node_password", hosts, group_vars, limit="agent1") self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertEqual((self.stub / "volumes" / "agent1_k3s-agent-node" / "password").read_text(), self.SECRET) diff --git a/tests/unit/test_runtime_autostart.py b/tests/unit/test_runtime_autostart.py index 1b109484..f17bc854 100644 --- a/tests/unit/test_runtime_autostart.py +++ b/tests/unit/test_runtime_autostart.py @@ -145,7 +145,7 @@ def test_the_runtime_is_up_before_anything_reads_the_daemon(self) -> None: def test_the_end_checks_follow_the_compose_start_on_servers_and_agents(self) -> None: tasks = yaml.safe_load((ROLE / "tasks" / "main.yml").read_text()) for role in ("server", "agent"): - [block] = [task for task in tasks if "block" in task and task.get("when") == f'github_runner_cluster_effective_node_role == "{role}"'] + [block] = [task for task in tasks if "block" in task and task.get("when") == f'github_runner_cluster_docker_role == "{role}"'] # A server starts through its turn (tasks/start_servers.yml) and an agent through tasks/start_agent.yml, each of which runs the Compose start. steps = [("compose" if step.get("ansible.builtin.include_tasks") in ("start_servers.yml", "start_agent.yml") else step.get("ansible.builtin.include_tasks")) or ("compose" if "community.docker.docker_compose_v2" in step else None) for step in block["block"]] with self.subTest(role=role): @@ -433,7 +433,7 @@ def run_role(self, *extra: str, tasks: str = "runtime,end_checks", context: str "runtime_tasks": tasks, "github_runner_cluster_docker_cli_context": context, "github_runner_cluster_docker_cli_environment": {"DOCKER_CONTEXT": context} if context else {}, - "github_runner_cluster_effective_node_role": "server", + "github_runner_cluster_docker_role": "server", "github_runner_cluster_runtime_path": f"{self.bin}{os.pathsep}/usr/bin{os.pathsep}/bin", # The tests are not root, so the daemon's property list is owned by the user running them and sits beside the stand-in daemon directory. "runtime_user": "example", @@ -949,11 +949,11 @@ def test_the_end_checks_catch_a_runtime_that_went_down_during_the_run(self) -> N def test_an_agent_host_is_checked_for_the_agent_container(self) -> None: self.state("runtime-up") - result = self.run_role(github_runner_cluster_runtime_autostart=False, github_runner_cluster_effective_node_role="agent") + result = self.run_role(github_runner_cluster_runtime_autostart=False, github_runner_cluster_docker_role="agent") self.assertNotEqual(result.returncode, 0) self.assertIn("The k3s container github-runner-k3s-agent on node1 is", result.stdout) (self.stub / "github-runner-k3s-agent.state").write_text("running") - self.assertEqual(self.run_role(github_runner_cluster_runtime_autostart=False, github_runner_cluster_effective_node_role="agent").returncode, 0) + self.assertEqual(self.run_role(github_runner_cluster_runtime_autostart=False, github_runner_cluster_docker_role="agent").returncode, 0) def test_a_container_renamed_by_an_extra_compose_file_is_the_one_checked(self) -> None: self.state("runtime-up") diff --git a/tests/unit/test_server_recreate.py b/tests/unit/test_server_recreate.py index 6281d918..ab68ebf9 100644 --- a/tests/unit/test_server_recreate.py +++ b/tests/unit/test_server_recreate.py @@ -505,7 +505,9 @@ def test_a_turn_checks_before_it_starts_and_waits_only_afterwards(self) -> None: def test_the_turns_loop_over_the_servers_in_the_play_and_run_for_one_host_each(self) -> None: (task,) = role_tasks("start_servers.yml") self.assertEqual(task["ansible.builtin.include_tasks"], "start_server.yml") - self.assertIn("topology.servers", task["loop"]) + # The hosts that run as servers are those whose recorded role is server (tasks/docker_role.yml), which the inventory's list of servers does not say. + self.assertNotIn("topology", task["loop"]) + self.assertIn("github_runner_cluster_docker_role", task["loop"]) self.assertIn("ansible_play_hosts_all", task["loop"]) self.assertEqual(task["when"], f"{task['loop_control']['loop_var']} == inventory_hostname")