From 06a48d636dd6b47d7991e332c62ab77dac71772b Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 3 Nov 2026 07:21:00 +0000 Subject: [PATCH 01/10] feat(cluster): let a native host be a server when the inventory asks A native host was forced to an agent and refused the server role and the bootstrap. It is now a server when it sets the server role or the bootstrap (github_runner_cluster_node_role or github_runner_cluster_bootstrap), and stays an agent otherwise. It still takes no automatic server place and does not count towards the automatic number, so an inventory whose native hosts set neither keeps them all agents whatever max_servers allows. A host marked github_runner_cluster_voter_eligible is named on every server's TLS SAN list, so a standby is covered before it is promoted, and the topology publishes each host's mode, eligibility and the join URLs of the other servers. A native bootstrap host is refused under headscale_in_cluster, whose bootstrap needs the Docker node's supervisor. --- plugins/filter/cluster_topology.py | 16 ++-- .../github_runner_cluster/tasks/topology.yml | 13 ++++ tests/unit/test_cluster_topology.py | 74 ++++++++++++++++--- 3 files changed, 85 insertions(+), 18 deletions(-) diff --git a/plugins/filter/cluster_topology.py b/plugins/filter/cluster_topology.py index 9b21fac4..ba515106 100644 --- a/plugins/filter/cluster_topology.py +++ b/plugins/filter/cluster_topology.py @@ -34,6 +34,7 @@ def _describe_host(name: str, host_vars: Mapping[str, Any], errors: list[str]) - "name": name, "role_override": role if role in NODE_ROLES else "", "native": mode == "native", + "eligible": boolean(host_vars.get("github_runner_cluster_voter_eligible", False), strict=False), "bootstrap_override": None if bootstrap_override is None else boolean(bootstrap_override, strict=False), "node_name": node_name, "address": address, @@ -52,9 +53,9 @@ def _largest_odd_at_most(limit: int) -> int: def github_runner_cluster_topology(members: list[str], hostvars: Mapping[str, Mapping[str, Any]], max_servers: Any, api_port: Any) -> dict[str, Any]: """Decide every cluster member's part from inventory order and per-host overrides. - :param members: the cluster group's hosts, in inventory order. :param hostvars: Ansible's hostvars, read for each member's per-host overrides. :param max_servers: the most servers to choose automatically. :param api_port: the k3s API port a joining node connects to. :returns: ``errors`` (empty when the result is valid), ``servers`` and ``bootstrap`` (host names), and ``hosts``, mapping each member to its ``role``, ``bootstrap``, ``node_name``, ``address``, ``server_url`` and ``tls_sans``. :raises AnsibleFilterError: if max_servers or api_port is not an integer. + :param members: the cluster group's hosts, in inventory order. :param hostvars: Ansible's hostvars, read for each member's per-host overrides. :param max_servers: the most servers to choose automatically. :param api_port: the k3s API port a joining node connects to. :returns: ``errors`` (empty when the result is valid), ``servers`` and ``bootstrap`` (host names), and ``hosts``, mapping each member to its ``role``, ``bootstrap``, ``node_name``, ``address``, ``server_url``, ``tls_sans``, ``native``, ``eligible`` and ``peer_server_urls`` (the join URL of every other server, in inventory order). :raises AnsibleFilterError: if max_servers or api_port is not an integer. - The automatic server count is the largest odd number no greater than both the number of hosts that are not forced to be agents (a host with github_runner_cluster_node_mode: native is always an agent) and max_servers, because etcd needs a majority and an even member count tolerates no more failures than the odd count below it. The first hosts in inventory order become servers; the first server not told to join elsewhere bootstraps the cluster, and every other node joins it. + The automatic server count is the largest odd number no greater than both the number of hosts that are not forced to be agents and max_servers, because etcd needs a majority and an even member count tolerates no more failures than the odd count below it. A host with github_runner_cluster_node_mode: native is never given a server place automatically and does not count towards that number: it is a server only when it sets github_runner_cluster_node_role: server or github_runner_cluster_bootstrap: true, so an inventory whose native hosts set neither keeps every one of them an agent whatever max_servers allows. The first hosts in inventory order become servers; the first server not told to join elsewhere bootstraps the cluster, and every other node joins it. Every server's TLS SAN list names every server and every host marked github_runner_cluster_voter_eligible, so a standby that is later promoted is already covered by every certificate. """ try: server_limit = int(max_servers) @@ -66,11 +67,9 @@ def github_runner_cluster_topology(members: list[str], hostvars: Mapping[str, Ma if server_limit < 1: errors.append(f"github_runner_cluster_max_servers must be at least 1 (got {server_limit}).") hosts = [_describe_host(name, hostvars[name], errors) for name in members] - # A native node runs k3s as a systemd service on the host and only ever as an agent, so it is never offered a server place, and a host that asks for one is refused. + # A native node is a server only when its host asks to be one: with automatic placement it would otherwise take a server place silently once native hosts stopped being forced to agents, so a native host that sets neither the role nor the bootstrap is an agent. for host in hosts: - if host["native"]: - if host["role_override"] == "server" or host["bootstrap_override"] is True: - errors.append(f"{host['name']} sets github_runner_cluster_node_mode: native, which runs a k3s agent only; it cannot be a server or bootstrap the cluster.") + if host["native"] and not host["role_override"] and host["bootstrap_override"] is not True: host["role_override"] = "agent" # A host told to bootstrap has to be a server, so it takes one of the server places rather than competing for one in inventory order. for host in hosts: @@ -121,13 +120,14 @@ def github_runner_cluster_topology(members: list[str], hostvars: Mapping[str, Ma else: host["server_url"] = "" errors.append(f"{host['name']} has nothing to join: no host in the cluster group bootstraps the cluster, and it sets no github_runner_cluster_server_url.") - host["tls_sans"] = host["tls_sans_override"] or [server["address"] for server in servers] + host["tls_sans"] = host["tls_sans_override"] or [candidate["address"] for candidate in hosts if candidate["role"] == "server" or candidate["eligible"]] + host["peer_server_urls"] = [f"https://{server['address']}:{port}" for server in servers if server is not host] return { "errors": errors, "servers": [host["name"] for host in servers], "bootstrap": bootstrap_host["name"] if bootstrap_host is not None else "", - "hosts": {host["name"]: {key: host[key] for key in ("role", "bootstrap", "node_name", "address", "server_url", "tls_sans")} for host in hosts}, + "hosts": {host["name"]: {key: host[key] for key in ("role", "bootstrap", "node_name", "address", "server_url", "tls_sans", "native", "eligible", "peer_server_urls")} for host in hosts}, } diff --git a/roles/github_runner_cluster/tasks/topology.yml b/roles/github_runner_cluster/tasks/topology.yml index 6f071893..894833b4 100644 --- a/roles/github_runner_cluster/tasks/topology.yml +++ b/roles/github_runner_cluster/tasks/topology.yml @@ -34,6 +34,19 @@ msg: "{{ github_runner_cluster_topology.errors | join(' ') }}" when: github_runner_cluster_topology.errors | length > 0 +- name: Fail if a native host would bootstrap a cluster that runs its own Headscale server + # headscale_in_cluster's bootstrap server is supervised by k3s/node-entrypoint.sh, which starts k3s off the mesh, waits for the Headscale static pod, then restarts k3s on the mesh; a native server runs k3s straight from systemd with --vpn-auth and has no such supervisor, so it could never reach a control server that only it runs. Every host computes the same answer from the inventory. + ansible.builtin.fail: + msg: >- + {{ github_runner_cluster_topology.bootstrap }} sets github_runner_cluster_node_mode: native and would bootstrap the cluster, but + github_runner_cluster_mesh is headscale_in_cluster, whose bootstrap server runs the Headscale server as a static pod under a + supervisor that only the Docker node has. Bootstrap from a Docker host (github_runner_cluster_bootstrap: true on it), or use + another mesh provider. Nothing was changed. + when: + - github_runner_cluster_mesh == 'headscale_in_cluster' + - github_runner_cluster_topology.bootstrap | length > 0 + - github_runner_cluster_topology.hosts[github_runner_cluster_topology.bootstrap].native | bool + - name: Record this host's own part ansible.builtin.set_fact: github_runner_cluster_effective_node_role: "{{ github_runner_cluster_topology.hosts[inventory_hostname].role }}" diff --git a/tests/unit/test_cluster_topology.py b/tests/unit/test_cluster_topology.py index 9ed8e78b..aa6ed4e7 100644 --- a/tests/unit/test_cluster_topology.py +++ b/tests/unit/test_cluster_topology.py @@ -166,9 +166,9 @@ def test_an_unknown_node_role_is_an_error(self) -> None: class NativeNodeTest(unittest.TestCase): - """A host that runs k3s natively under systemd is only ever an agent.""" + """A host that runs k3s natively under systemd is a server only when it asks to be one, and never counts towards the automatic server number.""" - def test_a_native_host_is_an_agent_and_takes_no_server_place(self) -> None: + def test_a_native_host_that_sets_nothing_is_an_agent_and_takes_no_server_place(self) -> None: hosts = plain(4) hosts["h1"]["github_runner_cluster_node_mode"] = "native" result = topology(hosts) @@ -185,6 +185,16 @@ def test_native_hosts_do_not_count_towards_the_automatic_server_number(self) -> self.assertEqual(result["servers"], ["h1"]) self.assertEqual(result["hosts"]["h4"]["server_url"], "https://h1:6443") + def test_native_agents_stay_agents_however_many_servers_max_servers_allows(self) -> None: + # The fleet case the guard was for: three Docker servers, max_servers 5 and two native agents that set nothing. Letting native hosts be servers must not turn these two into servers silently. + hosts = plain(5) + for name in ("h4", "h5"): + hosts[name]["github_runner_cluster_node_mode"] = "native" + result = topology(hosts, max_servers=5) + self.assertEqual(result["errors"], []) + self.assertEqual(result["servers"], ["h1", "h2", "h3"]) + self.assertEqual([result["hosts"][name]["role"] for name in ("h4", "h5")], ["agent", "agent"]) + def test_a_native_host_joins_the_bootstrap_server_like_any_agent(self) -> None: hosts = plain(4) hosts["h4"]["github_runner_cluster_node_mode"] = "native" @@ -200,14 +210,58 @@ def test_the_compose_mode_changes_nothing(self) -> None: host["github_runner_cluster_node_mode"] = "compose" self.assertEqual(topology(explicit), topology(plain(5))) - def test_a_native_host_cannot_be_a_server_or_bootstrap(self) -> None: - for override in ({"github_runner_cluster_node_role": "server"}, {"github_runner_cluster_bootstrap": True}): - with self.subTest(override=override): - hosts = plain(3) - hosts["h1"]["github_runner_cluster_node_mode"] = "native" - hosts["h1"].update(override) - errors = topology(hosts)["errors"] - self.assertTrue(any("h1 sets github_runner_cluster_node_mode: native" in error for error in errors), errors) + def test_a_native_host_that_sets_the_server_role_is_a_server_and_joins_as_one(self) -> None: + hosts = plain(3) + hosts["h3"].update({"github_runner_cluster_node_mode": "native", "github_runner_cluster_node_role": "server"}) + result = topology(hosts) + self.assertEqual(result["errors"], []) + self.assertEqual(result["servers"], ["h1", "h2", "h3"]) + self.assertEqual(result["bootstrap"], "h1") + self.assertEqual(result["hosts"]["h3"]["server_url"], "https://h1:6443") + self.assertTrue(result["hosts"]["h3"]["native"]) + + def test_a_native_host_that_sets_the_bootstrap_bootstraps_the_cluster(self) -> None: + hosts = plain(3) + hosts["h3"].update({"github_runner_cluster_node_mode": "native", "github_runner_cluster_bootstrap": True}) + result = topology(hosts) + self.assertEqual(result["errors"], []) + self.assertEqual(result["bootstrap"], "h3") + self.assertEqual(result["hosts"]["h3"]["role"], "server") + self.assertEqual(result["hosts"]["h3"]["server_url"], "") + self.assertEqual(result["hosts"]["h1"]["server_url"], "https://h3:6443") + + def test_an_explicit_native_server_takes_a_server_place_like_any_server_override(self) -> None: + hosts = plain(4) + hosts["h4"].update({"github_runner_cluster_node_mode": "native", "github_runner_cluster_node_role": "server"}) + result = topology(hosts) + self.assertEqual(result["errors"], []) + self.assertEqual(result["servers"], ["h1", "h2", "h4"]) + self.assertEqual(result["hosts"]["h3"]["role"], "agent") + + def test_an_eligible_host_is_named_in_every_servers_certificate_and_keeps_its_own_role(self) -> None: + hosts = plain(4) + hosts["h4"].update({"github_runner_cluster_node_mode": "native", "github_runner_cluster_voter_eligible": True}) + result = topology(hosts) + self.assertEqual(result["errors"], []) + self.assertEqual(result["hosts"]["h4"]["role"], "agent") + self.assertTrue(result["hosts"]["h4"]["eligible"]) + self.assertFalse(result["hosts"]["h1"]["eligible"]) + for name in ("h1", "h2", "h3", "h4"): + self.assertEqual(result["hosts"][name]["tls_sans"], ["h1", "h2", "h3", "h4"], name) + + def test_each_host_knows_every_other_servers_join_url_in_inventory_order(self) -> None: + result = topology(plain(4)) + self.assertEqual(result["hosts"]["h1"]["peer_server_urls"], ["https://h2:6443", "https://h3:6443"]) + self.assertEqual(result["hosts"]["h2"]["peer_server_urls"], ["https://h1:6443", "https://h3:6443"]) + self.assertEqual(result["hosts"]["h4"]["peer_server_urls"], ["https://h1:6443", "https://h2:6443", "https://h3:6443"]) + + def test_an_explicit_native_server_still_has_to_leave_an_odd_count(self) -> None: + hosts = plain(3) + hosts["h3"].update({"github_runner_cluster_node_mode": "native", "github_runner_cluster_node_role": "server"}) + hosts["h1"]["github_runner_cluster_node_role"] = "server" + hosts["h2"]["github_runner_cluster_node_role"] = "agent" + errors = topology(hosts)["errors"] + self.assertTrue(any("etcd needs an odd number" in error for error in errors), errors) def test_a_mode_that_is_neither_compose_nor_native_is_refused(self) -> None: hosts = plain(1) From 0399176c020976de2ead2d92f5113fbccb0159f7 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 3 Nov 2026 08:41:21 +0000 Subject: [PATCH 02/10] feat(cluster): hold each node to the version rules of its live role The version check took this node's role from the inventory. A native host can now switch between its server and agent units at run time, so where the cluster lists this node, its control-plane label decides which rules apply to it, as it already did for every other node. --- plugins/filter/cluster_versions.py | 15 ++++++++++----- tests/unit/test_cluster_versions.py | 10 +++++++++- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/plugins/filter/cluster_versions.py b/plugins/filter/cluster_versions.py index 77152e36..35857c4a 100644 --- a/plugins/filter/cluster_versions.py +++ b/plugins/filter/cluster_versions.py @@ -108,7 +108,7 @@ def cluster_k3s_version_change_blockers(k3s_version: Any, node_name: Any, own_re :param node_name: this host's node name: its container, and its own entry in the node list, are this node. :param own_report: what ``k3s --version`` printed in this node's running container (for example "k3s version v1.34.1+k3s1 (24fc436e)"), or an empty string when it does not run. :param server_reads: the loop results of reading the cluster through each server of the inventory (tasks/running_version.yml), each with its host as ``item`` and ``stdout`` either "stopped" or "running" followed by " " lines (the label "true" on a server, empty on an agent); an empty list when the inventory holds no server. - :param node_role: this host's role, "server" or "agent". + :param node_role: this host's role as the inventory gives it, "server" or "agent". Where the cluster lists this node, its live control-plane label decides instead, since a host's role can change at run time (a native host switched between its server and agent units). :param builds: whether this run builds the node's image (it is missing), which is when an agent's kubelet would take the pinned version. :returns: one sentence per rule broken. A version that cannot be read is a problem too, since the check would otherwise pass on what it never saw, and so is a cluster that some server runs but no server could be read through. """ @@ -126,9 +126,7 @@ def cluster_k3s_version_change_blockers(k3s_version: Any, node_name: Any, own_re if unread and not answered: problems.append(f"The cluster could not be read through any server of the inventory, so what it runs cannot be told: {'; '.join(unread)}.") # (source, version, is this node, is a server) - running: list[tuple[str, str, bool, bool]] = [] - if own_report.strip(): - running.append((f"This node's own container ({node_name})", own_report.strip(), True, node_role == "server")) + listed: list[tuple[str, str, bool, bool]] = [] seen: set[str] = set() for lines in answered: for line in lines: @@ -136,7 +134,14 @@ def cluster_k3s_version_change_blockers(k3s_version: Any, node_name: Any, own_re if not fields or fields[0] in seen: continue seen.add(fields[0]) - running.append((f"Node {fields[0]}", fields[1] if len(fields) > 1 else "", fields[0] == str(node_name), len(fields) > 2 and fields[2] == "true")) + listed.append((f"Node {fields[0]}", fields[1] if len(fields) > 1 else "", fields[0] == str(node_name), len(fields) > 2 and fields[2] == "true")) + live_self = [is_server for _, _, is_self, is_server in listed if is_self] + if live_self: + node_role = "server" if live_self[0] else "agent" + running: list[tuple[str, str, bool, bool]] = [] + if own_report.strip(): + running.append((f"This node's own container ({node_name})", own_report.strip(), True, node_role == "server")) + running.extend(listed) target = _release(pinned) for source, version, is_self, is_server in running: found = RUNNING_VERSION.search(version) diff --git a/tests/unit/test_cluster_versions.py b/tests/unit/test_cluster_versions.py index e74655e8..5db04e50 100644 --- a/tests/unit/test_cluster_versions.py +++ b/tests/unit/test_cluster_versions.py @@ -194,6 +194,14 @@ def test_a_server_may_not_be_built_below_a_node_s_minor(self) -> None: reads = [answered("a", "server1 v1.34.1+k3s1 true\nagent1 v1.35.0+k3s1")] self.assertEqual(versions.cluster_k3s_version_change_blockers("v1.34.1-k3s1", "server1", "", reads, "server"), ["Agent agent1 runs v1.35.0+k3s1, a newer minor than v1.34.1-k3s1; a kubelet may not be newer than the API servers."]) + def test_this_node_s_live_role_decides_the_rules_over_the_inventory_s(self) -> None: + # A native host the inventory lists as an agent that the cluster now runs as a server (its units were switched at run time) is held to the servers' rule, and the other way round. + reads = [answered("a", "server1 v1.34.1+k3s1 true\nnative1 v1.34.1+k3s1 true\nagent1 v1.35.0+k3s1")] + self.assertEqual(versions.cluster_k3s_version_change_blockers("v1.34.1-k3s1", "native1", "", reads, "agent"), ["Agent agent1 runs v1.35.0+k3s1, a newer minor than v1.34.1-k3s1; a kubelet may not be newer than the API servers."]) + demoted = [answered("a", "server1 v1.34.1+k3s1 true\nnative1 v1.34.1+k3s1\nagent1 v1.35.0+k3s1")] + self.assertEqual(versions.cluster_k3s_version_change_blockers("v1.34.1-k3s1", "native1", "", demoted, "server"), []) + self.assertEqual(versions.cluster_k3s_version_change_blockers("v1.35.0-k3s1", "native1", "", demoted, "server"), ["Server server1 runs v1.34.1+k3s1, an older minor than v1.35.0-k3s1; an agent's kubelet may not be newer than the API servers, so upgrade the servers first."]) + def test_this_node_s_own_entry_in_the_node_list_counts_as_this_node(self) -> None: reads = [answered("a", "server1 v1.35.0+k3s1 true\nserver2 v1.35.0+k3s1 true")] self.assertEqual(versions.cluster_k3s_version_change_blockers("v1.34.9-k3s1", "server1", "", reads, "server")[0], "Node server1 runs v1.35.0+k3s1, newer than v1.34.9-k3s1; k3s cannot be downgraded.") @@ -365,7 +373,7 @@ def test_the_host_on_another_minor_fails_when_run_on_its_own(self) -> None: def running(self, *extra: str, pin: str | None = None, run_on: str = "node2", roles: dict[str, str] | None = None, bootstrap: str | None = "node1", unreachable: str | None = None, timeout: int | None = None) -> subprocess.CompletedProcess[str]: """Run tasks/running_version.yml on one host of a three-host group (servers unless ``roles`` says otherwise) whose topology names ``bootstrap`` as its bootstrap host, or none; ``unreachable`` names a host reached over SSH on a port nothing listens on.""" roles = {"node1": "server", "node2": "server", "node3": "server", **(roles or {})} - topology = {"hosts": {name: {"role": role, "bootstrap": name == bootstrap, "node_name": name} for name, role in roles.items()}} + topology = {"hosts": {name: {"role": role, "bootstrap": name == bootstrap, "node_name": name, "native": False, "eligible": False} for name, role in roles.items()}} common: dict[str, Any] = {"github_runner_cluster_topology": topology, **({"github_runner_cluster_k3s_version": pin} if pin else {})} hosts = { name: { From d0f976ed20d017630f6debd5bb1c3650a63acfec Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 3 Nov 2026 08:42:04 +0000 Subject: [PATCH 03/10] feat(cluster): run a native host as a k3s server and etcd voter A native host now has two units, k3s-agent and k3s, each with its own configuration file passed as --config, and a role file under k3s's data directory that both units' ExecCondition reads, so the two never run together and Restart=always never starts the unit of a role the host no longer has. The server's configuration carries the flags every Docker server passes (the critical ones k3s requires to match), and etcd's peer URL is the host's Tailscale address, written as node-ip into a drop-in before each start. cluster-init is written only for the bootstrap host with no datastore, none moved aside, and no other server answering, so a former member always joins the cluster that exists. A server's restart, first start beside a cluster and promotion go ahead only while every other etcd voter, read from the cluster's labels, is Ready, one native server at a time. A switch between the units keeps the node's name, password and device: it empties the node first, and a demotion leaves at least three Ready voters, removes the member through k3s, and moves the datastore aside. The version checks read native servers through k3s kubectl, and regrading is refused on a native host. --- examples/native-server/inventory.yml | 31 + plugins/filter/cluster_native.py | 87 ++- roles/github_runner_cluster/README.md | 46 +- roles/github_runner_cluster/defaults/main.yml | 29 +- .../files/native/fleet-role-is.sh | 15 + .../files/native/write-node-ip.sh | 53 ++ .../meta/argument_specs.yml | 24 +- .../tasks/cluster_nodes_read.yml | 59 +- roles/github_runner_cluster/tasks/main.yml | 2 +- .../tasks/native/api.yml | 25 + .../tasks/native/config.yml | 119 +++- .../tasks/native/demote.yml | 154 +++++ .../tasks/native/drain.yml | 57 ++ .../tasks/native/end_checks.yml | 72 ++- .../tasks/native/main.yml | 17 +- .../tasks/native/role.yml | 102 +++ .../tasks/native/server_ready.yml | 30 + .../tasks/native/server_turn.yml | 131 ++++ .../tasks/native/service.yml | 126 ++-- .../native/{config.yaml.j2 => agent.yaml.j2} | 4 +- .../templates/native/join.yaml.j2 | 10 + .../templates/native/k3s-agent.service.j2 | 4 +- .../templates/native/k3s.service.j2 | 42 ++ .../templates/native/server.yaml.j2 | 31 + tests/unit/test_native_node.py | 582 ++++++++++++++++-- 25 files changed, 1665 insertions(+), 187 deletions(-) create mode 100644 examples/native-server/inventory.yml create mode 100755 roles/github_runner_cluster/files/native/fleet-role-is.sh create mode 100755 roles/github_runner_cluster/files/native/write-node-ip.sh create mode 100644 roles/github_runner_cluster/tasks/native/api.yml create mode 100644 roles/github_runner_cluster/tasks/native/demote.yml create mode 100644 roles/github_runner_cluster/tasks/native/drain.yml create mode 100644 roles/github_runner_cluster/tasks/native/role.yml create mode 100644 roles/github_runner_cluster/tasks/native/server_ready.yml create mode 100644 roles/github_runner_cluster/tasks/native/server_turn.yml rename roles/github_runner_cluster/templates/native/{config.yaml.j2 => agent.yaml.j2} (75%) create mode 100644 roles/github_runner_cluster/templates/native/join.yaml.j2 create mode 100644 roles/github_runner_cluster/templates/native/k3s.service.j2 create mode 100644 roles/github_runner_cluster/templates/native/server.yaml.j2 diff --git a/examples/native-server/inventory.yml b/examples/native-server/inventory.yml new file mode 100644 index 00000000..5ddb0a86 --- /dev/null +++ b/examples/native-server/inventory.yml @@ -0,0 +1,31 @@ +--- +# A cluster with a native server: node1 and node2 are Docker servers, and linux1 runs k3s directly under systemd as the third server and etcd voter, with its own Tailscale client as the node's device and its etcd peer URL on its Tailscale address. linux2 is a native agent staged as a standby (github_runner_cluster_voter_eligible): its server configuration is written and its name is on every server's certificate, but it stays an agent until its role is switched. See the cluster role's README, Native servers. Run with: ansible-playbook -i examples/native-server/inventory.yml exadev.github_runner.site --limit linux1,linux2 --check # then without --check +all: + children: + github_runner_cluster: + vars: + github_runner_cluster_tailnet_domain: example-tailnet.ts.net + github_runner_secrets_source: onepassword + github_runner_secrets_onepassword_vault: Example + github_runner_secrets_onepassword_item: github-runner + hosts: + node1: + ansible_host: node1.example.com + node2: + ansible_host: node2.example.com + linux1: + # Reached by its LAN name, not over the tailnet, as for a native agent. + ansible_host: linux1.local + github_runner_cluster_node_mode: native + # A native host is a server only when it says so; it never takes an automatic server place. + github_runner_cluster_node_role: server + github_runner_cluster_native_tailscale_leave_other_tailnet: true + github_runner_linux_host_k3s_prerequisites: true + github_runner_linux_host_keep_awake: true + linux2: + ansible_host: linux2.local + github_runner_cluster_node_mode: native + github_runner_cluster_voter_eligible: true + github_runner_cluster_native_tailscale_leave_other_tailnet: true + github_runner_linux_host_k3s_prerequisites: true + github_runner_linux_host_keep_awake: true diff --git a/plugins/filter/cluster_native.py b/plugins/filter/cluster_native.py index 771d64ef..3cf8177a 100644 --- a/plugins/filter/cluster_native.py +++ b/plugins/filter/cluster_native.py @@ -1,4 +1,4 @@ -"""Filters the github_runner_cluster role's native node uses: which k3s release asset a host's architecture takes, the release and package versions the node's pins name, and which tailnet the host's Tailscale client is logged in to.""" +"""Filters the github_runner_cluster role's native node uses: which k3s release asset a host's architecture takes, the release and package versions the node's pins name, which tailnet the host's Tailscale client is logged in to, which role the host runs and whether it bootstraps a cluster, and whether the cluster's other voters are Ready enough for a server to start, restart or join.""" from __future__ import annotations @@ -82,6 +82,88 @@ def cluster_native_tailnet_login(status: Any, tailnet_domain: Any) -> dict[str, return {"state": state, "backend": backend, "domain": domain} +NODE_ROLES = ("server", "agent") + + +def _reason(result: dict[str, Any]) -> str: + """The last line of what a failed command or task said, or its exit status when it said nothing.""" + lines = [line.strip() for line in str(result.get("stderr") or result.get("msg") or "").splitlines() if line.strip()] + return lines[-1] if lines else f"exit status {result.get('rc')}" + + +def cluster_native_role(recorded: Any, agent_installed: Any, eligible: Any, inventory_role: Any, running: Any = ()) -> dict[str, str]: + """Decide the role a native host runs now and the role it is to run after this run. + + :param recorded: what the host's role file holds (empty when it has none). :param agent_installed: whether the host has the k3s-agent unit, which a host set up before the role file existed has. :param eligible: whether the host is marked github_runner_cluster_voter_eligible. :param inventory_role: the role the topology gives the host. :param running: the roles whose unit is active now. :returns: ``current`` (``server``, ``agent``, or empty on a first install) and ``desired``. :raises AnsibleFilterError: if the role file holds anything but a role, or the inventory role or a running role is not one. + + The role file is the host's own record of its role and outlives restarts, reboots and runs of the role. A switch writes the new role to it before it stops the old unit, so while exactly one unit runs, that unit's role is the one the host runs now, and a switch that stopped part-way (a member removal that did not finish) is taken up again by the next run. With neither unit running, the file says; a host from before the file existed ran only the agent, so its unit counts as the agent role. For a host marked eligible the inventory gives only the role it starts with: once the file names a role, that role is kept, since whatever moves votes between hosts changes it at run time and a run of the role must not undo that. For any other host the inventory decides. + """ + text = "" if recorded is None else str(recorded).strip() + if text and text not in NODE_ROLES: + raise AnsibleFilterError(f"The role file holds {text!r}; it must be 'server' or 'agent'. Write the role the host runs into it, or remove it on a host that runs neither unit.") + if inventory_role not in NODE_ROLES: + raise AnsibleFilterError(f"{inventory_role!r} is not a node role (server or agent).") + active = [str(role) for role in running] + if any(role not in NODE_ROLES for role in active): + raise AnsibleFilterError(f"{active!r} are not node roles (server or agent).") + if len(active) == 1: + current = active[0] + else: + current = text or ("agent" if bool(agent_installed) else "") + desired = text if text and bool(eligible) else str(inventory_role) + return {"current": current, "desired": desired} + + +def cluster_native_cluster_init(reads: Any, this_host: Any) -> dict[str, Any]: + """Say whether a native bootstrap host with no datastore of its own may start a new cluster: only when no other server of the inventory runs one. + + :param reads: the loop results of reading the cluster through each server of the inventory (tasks/cluster_nodes_read.yml's github_runner_cluster_running_reads), each with its host as ``item`` and ``stdout`` "stopped" or "running" followed by node lines. :param this_host: this host's inventory name, whose own read says nothing about another cluster. :returns: ``cluster_init`` (true only when every other server read as stopped, or there is none), ``running`` (the hosts that answered as running a server) and ``unread`` (one line per server that could not be read, which leaves the question open, so the caller refuses). + """ + if not isinstance(reads, list): + raise AnsibleFilterError("The reads must be a list of loop results.") + running: list[str] = [] + unread: list[str] = [] + for result in reads: + if not isinstance(result, dict): + raise AnsibleFilterError(f"{result!r} is not a loop result.") + host = str(result.get("item", "?")) + if host == str(this_host): + continue + lines = str(result.get("stdout", "")).split() + if result.get("unreachable") or result.get("rc") != 0: + unread.append(f"{host} could not be read ({_reason(result)})") + elif lines[:1] == ["running"]: + running.append(host) + elif lines[:1] != ["stopped"]: + unread.append(f"{host} gave an answer this check cannot read") + return {"cluster_init": not running and not unread, "running": running, "unread": unread} + + +def cluster_native_voter_blockers(read: Any, node_name: Any, min_others: Any = 0) -> list[str]: + """Say why a native server must not start, restart or join now: every other etcd voter the cluster lists must be Ready, so the cluster keeps its quorum while this one is away, and a learner is only ever added beside a healthy membership. + + The voters are read from the cluster, each node's etcd role label, never from the inventory, because a host's role can change at run time. A cluster that lists no other voter (a first bootstrap, or a single server restarting) has no quorum to keep. + + :param read: the result of ``kubectl get nodes`` printing ``,,`` lines (commas, since either of the last two may be empty), as Ansible registers it. :param node_name: this host's node name, which is left out. :param min_others: the fewest other voters there must be, for a change that takes this server's vote away (a demotion); 0 for one that keeps it. :returns: one sentence per other voter that is not Ready, one more when there are fewer other voters than ``min_others``, or one sentence saying the nodes could not be listed, or an empty list. + """ + if not isinstance(read, dict): + raise AnsibleFilterError(f"{read!r} is not a task result.") + if read.get("unreachable") or read.get("rc") != 0: + return [f"the cluster's nodes could not be listed ({_reason(read)}), so whether the other voters are Ready cannot be told"] + problems: list[str] = [] + others: list[str] = [] + for line in str(read.get("stdout", "")).splitlines(): + fields = [field.strip() for field in line.split(",")] + if len(fields) != 3 or fields[0] == str(node_name) or fields[2] != "true": + continue + others.append(fields[0]) + if fields[1] != "True": + problems.append(f"voter {fields[0]} is not Ready (its Ready condition is {fields[1] or 'unknown'})") + if len(others) < int(min_others): + problems.append(f"only {len(others)} other voters ({', '.join(others) or 'none'}) would remain, fewer than the {int(min_others)} a change that takes this server's vote away leaves") + return problems + + class FilterModule: """Registers the native node's filters with Ansible.""" @@ -92,4 +174,7 @@ def filters(self) -> dict[str, Any]: "cluster_native_k3s_release": cluster_native_k3s_release, "cluster_native_tailscale_package_version": cluster_native_tailscale_package_version, "cluster_native_tailnet_login": cluster_native_tailnet_login, + "cluster_native_role": cluster_native_role, + "cluster_native_cluster_init": cluster_native_cluster_init, + "cluster_native_voter_blockers": cluster_native_voter_blockers, } diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index 7102319d..aa10b112 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -4,7 +4,7 @@ Runs a k3s cluster under Docker Compose on each host, from the `docker-compose.y Which hosts are servers is decided automatically from the inventory group named by `github_runner_cluster_group`, in inventory order. The first N hosts are servers, where N is the largest odd number no greater than both the number of hosts and `github_runner_cluster_max_servers`: 1 or 2 hosts give 1 server, 3 or 4 give 3, and 5 or more give 5. The first server bootstraps the cluster, the other servers join it, and the remaining hosts join it as agents. The group is used rather than the play's hosts so that `--limit` never changes the result. The logic lives in `tasks/topology.yml` and the collection's `exadev.github_runner.cluster_topology` filter (`plugins/filter/cluster_topology.py`), and publishes `github_runner_cluster_effective_node_role`, `github_runner_cluster_effective_bootstrap`, `github_runner_cluster_effective_server_url` and `github_runner_cluster_effective_tls_sans` for each host. -A host that sets `github_runner_cluster_node_mode: native` does not use Docker at all: it runs the k3s agent directly under systemd, as an agent only, with the host's own Tailscale client as the node's device on the mesh. See [Native nodes](#native-nodes), which also says what that gives up. Everything below that mentions a container, Compose or a Docker context is about the default mode, `compose`. +A host that sets `github_runner_cluster_node_mode: native` does not use Docker at all: it runs k3s directly under systemd, as an agent or, when its inventory says so, a server and etcd voter, with the host's own Tailscale client as the node's device on the mesh. See [Native nodes](#native-nodes), which also says what that gives up. Everything below that mentions a container, Compose or a Docker context is about the default mode, `compose`. ## Docker Compose versions @@ -76,20 +76,36 @@ The manual Secret deletion is for a node that is genuinely new under an old name ## Native nodes -`github_runner_cluster_node_mode: native` in a host's `host_vars` runs that host's k3s node directly on the host, as the systemd service `k3s-agent`, with no Docker or Compose layer. The default, `compose`, is everything else in this README. A node under Compose nests containerd inside a container, so every image layer is overlayfs on overlayfs and pod egress goes through the container's NAT as well; a native node's containerd uses the host's filesystem and network. It is for a Linux host that systemd runs, and only as an agent: `tasks/topology.yml` never gives it a server place, does not count it towards the automatic server number, and refuses a native host that sets `github_runner_cluster_node_role: server` or `github_runner_cluster_bootstrap: true`. It also refuses a native host that sets `github_runner_cluster_network_mode: host`, since that setting puts a Docker node's container on the Docker host's network and a native node has no container (it already runs on the host's own network, with the host's own Tailscale client). Add the host to the cluster group like any other (its place in the group does not matter, since it can only be an agent), set `github_runner_cluster_node_mode: native` and `github_runner_cluster_tailnet_domain` for it, and run the playbook with `--limit` to it. Its tasks that change the host use `become` on their own, so the connecting user needs passwordless sudo or `--ask-become-pass`. `examples/native-agent/inventory.yml` is a starting point. Preparing the host itself (the kernel modules and sysctls k3s needs, not suspending, a quiet Wi-Fi radio, a headless boot, security updates that never reboot it) is the `github_runner_linux_host` role's, which `playbooks/site.yml` runs first; k3s loads the modules and sets most of the sysctls itself when it starts, so the node does not depend on it. +`github_runner_cluster_node_mode: native` in a host's `host_vars` runs that host's k3s node directly on the host, as the systemd service `k3s-agent` (an agent) or `k3s` (a server), with no Docker or Compose layer. The default, `compose`, is everything else in this README. A node under Compose nests containerd inside a container, so every image layer is overlayfs on overlayfs and pod egress goes through the container's NAT as well; a native node's containerd uses the host's filesystem and network. It is for a Linux host that systemd runs. A native host is a server only when it asks to be: `github_runner_cluster_node_role: server`, or `github_runner_cluster_bootstrap: true` to bootstrap the cluster. `tasks/topology.yml` never gives it a server place automatically and does not count it towards the automatic server number, so an inventory whose native hosts set neither keeps all of them agents whatever `github_runner_cluster_max_servers` allows; the server count with them must still be odd. See [Native servers](#native-servers). It also refuses a native host that sets `github_runner_cluster_network_mode: host`, since that setting puts a Docker node's container on the Docker host's network and a native node has no container (it already runs on the host's own network, with the host's own Tailscale client). Add the host to the cluster group like any other (its place in the group does not matter, since it never takes an automatic server place), set `github_runner_cluster_node_mode: native` and `github_runner_cluster_tailnet_domain` for it, and run the playbook with `--limit` to it. Its tasks that change the host use `become` on their own, so the connecting user needs passwordless sudo or `--ask-become-pass`. `examples/native-agent/inventory.yml` is a starting point, and `examples/native-server/inventory.yml` for a native server and a standby. Preparing the host itself (the kernel modules and sysctls k3s needs, not suspending, a quiet Wi-Fi radio, a headless boot, security updates that never reboot it) is the `github_runner_linux_host` role's, which `playbooks/site.yml` runs first; k3s loads the modules and sets most of the sysctls itself when it starts, so the node does not depend on it. The role reads the same secrets and settings as the Docker path and joins the same cluster: `github_runner_cluster_k3s_token`, the mesh's join key and control server (`tasks/mesh/main.yml`, whichever provider is in use), the node name rules (`github_runner_cluster_node_name`, else the inventory name), the API port (`github_runner_cluster_api_port`, checked to be one port for the whole group by the same `tasks/api_port.yml`), the join URL `tasks/topology.yml` works out, `github_runner_cluster_node_taints` (written as `node-taint` in k3s's configuration, so they are applied each time the node registers, as they are under Compose), and the pins `github_runner_cluster_k3s_version` and `github_runner_cluster_tailscale_version`. Node labels are not set by this role under either mode: the ARC role labels nodes by name from the API (`github_runner_arc_labelled_nodes`), so a native node gets its labels the same way. A run does this, in order, and every step is idempotent and honours `--check`: 1. Checks the host (Linux, systemd, an architecture k3s publishes a binary for, Debian or Ubuntu if the role installs Tailscale), the topology, the pins, the API port, the token and the mesh, before anything is changed. -2. Compares the k3s version it is to install with the installed binary and with every node as the servers report it, with the same upgrade path and version skew rules as the Docker path (see [Versions](#versions)), reading the cluster through the servers' containers. `github_runner_cluster_allow_k3s_version_jump` skips it. -3. Installs the pinned Tailscale package, adding Tailscale's apt repository when the host has none, and keeps its service running; then checks which tailnet the client is logged in to (below). +2. Compares the k3s version it is to install with the installed binary and with every node as the servers report it, with the same upgrade path and version skew rules as the Docker path (see [Versions](#versions)), reading the cluster through every server of the inventory and every host marked eligible (a Docker server's container, a native server's `k3s kubectl`). Each node's role in those rules is its live `control-plane` label, this node's included, so a host whose role changed at run time is held to the rules of the role it runs. `github_runner_cluster_allow_k3s_version_jump` skips it. +3. Decides the role the host runs (below), and for a server whether it bootstraps a new cluster or joins one. Then installs the pinned Tailscale package, adding Tailscale's apt repository when the host has none, and keeps its service running; then checks which tailnet the client is logged in to (below). 4. Downloads the k3s release binary `github_runner_cluster_k3s_version` names (the image tag's `-k3s` written `+k3s`, as GitHub names the release) to `/usr/local/bin/k3s`, checked against the SHA-256 the same release lists for the host's architecture or `github_runner_cluster_native_k3s_sha256`. The file is replaced in one rename, so a running k3s keeps its old binary until it restarts. -5. Writes `/etc/rancher/k3s/config.yaml` (server URL, node name, taints, the kubelet's resolver file), the node's token and `--vpn-auth` value as two files of their own (mode 0600, written by tasks that log nothing, so the configuration names the files and holds no secret), the two helper scripts under `/usr/local/lib/github-runner-k3s/`, and the unit `/etc/systemd/system/k3s-agent.service`. -6. Enables and starts the service, or restarts a running one when its binary, configuration, secrets or unit changed (pods keep running, because the unit's `KillMode=process` leaves their containerd shims alone), and then checks the node: the service is active, the host's Tailscale client is on the cluster's tailnet, and the bootstrap server sees the node Ready on a heartbeat later than the start (the same rule as a recreated server's, `tasks/server_ready.yml`). A cluster whose servers are outside the inventory has no server to ask, and the run says so. +5. Writes `/etc/rancher/k3s/agent.yaml` (server URL, node name, taints, the kubelet's resolver file), on a server or a host marked eligible `/etc/rancher/k3s/server.yaml` and its drop-in `server.yaml.d/10-join.yaml` (below), the node's token and `--vpn-auth` value as two files of their own (mode 0600, written by tasks that log nothing, so the configurations name the files and hold no secret), the helper scripts under `/usr/local/lib/github-runner-k3s/`, both units `/etc/systemd/system/k3s-agent.service` and `k3s.service`, and the role file `/var/lib/rancher/k3s/fleet-role` when the host has none. Earlier releases wrote the agent's flags to `/etc/rancher/k3s/config.yaml`, which is removed, since each unit passes its own file as `--config`. +6. Runs the unit of the host's role: enables and starts it, restarts a running one when its binary, configuration, secrets or unit changed (pods keep running, because the units' `KillMode=process` leaves their containerd shims alone), or switches the host between its units when its role changed. A restart that is refused or fails is not lost: a marker beside the role file (`fleet-restart-pending-`) says the unit's files changed since it last started, and the next run restarts it. Then it checks the node: the unit of the role is active and the other is not, the host's Tailscale client is on the cluster's tailnet, and a server sees the node Ready on a heartbeat later than the start (the same rule as a recreated server's, `tasks/server_ready.yml`), and for a server also a voting member of etcd. A cluster whose servers are outside the inventory has no server to ask, and the run says so. -`github_runner_cluster_native_start: false` stops after step 5 and leaves the service as it finds it (a first install stays disabled and stopped), which stages a host before it joins. +`github_runner_cluster_native_start: false` stops after step 5 and leaves the units as it finds them (a first install stays disabled and stopped, and a host whose role is to change keeps running its old unit), which stages a host before it joins or before its role is switched. + +### Native servers + +A native server is configured as every Docker server is, because k3s refuses a server whose critical configuration differs from the cluster's (`critical configuration value mismatch between servers`, `pkg/cluster/bootstrap.go`; the fields are `CriticalControlArgs` in `pkg/daemons/config/types.go` at [v1.34.1+k3s1](https://github.com/k3s-io/k3s/blob/v1.34.1%2Bk3s1/pkg/daemons/config/types.go)): `server.yaml` disables traefik and the network policy controller, leaves the cluster and service networks at k3s's own, names every server and every eligible host as a TLS SAN, passes `github_runner_cluster_etcd_heartbeat_interval` and `github_runner_cluster_etcd_election_timeout` as `etcd-arg` (checked by the same `tasks/etcd_timing.yml`, so the servers agree), and `https-listen-port` when `github_runner_cluster_api_port` is not k3s's own. The admin kubeconfig is root's alone (`write-kubeconfig-mode: "0600"`), and the role reads the cluster through it with `k3s kubectl` under `become`. + +**etcd's peer address is the host's Tailscale address.** k3s builds etcd's peer URL from its private IP, which it takes from `--node-ip` before the `--vpn-auth` integration sets only the advertise address (`pkg/cli/server/server.go`: "Etcd IP (PrivateIP) remains the local IP"), so without it a native server's etcd would advertise a LAN address and lose its peers whenever the host changes network. The server unit runs `files/native/write-node-ip.sh` before every start, which writes `node-ip` into `server.yaml.d/00-node-ip.yaml` from the client's own address (logging the client in first with exactly the `tailscale up` k3s runs, from the same `--vpn-auth` file, when it has none yet). k3s reads `server.yaml`, then every `*.yaml` in `server.yaml.d` in name order (`dotDFiles` in `pkg/configfilearg/parser.go`). + +**Two units, one role.** Both units are written on every native host, and each starts only while the role file `/var/lib/rancher/k3s/fleet-role` names its role: each unit's `ExecCondition` runs `files/native/fleet-role-is.sh`, and systemd skips a start whose condition exits 1 to 254 without marking the unit failed, and `Restart=always` never restarts a unit whose start was skipped that way (`service_shall_restart` in systemd's `src/core/service.c`), so the two never run together and a demoted server is never started again by its own restart policy. A host set up before the role file existed runs the agent unit and is recorded as an agent. For a host marked `github_runner_cluster_voter_eligible` the inventory gives only the role it starts with; once its role file names a role, a run keeps it, since the controller that moves votes changes it at run time. For any other native host the inventory decides, and a run whose inventory role differs from the role file switches the host. + +**Bootstrapping, joining and rejoining.** `server.yaml.d/10-join.yaml` holds `cluster-init: true` only when the host is the inventory's bootstrap host, has no datastore (`server/db`) and none a demotion moved aside (`server/db.demoted`), and no other server of the inventory (Docker or native, or eligible) answers as running one; a server that cannot be read refuses the run, since starting a second cluster beside the real one would split the fleet. Otherwise it holds `server:`, the bootstrap host's join URL, or for the bootstrap host itself the first other server that answered, so a host that was ever a member, or a bootstrap host that comes back after the cluster moved on, always joins the cluster that exists. k3s joins a new server as an etcd learner and the leader promotes it once it has caught up (`MemberAddAsLearner` in `pkg/etcd/etcd.go`), so a learner never costs quorum, and the turn ends only once the node's `EtcdIsVoter` condition is True. A native bootstrap host is refused under `headscale_in_cluster`, whose bootstrap server runs Headscale as a static pod under a supervisor only the Docker node has (`files/k3s/node-entrypoint.sh`). + +**The restart guard.** Native servers take turns, one at a time, as recreated Docker servers do ([Recreating servers one at a time](#recreating-servers-one-at-a-time)). A restart, a first start beside an existing cluster, or a promotion goes ahead only while every other etcd voter is Ready, the voters being read from each node's `node-role.kubernetes.io/etcd` label as the cluster reports it now, never from the inventory. Under `--check` the guard still reads and refuses, and the change is reported, not made. + +**Switching role.** The node keeps its name, its node password (`/etc/rancher/node/password`, which neither unit is told to change) and its Tailscale device. Both directions cordon the node, wait up to `github_runner_cluster_native_runner_wait_timeout` seconds for its ARC runner pods to finish, and drain it; a node that cannot be emptied is uncordoned and the run fails with the units as they were, so a planned switch never cuts a job. A promotion then writes `server` to the role file, stops and disables the agent unit and starts the server unit. A demotion is make-before-break: it goes ahead only while at least `github_runner_cluster_native_demotion_min_voters` (3) other voters, every one Ready, remain, so the cluster gains a voter before it loses one (etcd calls a two-member cluster unsafe). It writes `agent` to the role file first, so that neither `Restart=always` nor a reboot can start the server again (k3s marks a removed member's datastore with a tombstone and would rejoin it as a learner), then removes the etcd member through k3s's own controller (`etcd.k3s.cattle.io/remove=true` on the Node through another server, waiting for k3s to record `etcd.k3s.cattle.io/removed-node-name`; `pkg/etcd/member_controller.go`), stops and disables the server unit, moves `server/db` aside to `server/db.demoted` (replacing an older copy, and keeping the snapshots in it), because k3s reuses any datastore it finds and would start the old member again on a later promotion, takes the server role labels and k3s's etcd annotations off the Node, and starts the agent unit. Moving votes between hosts on its own is not this role's: it only makes a switch safe when the inventory or the role file asks for one. + +**Firewall.** The role opens nothing. Every peer reaches a native server over `tailscale0`; a host whose firewall limits the API (`github_runner_cluster_api_port`), etcd (2379 and 2380) and kubelet (10250) ports to `tailscale0` and loopback keeps them off the LAN. ### Tailscale @@ -111,11 +127,11 @@ The container nodes point the kubelet at a resolver file naming only Tailscale's ### What a native node gives up -- **The Compose project.** There is no `docker-compose.yml`, `.env`, build context, `docker-prune` service or Docker context for the node, no `github_runner_cluster_dir` and no recreate. systemd restarts the process (`Restart=always`), the role restarts it when something it reads changed, and the host's own logs are `journalctl -u k3s-agent`. +- **The Compose project.** There is no `docker-compose.yml`, `.env`, build context, `docker-prune` service or Docker context for the node, no `github_runner_cluster_dir` and no recreate. systemd restarts the process (`Restart=always`), the role restarts it when something it reads changed, and the host's own logs are `journalctl -u k3s-agent` or `journalctl -u k3s`. - **The image pin.** The k3s binary is pinned in its place. A Docker node ignores a changed pin until its image is rebuilt by hand; a native node's binary is replaced by the next run and the service restarted, after the version check has allowed it. Upgrade as for any agent: servers first, one minor at a time (see [Versions](#versions)), and cordon and drain the node first if its runner pods must not see the restart (they survive it, but the kubelet restarts). - **The node password volume.** k3s keeps the node's password in `/etc/rancher/node/password` on the host, which the role never reads, writes or deletes, so it survives restarts, upgrades and reruns without anything like the seeding in [Node passwords and recreating servers](#node-passwords-and-recreating-servers). It is lost only if that file is removed or the host is reinstalled, and then the node is "a genuinely new node under an old name" ([A replaced node](#a-replaced-node)): `github_runner_cluster_wait_for_nodes` on a server host clears the stale Secret and Node object, or delete them by hand. -- **Removing a node.** Stopping the service leaves the pods' containerd shims running. The role installs neither k3s's `k3s-killall.sh` nor its uninstall script (the installer writes them; see [Uninstalling](https://docs.k3s.io/installation/uninstall) ([archived](http://web.archive.org/web/2026/https://docs.k3s.io/installation/uninstall))). Drain the node, delete its Node object, then `systemctl disable --now k3s-agent` and stop the shims. -- **Servers.** A native node is an agent: no etcd member, and memory-pressure regrading does not apply to it. +- **Removing a node.** Stopping a unit leaves the pods' containerd shims running. The role installs neither k3s's `k3s-killall.sh` nor its uninstall script (the installer writes them; see [Uninstalling](https://docs.k3s.io/installation/uninstall) ([archived](http://web.archive.org/web/2026/https://docs.k3s.io/installation/uninstall))). Drain the node, delete its Node object, then `systemctl disable --now k3s-agent k3s` and stop the shims. A server's etcd member is removed first, as for a demotion. +- **Memory-pressure regrading.** Its actuator drives a Docker server's container, so the role refuses `github_runner_cluster_regrade` on a native host. - **Tailnet names in pods** (above), and **a Docker-free host's `docker` tooling**: nothing in this mode runs `docker` on the host, so it need not be installed. ### A Wi-Fi-only host @@ -124,7 +140,7 @@ A node on Wi-Fi drops off the network whenever the radio does, and it then flaps ### Tests -`tests/unit/test_native_node.py` covers it without a real host: the filters in `plugins/filter/cluster_native.py`, the two helper scripts, the defaults, argument specs and this README, and the role's tasks (`tests/native/run.yml`) against stand-ins for `systemctl`, `tailscale`, `dpkg-query`, `apt-get` and `docker` on `PATH` and a local server for the k3s release. It checks a first run, idempotence, `--check`, a changed setting restarting the service once, a client in another tailnet (refused, then logged out first when allowed), a client k3s did not log in, a refused downgrade, a wrong sum, an unsupported host, the secrets staying out of the output, and the node's readiness as a server reports it. +`tests/unit/test_native_node.py` covers it without a real host: the filters in `plugins/filter/cluster_native.py`, the helper scripts, the defaults, argument specs and this README, and the role's tasks (`tests/native/run.yml`) against stand-ins for `systemctl`, `tailscale`, `dpkg-query`, `apt-get`, `docker` and `kubectl` on `PATH`, a local server for the k3s release, and a server data directory laid out as k3s leaves it. For servers it checks a bootstrap with `cluster-init` and the Docker servers' flags, a join, the voter guard refusing a join and a restart (and the refused restart being made by the next run), a promotion in place, an abandoned promotion whose runner pods did not finish, a demotion refused while the member is a voter and made once it is removed, with the datastore moved aside and the identity kept, and a former member never starting a second cluster. `tests/mesh/run.sh native_server` runs a real native server as a privileged systemd container beside the Docker servers: it joins etcd as a learner, is promoted to a voter, and keeps the cluster writable throughout. It checks a first run, idempotence, `--check`, a changed setting restarting the service once, a client in another tailnet (refused, then logged out first when allowed), a client k3s did not log in, a refused downgrade, a wrong sum, an unsupported host, the secrets staying out of the output, and the node's readiness as a server reports it. ## Docker host disk @@ -381,7 +397,7 @@ Each Compose project runs its own `docker-prune` service against the same daemon - `github_runner_cluster_node_taints`: Kubernetes taints (`key=value:Effect`, no spaces) the host's node registers with, empty by default. `NoSchedule` keeps pods without a matching toleration off it and `NoExecute` also evicts the ones already there. k3s applies them on every registration, so they come back if the node object is deleted, which a taint set with `kubectl` does not. - `github_runner_cluster_etcd_heartbeat_interval`, `github_runner_cluster_etcd_election_timeout`: etcd's timing in milliseconds for every server, empty by default (k3s's own). Set together, the same on every server, the election timeout at least five times the heartbeat interval; see [etcd timing](#etcd-timing). - `github_runner_cluster_regrade` (default false): run the `regrade` service, which steps this server down under memory pressure while it keeps its etcd vote (see [Memory-pressure regrading](#memory-pressure-regrading)). Servers only. `github_runner_cluster_regrade_pressure_threshold` (percent, default 10) and `github_runner_cluster_regrade_available_floor_percent` (default 15) are the limit; `github_runner_cluster_regrade_step_up` (`automatic`, the default, or `manual`), `github_runner_cluster_regrade_sample_seconds`, `github_runner_cluster_regrade_sustain_seconds`, `github_runner_cluster_regrade_cooldown_seconds`, `github_runner_cluster_regrade_operation_timeout_seconds` and `github_runner_cluster_regrade_lease_duration_seconds` are the timing, all derived from `github_runner_cluster_regrade_pressure_window_seconds` (the kernel's `avg300`, 300 seconds, not a setting) unless set. `github_runner_cluster_regrade_level_label` and `github_runner_cluster_regrade_taint` are derived, never set directly. -- Per-host overrides: `github_runner_cluster_node_role` (`server` or `agent`), `github_runner_cluster_bootstrap` (true picks the bootstrap host, false excludes a host), `github_runner_cluster_server_url` (join this server instead, for a cluster whose servers are outside the inventory), `github_runner_cluster_tls_sans`, `github_runner_cluster_node_address`. The run fails if the result has an even number of servers, no servers, or more than one bootstrap host. +- Per-host overrides: `github_runner_cluster_node_role` (`server` or `agent`), `github_runner_cluster_bootstrap` (true picks the bootstrap host, false excludes a host), `github_runner_cluster_server_url` (join this server instead, for a cluster whose servers are outside the inventory), `github_runner_cluster_tls_sans`, `github_runner_cluster_node_address`, and `github_runner_cluster_voter_eligible` (true stages a native host as a standby server, its server configuration written and its name on every server's certificate, without making it a server; see [Native servers](#native-servers)). The run fails if the result has an even number of servers, no servers, or more than one bootstrap host. - `github_runner_cluster_docker_context` (default empty): pins the Docker context this host's node runs in (see [Docker contexts](#docker-contexts)). - `github_runner_cluster_network_mode` (default `bridge`, or `host`): the network the node's container runs on, and with `host`, `github_runner_cluster_lan_cidr` (required: the LAN the nodes share) and `github_runner_cluster_network_probe_image` (default `busybox:1.37`, for reading the host's addresses) (see [Networking](#networking)). - `github_runner_cluster_runtime_autostart_at` (default `auto`; `login` or `boot`) and `github_runner_cluster_runtime_boot_profile` (default `node`): whether the Colima profile is started by a launchd agent at login or by a launchd daemon at boot, and which profile `auto` starts at boot (see [Container runtime after a reboot](#container-runtime-after-a-reboot-macos)). @@ -392,13 +408,15 @@ Each Compose project runs its own `docker-prune` service against the same daemon - `github_runner_cluster_server_ready_timeout` (default 600): the most seconds, about, the role waits for a recreated server's node to report Ready before the next server's turn (see [Node passwords and recreating servers](#node-passwords-and-recreating-servers)). - `github_runner_cluster_node_mode` (default `compose`): how this host runs its k3s node, `compose` or `native` (see [Native nodes](#native-nodes)). Per host, in its `host_vars`. - `github_runner_cluster_native_start` (default `true`): whether a native node's service is enabled, started and restarted when something it reads changed, and the node then waited for. `false` only installs and configures. -- `github_runner_cluster_native_ready_timeout` (default `github_runner_cluster_server_ready_timeout`): the most seconds, about, the role waits for a started native node to be Ready on a new heartbeat and on the cluster's tailnet. +- `github_runner_cluster_native_ready_timeout` (default `github_runner_cluster_server_ready_timeout`): the most seconds, about, the role waits for a started native node to be Ready on a new heartbeat and on the cluster's tailnet, and a server also an etcd voter. +- `github_runner_cluster_native_demotion_min_voters` (3, not a setting): the fewest voters, every one Ready, a demotion leaves; three is the smallest etcd membership that keeps quorum through a failure. +- `github_runner_cluster_native_runner_wait_timeout` (default `github_runner_cluster_native_ready_timeout`): the most seconds, about, a switch between the server and agent units waits for the runner pods on the cordoned node to finish before it uncordons the node and fails, changing nothing else. - `github_runner_cluster_native_k3s_download_url` (default k3s's GitHub releases): where the k3s binary and its sum file are downloaded from, as `//`. - `github_runner_cluster_native_k3s_sha256` (default empty): the SHA-256 of the k3s binary for this host's architecture, instead of the sum the release itself lists. Set it from a source other than the download to catch a release that was replaced. - `github_runner_cluster_native_tailscale_install` (default `true`): whether the role installs the pinned Tailscale package from Tailscale's apt repository (Debian and Ubuntu). `false` leaves the host's client as it is and only checks its login. - `github_runner_cluster_native_tailscale_repository_url` (default `https://pkgs.tailscale.com/stable`): where that repository's signing key and list file are fetched from, when the host has none. - `github_runner_cluster_native_tailscale_leave_other_tailnet` (default `false`): whether the role logs a client out of another tailnet so that k3s can join the cluster's; `false` stops the run instead. -- `github_runner_cluster_native_service` and `github_runner_cluster_native_join_host` are derived: the service's name (`k3s-agent`) and the host in the node's join URL, which the Tailscale wrapper waits for. `github_runner_cluster_native_bin_dir`, `github_runner_cluster_native_lib_dir`, `github_runner_cluster_native_config_dir`, `github_runner_cluster_native_unit_dir`, `github_runner_cluster_native_tailscale_cli`, `github_runner_cluster_native_apt_keyring` and `github_runner_cluster_native_apt_source` are where the native node's files live, which are not settings, and are variables only so the tests can point them at a stand-in. +- `github_runner_cluster_native_agent_service`, `github_runner_cluster_native_server_service`, `github_runner_cluster_native_service`, `github_runner_cluster_native_join_url`, `github_runner_cluster_native_join_host`, `github_runner_cluster_native_role_file` and `github_runner_cluster_native_restart_marker` are derived: the units' names (`k3s-agent` and `k3s`), the unit of the host's role, the URL the node joins and the host in it, which the Tailscale wrapper waits for, the role file both units' conditions read, and the prefix of the per-unit file that says a restart is waiting. `github_runner_cluster_native_bin_dir`, `github_runner_cluster_native_lib_dir`, `github_runner_cluster_native_config_dir`, `github_runner_cluster_native_unit_dir`, `github_runner_cluster_native_data_dir`, `github_runner_cluster_native_tailscale_cli`, `github_runner_cluster_native_apt_keyring` and `github_runner_cluster_native_apt_source` are where the native node's files live, which are not settings, and are variables only so the tests can point them at a stand-in. ## Examples diff --git a/roles/github_runner_cluster/defaults/main.yml b/roles/github_runner_cluster/defaults/main.yml index 689e4807..00fb04b3 100644 --- a/roles/github_runner_cluster/defaults/main.yml +++ b/roles/github_runner_cluster/defaults/main.yml @@ -62,6 +62,8 @@ github_runner_cluster_k3s_tested_version: "v1.34.1-k3s1" github_runner_cluster_allow_k3s_version_jump: false # The most seconds the version check spends reading what one server runs, a positive whole number: it bounds the whole read on that server's host (its Docker daemon, the docker exec and the API call, through the task's own timeout) and is also the API call's --request-timeout. Long enough for a busy API server to list the nodes, short enough that a server whose Docker daemon or API server hangs does not hold up every host's run; a server that does not answer in time counts as one that could not be read. 0 is refused, since kubectl reads it as no timeout at all. A read cut off this way can leave its docker process running on that host until the daemon answers. github_runner_cluster_cluster_read_timeout: 30 +# Derived, never set directly: true makes tasks/cluster_nodes_read.yml read the cluster through its servers even when github_runner_cluster_allow_k3s_version_jump skips the version comparison, for a read the role needs for another reason (whether a cluster exists for a native bootstrap host to join, tasks/native/role.yml). The task that needs it passes it as a variable of the include. +github_runner_cluster_running_read_always: false # Secrets, as plain variables from any source Ansible reads (ansible-vault, a lookup, or a secrets adapter role such as github_runner_secrets_onepassword). github_runner_cluster_k3s_token is the shared node-join secret and has no default. github_runner_cluster_tailscale_join_key is, with the default tailscale mesh provider, what k3s's own --vpn-auth integration (see docker-compose.yml) uses on every host: it runs Tailscale INSIDE each node's own k3s container, joining the tailnet as its own device and using flannel's native "tailscale" backend for pod traffic. github_runner_cluster_tailscale_api_token, when set, lets tasks/mesh/tailscale_api.yml keep the tailnet's ACL grant and the join key itself provisioned; see that file's comments for what it manages and the one manual step (creating the API token) that has no API to automate. github_runner_cluster_tailscale_join_key: "" @@ -232,10 +234,10 @@ github_runner_cluster_deployed_version_annotation: "github-runner.exadev.com/dep # The collection version this role was released as, stamped by scripts/release/stamp_version.py. A server host records it as an annotation on its own node once the node has registered, and the heartbeat publishes it in fleet-health.json. It is what was applied, not what is running, and an agent host, which has no API of its own, records nothing. github_runner_cluster_collection_version: "2.22.0" -# How this host runs its k3s node. compose (the default) is everything above: k3s in a container under Docker Compose, with Tailscale inside the container. native runs k3s as an agent directly on the host, as the systemd service k3s-agent, with no Docker or Compose layer: the node's containerd then uses the host's filesystem and network instead of nesting inside a container's. Per host, in its host_vars. native is for a Linux host that systemd runs, and only as an agent: tasks/topology.yml never makes it a server, and refuses a native host that asks to be one. See the role README's Native nodes section, which also says what a native node gives up. +# How this host runs its k3s node. compose (the default) is everything above: k3s in a container under Docker Compose, with Tailscale inside the container. native runs k3s directly on the host under systemd, as the service k3s-agent (an agent) or k3s (a server), with no Docker or Compose layer: the node's containerd then uses the host's filesystem and network instead of nesting inside a container's, and a server's etcd writes to the host's own disk. Per host, in its host_vars. native is for a Linux host that systemd runs. A native host is a server only when it sets github_runner_cluster_node_role: server or github_runner_cluster_bootstrap: true; it is never given a server place automatically and does not count towards the automatic number, so an inventory whose native hosts set neither keeps every one of them an agent. A host with github_runner_cluster_voter_eligible: true (a per-host inventory variable, like the other overrides) is also staged as a standby server, its configuration written and its name on every server's certificate, so that it can later be promoted. See the role README's Native nodes section, which also says what a native node gives up. github_runner_cluster_node_mode: compose -# Native mode only, from here on. Whether the role enables and starts the k3s-agent service, restarts it when its binary or configuration changed, and then waits for the node to report Ready as a server sees it. false installs and configures everything and leaves the service as it finds it (a first install stays disabled and stopped), for staging a host before the cluster is joined. +# Native mode only, from here on. Whether the role enables and starts the node's service (k3s-agent or k3s), restarts it when its binary or configuration changed, switches the host between the server and the agent unit when its role changed, and then waits for the node to report Ready as a server sees it. false installs and configures everything and leaves the services as it finds them (a first install stays disabled and stopped, and a host whose role changed keeps running its old unit), for staging a host before the cluster is joined or before its role is switched. github_runner_cluster_native_start: true # The most seconds, about, the role waits for the started node to report Ready on a heartbeat from the k3s that was just started. The same bound as a recreated server's. github_runner_cluster_native_ready_timeout: "{{ github_runner_cluster_server_ready_timeout }}" @@ -250,15 +252,28 @@ github_runner_cluster_native_tailscale_repository_url: https://pkgs.tailscale.co # The host's Tailscale client is the node's: k3s's --vpn-auth runs `tailscale up` against it, and the node's tailnet device is the host's. A client logged in to another tailnet would not join the cluster's, so the role refuses to carry on with the client in one, unless this is true, which logs the client out of it first (`tailscale logout`): the host leaves that tailnet, and anything reaching the host over it stops. Only for a host that is dedicated to the cluster. github_runner_cluster_native_tailscale_leave_other_tailnet: false -# Derived, never set directly: the name of the node's systemd service, as k3s's own install script names an agent's. -github_runner_cluster_native_service: k3s-agent -# Derived, never set directly: the host name in the node's join URL, which the Tailscale wrapper waits for after `tailscale up`. -github_runner_cluster_native_join_host: "{{ github_runner_cluster_effective_server_url | urlsplit('hostname') }}" -# Not settings: where the native node's files live. Variables only so the tests can point them at a stand-in. The binary goes in bin_dir, the PATH wrapper and the resolver helper under lib_dir, the configuration and the two secrets (token, vpn-auth) in config_dir, and the unit in unit_dir; tailscale_cli is the Tailscale command the package installs. +# Derived, never set directly: the names k3s's own install script gives the units of an agent and a server, and the unit this host runs, from the role tasks/native/role.yml decided. +github_runner_cluster_native_agent_service: k3s-agent +github_runner_cluster_native_server_service: k3s +github_runner_cluster_native_service: "{{ github_runner_cluster_native_server_service if github_runner_cluster_native_role == 'server' else github_runner_cluster_native_agent_service }}" +# Derived, never set directly: the server this host's k3s joins, as https://
:. A node that joins the bootstrap host dials it; the bootstrap host itself, when it runs as an agent or rejoins a cluster that already exists, dials the first other server of the inventory. Empty for a host with nothing to dial. +github_runner_cluster_native_join_url: "{{ github_runner_cluster_effective_server_url if github_runner_cluster_effective_server_url | length > 0 else (github_runner_cluster_topology.hosts[inventory_hostname].peer_server_urls | first | default('')) }}" +# Derived, never set directly: the host name in that URL, which the Tailscale wrapper waits for. +github_runner_cluster_native_join_host: "{{ github_runner_cluster_native_join_url | urlsplit('hostname') if github_runner_cluster_native_join_url | length > 0 else '' }}" +# Derived, never set directly: the file that names the role the host runs, server or agent, which each unit's ExecCondition reads (files/native/fleet-role-is.sh), so the host's role outlives a restart, a reboot and a run of the role. +github_runner_cluster_native_role_file: "{{ github_runner_cluster_native_data_dir }}/fleet-role" +# Not a setting: the fewest voters a demotion leaves, every one Ready. Three is the smallest etcd membership that keeps quorum through a failure (a majority of three is two; a majority of two is both, which etcd calls unsafe), so a demotion that would leave fewer is refused and the cluster gains a voter first. +github_runner_cluster_native_demotion_min_voters: 3 +# Derived, never set directly: the prefix of the file, one per unit (-k3s-agent, -k3s), that says the unit's files changed since it last started, so a restart that a run refused or did not reach is made by the next run. +github_runner_cluster_native_restart_marker: "{{ github_runner_cluster_native_data_dir }}/fleet-restart-pending" +# How long, about, a role switch waits for the runner pods on the node to finish after it has cordoned it, in seconds, before it gives up, uncordons the node and fails, changing nothing else: a planned switch never cuts a job. Each poll lists the node's pods, so a job that finishes is seen on the next poll. +github_runner_cluster_native_runner_wait_timeout: "{{ github_runner_cluster_native_ready_timeout }}" +# Not settings: where the native node's files live. Variables only so the tests can point them at a stand-in. The binary goes in bin_dir, the PATH wrapper and the helpers under lib_dir, the configurations and the two secrets (token, vpn-auth) in config_dir, the units in unit_dir, and data_dir is k3s's own data directory (a server's datastore is server/db under it, and the role file sits beside it); tailscale_cli is the Tailscale command the package installs. github_runner_cluster_native_bin_dir: /usr/local/bin github_runner_cluster_native_lib_dir: /usr/local/lib/github-runner-k3s github_runner_cluster_native_config_dir: /etc/rancher/k3s github_runner_cluster_native_unit_dir: /etc/systemd/system +github_runner_cluster_native_data_dir: /var/lib/rancher/k3s github_runner_cluster_native_tailscale_cli: /usr/bin/tailscale # Not settings: where Tailscale's apt signing key and repository list file go, which are the paths Tailscale's own install instructions use. A host that has either already is left alone. Variables only so the tests can point them at a stand-in. github_runner_cluster_native_apt_keyring: /usr/share/keyrings/tailscale-archive-keyring.gpg diff --git a/roles/github_runner_cluster/files/native/fleet-role-is.sh b/roles/github_runner_cluster/files/native/fleet-role-is.sh new file mode 100755 index 00000000..7a2d54ea --- /dev/null +++ b/roles/github_runner_cluster/files/native/fleet-role-is.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# The ExecCondition= of both native k3s units (templates/native/k3s.service.j2 and k3s-agent.service.j2): succeeds only when the role file names the role the unit runs, so the server and the agent unit can never run together, and Restart=always can never start a unit whose role the host no longer has (a demoted server, say). systemd skips the rest of a unit's start, without marking it failed, when its ExecCondition= exits 1 to 254 (systemd.service(5), ExecCondition=, systemd 243 and later), and Restart=always does not restart a unit whose start was skipped that way (service_shall_restart in src/core/service.c at v255, the systemd of Ubuntu 24.04: `case SERVICE_RESTART_ALWAYS: return s->result != SERVICE_SKIP_CONDITION;`). +# +# Usage: fleet-role-is.sh ROLE ROLE_FILE. ROLE is server or agent; ROLE_FILE holds one of them, written by the cluster role (tasks/native/role.yml) and, later, by whatever changes the host's role. A missing or unreadable file names no role, so neither unit starts until something has decided one. +set -u + +role=${1:?usage: fleet-role-is.sh ROLE ROLE_FILE} +file=${2:?usage: fleet-role-is.sh ROLE ROLE_FILE} + +recorded=$(cat "$file" 2>/dev/null) || recorded="" +if [ "$recorded" = "$role" ]; then + exit 0 +fi +echo "fleet-role-is: ${file} names '${recorded:-no role}', not ${role}, so this unit does not start" >&2 +exit 1 diff --git a/roles/github_runner_cluster/files/native/write-node-ip.sh b/roles/github_runner_cluster/files/native/write-node-ip.sh new file mode 100755 index 00000000..dac1ea01 --- /dev/null +++ b/roles/github_runner_cluster/files/native/write-node-ip.sh @@ -0,0 +1,53 @@ +#!/bin/sh +# Writes the k3s configuration drop-in that sets a native server's node-ip to the host's own Tailscale IPv4 address, from the server unit's ExecStartPre (templates/native/k3s.service.j2), before every start. +# +# k3s builds etcd's peer URL from its PrivateIP, which it takes from --node-ip before the --vpn-auth integration sets the advertise address (pkg/cli/server/server.go at v1.34.1+k3s1: `serverConfig.ControlConfig.PrivateIP = util.GetFirstValidIPString(cmds.AgentConfig.NodeIP.Value())`, and `Etcd IP (PrivateIP) remains the local IP` otherwise; the peer URL is `https://
:2380`, peerURL in pkg/etcd/etcd.go). Without node-ip a native server's etcd would advertise a LAN address, which breaks the member's peer traffic as soon as a laptop changes network; the Docker servers pass --node-ip for the same reason (k3s/node-entrypoint.sh). k3s reads every *.yaml in the directory beside its --config file after the file itself, in name order (dotDFiles and readConfigFile in pkg/configfilearg/parser.go), so this drop-in sets node-ip without the role writing the address into server.yaml. +# +# The address is the one the host's Tailscale client has now. A client with none (a host whose client has never been logged in to the cluster's tailnet) is logged in first with exactly the `tailscale up` k3s's own --vpn-auth start runs (StartVPN in pkg/vpn/vpn.go: `up --authkey --timeout=30s --reset`, then `--login-server ` when set and the words after `,extraArgs=`), from the same --vpn-auth file, since k3s runs that only after it has read node-ip. A logged-in client keeps its device and address through that call, so it registers nothing new. +# +# Usage: write-node-ip.sh OUTPUT_FILE VPN_AUTH_FILE. Reads GITHUB_RUNNER_TAILSCALE, the real Tailscale command, from the unit's environment. The join key is never printed. The file is replaced in one rename, so k3s never reads half of it. +set -eu + +output=${1:?usage: write-node-ip.sh OUTPUT_FILE VPN_AUTH_FILE} +vpn_auth_file=${2:?usage: write-node-ip.sh OUTPUT_FILE VPN_AUTH_FILE} +tailscale=${GITHUB_RUNNER_TAILSCALE:?GITHUB_RUNNER_TAILSCALE must name the real tailscale command} + +address() { "$tailscale" ip -4 2>/dev/null | head -n 1; } + +ip=$(address || true) +if [ -z "$ip" ]; then + vpn_auth=$(cat "$vpn_auth_file") + # The parameters before ",extraArgs=" are comma-separated key=value pairs; the words after it are passed to `tailscale up` as they are (processCLIArgs and getVPNAuthInfo in pkg/vpn/vpn.go). + case "$vpn_auth" in + *",extraArgs="*) parameters=${vpn_auth%%,extraArgs=*}; extra=${vpn_auth#*,extraArgs=} ;; + *) parameters=$vpn_auth; extra="" ;; + esac + join_key="" + control_server_url="" + old_ifs=$IFS + IFS=, + for parameter in $parameters; do + case "$parameter" in + joinKey=*) join_key=${parameter#joinKey=} ;; + controlServerURL=*) control_server_url=${parameter#controlServerURL=} ;; + esac + done + IFS=$old_ifs + [ -n "$join_key" ] || { echo "write-node-ip: ${vpn_auth_file} names no joinKey" >&2; exit 1; } + set -- up --authkey "$join_key" --timeout=30s --reset + if [ -n "$control_server_url" ]; then set -- "$@" --login-server "$control_server_url"; fi + # shellcheck disable=SC2086 # The extra arguments are separate words, as k3s splits them (strings.Fields). + set -- "$@" $extra + "$tailscale" "$@" >/dev/null + ip=$(address) +fi + +case "$ip" in + *.*.*.*) ;; + *) echo "write-node-ip: the Tailscale client reports no IPv4 address ('${ip}')" >&2; exit 1 ;; +esac + +mkdir -p "$(dirname "$output")" +temporary="${output}.tmp" +printf '# Written by write-node-ip.sh before every start of the server unit: the host'"'"'s Tailscale IPv4 address, which etcd advertises as its peer URL.\nnode-ip: "%s"\n' "$ip" > "$temporary" +mv "$temporary" "$output" diff --git a/roles/github_runner_cluster/meta/argument_specs.yml b/roles/github_runner_cluster/meta/argument_specs.yml index 2708cc20..9ba35d3c 100644 --- a/roles/github_runner_cluster/meta/argument_specs.yml +++ b/roles/github_runner_cluster/meta/argument_specs.yml @@ -5,21 +5,29 @@ argument_specs: short_description: Brings a host up as a k3s node, in a container under Docker Compose or natively under systemd. description: - With github_runner_cluster_node_mode compose (the default) the host runs its k3s node as a server or an agent in a container under Docker Compose, with Tailscale inside the container. - - With github_runner_cluster_node_mode native a Linux host runs k3s as an agent directly under systemd, with no Docker or Compose layer, and its own Tailscale client joins the cluster's tailnet through k3s's --vpn-auth. Its tasks that change the host use become (sudo), so run the play with passwordless sudo for the connecting user or with --ask-become-pass. + - With github_runner_cluster_node_mode native a Linux host runs k3s directly under systemd, as the k3s-agent unit or, when the inventory makes it a server, the k3s unit, with no Docker or Compose layer, and its own Tailscale client joins the cluster's tailnet through k3s's --vpn-auth. Its tasks that change the host use become (sudo), so run the play with passwordless sudo for the connecting user or with --ask-become-pass. options: github_runner_cluster_node_mode: type: str default: compose choices: [compose, native] - description: How this host runs its k3s node. compose is a container under Docker Compose. native is the k3s-agent systemd service on a Linux host, always as an agent. + description: How this host runs its k3s node. compose is a container under Docker Compose. native is a systemd unit on a Linux host, k3s-agent for an agent and k3s for a server; a native host is a server only when it sets github_runner_cluster_node_role server or github_runner_cluster_bootstrap true. github_runner_cluster_native_start: type: bool default: true - description: Whether a native node's service is enabled, started and restarted when its binary or configuration changed, and the node then waited for until it reports Ready. false installs and configures everything and leaves the service as it is. + description: Whether a native node's unit is enabled, started, restarted when its binary or configuration changed and switched when the host's role changed, and the node then waited for until it reports Ready. false installs and configures everything and leaves the units as they are. github_runner_cluster_native_ready_timeout: type: int default: 600 description: The most seconds, about, the role waits for a native node to report Ready after it started. + github_runner_cluster_native_runner_wait_timeout: + type: int + default: 600 + description: The most seconds, about, a switch between the server and agent units waits for the runner pods on the cordoned node to finish before it uncordons the node and fails, changing nothing else. + github_runner_cluster_native_demotion_min_voters: + type: int + default: 3 + description: The fewest other etcd voters, every one Ready, a demotion of a native server leaves. Not a setting; three is the smallest membership that keeps quorum through a failure. github_runner_cluster_native_k3s_download_url: type: str default: https://github.com/k3s-io/k3s/releases/download @@ -47,15 +55,19 @@ argument_specs: github_runner_cluster_native_lib_dir: type: path default: /usr/local/lib/github-runner-k3s - description: Where the Tailscale wrapper and the resolver helper are installed. Not a setting, and a variable only so the tests can point it at a stand-in. + description: Where the Tailscale wrapper and the helpers the units run are installed. Not a setting, and a variable only so the tests can point it at a stand-in. github_runner_cluster_native_config_dir: type: path default: /etc/rancher/k3s - description: Where k3s reads its configuration, and where the role keeps the node's token and --vpn-auth value. Not a setting, and a variable only so the tests can point it at a stand-in. + description: Where k3s reads its configurations (agent.yaml, server.yaml and server.yaml.d), and where the role keeps the node's token and --vpn-auth value. Not a setting, and a variable only so the tests can point it at a stand-in. github_runner_cluster_native_unit_dir: type: path default: /etc/systemd/system - description: Where the k3s-agent unit is installed. Not a setting, and a variable only so the tests can point it at a stand-in. + description: Where the k3s-agent and k3s units are installed. Not a setting, and a variable only so the tests can point it at a stand-in. + github_runner_cluster_native_data_dir: + type: path + default: /var/lib/rancher/k3s + description: k3s's own data directory, which holds a server's datastore (server/db) and the role file. Not a setting, and a variable only so the tests can point it at a stand-in. github_runner_cluster_native_tailscale_cli: type: path default: /usr/bin/tailscale diff --git a/roles/github_runner_cluster/tasks/cluster_nodes_read.yml b/roles/github_runner_cluster/tasks/cluster_nodes_read.yml index 1d735f74..a412652c 100644 --- a/roles/github_runner_cluster/tasks/cluster_nodes_read.yml +++ b/roles/github_runner_cluster/tasks/cluster_nodes_read.yml @@ -1,8 +1,17 @@ --- -# Reads every node's k3s version through each server of the inventory and says what the read could not compare. Shared by running_version.yml (the Docker path) and native/version_check.yml (the native node). Registers github_runner_cluster_running_reads, the loop results that the cluster_k3s_version_change_blockers filter (plugins/filter/cluster_versions.py) takes as the server reads. Needs topology.yml to have run. Every task only reads, and runs under --check too. +# Reads every node's k3s version and live role through each server of the inventory and says what the read could not compare. Shared by running_version.yml (the Docker path), native/version_check.yml (the native node) and native/role.yml (whether a cluster exists for a native bootstrap host to join). Registers github_runner_cluster_running_reads, whose results the cluster_k3s_version_change_blockers filter (plugins/filter/cluster_versions.py) takes as the server reads: one per server, a Docker server's read through its container and a native server's through `k3s kubectl` on its host. Needs topology.yml to have run. Every task only reads, and runs under --check too; github_runner_cluster_running_read_always makes it read even when github_runner_cluster_allow_k3s_version_jump skips the version comparison. -- name: Read every node's k3s version through each server of the inventory - # Through every server of the group, this one first, each on its own host with its own pinned Docker context, since any one of them may be stopped or being rebuilt while the others run. The image's kubectl inside a server's container reads that server's kubeconfig, so the host needs none; it is run as kubectl, since `k3s kubectl` in the rancher/k3s image answers "unknown command". Each read prints "stopped" when the server's container does not run, or "running" and one " " line per node. Each read is bounded by github_runner_cluster_cluster_read_timeout as a whole (the task's own timeout, so a hung Docker daemon or docker exec cannot hold up the run) and as the API call's --request-timeout. A read that fails (a daemon or host that cannot be reached, a read that runs out of time) is recorded rather than failing here, so the filter can say which servers could not be read. Each node line also carries its control-plane label, so the filter can tell servers from agents. +- name: Work out which servers of the inventory to read through, Docker and native + # Every server of the group, this one first, and every host marked eligible, which may run as a server whatever the inventory says (the role controller changes it at run time), since any one of them may be stopped or being rebuilt while the others run. A host that is not running a server reads as stopped. + ansible.builtin.set_fact: + github_runner_cluster_running_servers: >- + {{ (([inventory_hostname] if github_runner_cluster_effective_node_role == 'server' or github_runner_cluster_topology.hosts[inventory_hostname].eligible else []) + + (github_runner_cluster_topology.hosts | dict2items | selectattr('value.role', 'equalto', 'server') | map(attribute='key') | list) + + (github_runner_cluster_topology.hosts | dict2items | selectattr('value.eligible') | map(attribute='key') | list)) | unique }} + when: not github_runner_cluster_allow_k3s_version_jump | bool or github_runner_cluster_running_read_always | bool + +- name: Read every node's k3s version through each Docker server of the inventory + # Each on its own host with its own pinned Docker context. The image's kubectl inside a server's container reads that server's kubeconfig, so the host needs none; it is run as kubectl, since `k3s kubectl` in the rancher/k3s image answers "unknown command". Each read prints "stopped" when the server's container does not run, or "running" and one " " line per node, the label telling servers from agents as the cluster sees them now. Each read is bounded by github_runner_cluster_cluster_read_timeout as a whole (the task's own timeout, so a hung Docker daemon or docker exec cannot hold up the run) and as the API call's --request-timeout. A read that fails (a daemon or host that cannot be reached, a read that runs out of time) is recorded rather than failing here, so the filter can say which servers could not be read. ansible.builtin.shell: | set -eu state=$(docker ps --all --filter "name=^/${container}\$" --format '{{ '{{' }}.State{{ '}}' }}') @@ -17,25 +26,53 @@ {{ ({'DOCKER_CONTEXT': github_runner_cluster_running_read_context} if github_runner_cluster_running_read_context | length > 0 else {}) | combine({'container': github_runner_cluster_running_read_container, 'timeout': github_runner_cluster_cluster_read_timeout | string}) }} delegate_to: "{{ item }}" - loop: "{{ github_runner_cluster_running_servers }}" + loop: "{{ github_runner_cluster_running_servers | reject('in', github_runner_cluster_topology.hosts | dict2items | selectattr('value.native') | map(attribute='key') | list) | list }}" vars: - github_runner_cluster_running_servers: >- - {{ ([inventory_hostname] if github_runner_cluster_effective_node_role == 'server' else []) - + (github_runner_cluster_topology.hosts | dict2items | selectattr('value.role', 'equalto', 'server') - | map(attribute='key') | reject('equalto', inventory_hostname) | list) }} github_runner_cluster_running_read_container: >- {{ github_runner_cluster_container_name if item == inventory_hostname else hostvars[item].github_runner_cluster_container_name | default('github-runner-k3s') }} github_runner_cluster_running_read_context: >- {{ (github_runner_cluster_docker_context if item == inventory_hostname else hostvars[item].github_runner_cluster_docker_context | default('')) | trim }} - register: github_runner_cluster_running_reads + register: github_runner_cluster_running_reads_docker + changed_when: false + failed_when: false + ignore_unreachable: true + check_mode: false + timeout: "{{ github_runner_cluster_cluster_read_timeout | int }}" + when: not github_runner_cluster_allow_k3s_version_jump | bool or github_runner_cluster_running_read_always | bool + +- name: Read every node's k3s version through each native server of the inventory + # The same read, through `k3s kubectl` on a native server's host, which reads the admin kubeconfig k3s writes there for root only (hence become), and "stopped" when the server unit is not active. The unit's name and the binary's place are the role's constants, read from the inventory where a host sets them (the tests do). + become: true + ansible.builtin.shell: | + set -eu + if ! systemctl is-active --quiet "$service"; then + echo stopped + exit 0 + fi + echo running + "$k3s" kubectl get nodes --request-timeout="${timeout}s" \ + -o 'jsonpath={range .items[*]}{.metadata.name}{" "}{.status.nodeInfo.kubeletVersion}{" "}{.metadata.labels.node-role\.kubernetes\.io/control-plane}{"\n"}{end}' + environment: + service: "{{ github_runner_cluster_native_server_service }}" + k3s: "{{ hostvars[item].github_runner_cluster_native_bin_dir | default(github_runner_cluster_native_bin_dir) }}/k3s" + timeout: "{{ github_runner_cluster_cluster_read_timeout | string }}" + delegate_to: "{{ item }}" + loop: "{{ github_runner_cluster_running_servers | select('in', github_runner_cluster_topology.hosts | dict2items | selectattr('value.native') | map(attribute='key') | list) | list }}" + register: github_runner_cluster_running_reads_native changed_when: false failed_when: false ignore_unreachable: true check_mode: false timeout: "{{ github_runner_cluster_cluster_read_timeout | int }}" - when: not github_runner_cluster_allow_k3s_version_jump | bool + when: not github_runner_cluster_allow_k3s_version_jump | bool or github_runner_cluster_running_read_always | bool + +- name: Gather the reads through every server + ansible.builtin.set_fact: + github_runner_cluster_running_reads: + results: "{{ github_runner_cluster_running_reads_docker.results + github_runner_cluster_running_reads_native.results }}" + when: not github_runner_cluster_allow_k3s_version_jump | bool or github_runner_cluster_running_read_always | bool - name: Say what the version check could not compare # Only when no server answered and none failed: every server of the inventory was read as stopped, or there is none. Which of those it is, and whether the cluster lives outside the inventory, comes from the topology: a group with no bootstrap host joins a cluster that one of its hosts did not bootstrap. @@ -48,6 +85,6 @@ msg: >- {% if github_runner_cluster_running_reads.results | default([]) | length == 0 %}No server of {{ github_runner_cluster_group }} is in the inventory, so the cluster this node joins cannot be read; only {{ github_runner_cluster_own_subject }} was compared.{% elif github_runner_cluster_running_joins_outside | bool %}{{ github_runner_cluster_group }} joins a cluster whose other servers are outside the inventory, and none of its own servers runs its container, so the cluster could not be read; only {{ github_runner_cluster_own_subject }} was compared.{% else %}No server of {{ github_runner_cluster_group }} runs its container, so there is no cluster to compare with yet (a first bootstrap, or a cluster whose servers are all stopped); only {{ github_runner_cluster_own_subject }} was compared.{% endif %} when: - - not github_runner_cluster_allow_k3s_version_jump | bool + - not github_runner_cluster_allow_k3s_version_jump | bool or github_runner_cluster_running_read_always | bool - github_runner_cluster_running_reads.results | default([]) | rejectattr('stdout', 'defined') | list | length == 0 - github_runner_cluster_running_reads.results | default([]) | map(attribute='stdout') | map('trim') | reject('equalto', 'stopped') | list | length == 0 diff --git a/roles/github_runner_cluster/tasks/main.yml b/roles/github_runner_cluster/tasks/main.yml index a6ccf621..bc92c524 100644 --- a/roles/github_runner_cluster/tasks/main.yml +++ b/roles/github_runner_cluster/tasks/main.yml @@ -1,7 +1,7 @@ --- # Brings this host up as a k3s node, natively under systemd (github_runner_cluster_node_mode: native, tasks/native/main.yml) or, by default, under Docker Compose: a server (bootstrapping the cluster or joining it) or an agent, as tasks/topology.yml decides from the cluster group. Safe to rerun: the .env is templated and docker compose converges to it. -- name: Bring this host up as a native k3s agent, with no Docker +- name: Bring this host up as a native k3s server or agent, with no Docker # github_runner_cluster_node_mode: native. Everything below drives Docker, which a native node does not have, so the role ends after this; see tasks/native/main.yml. ansible.builtin.include_tasks: native/main.yml when: github_runner_cluster_node_mode == "native" diff --git a/roles/github_runner_cluster/tasks/native/api.yml b/roles/github_runner_cluster/tasks/native/api.yml new file mode 100644 index 00000000..a3f8d1f4 --- /dev/null +++ b/roles/github_runner_cluster/tasks/native/api.yml @@ -0,0 +1,25 @@ +--- +# Works out how the role reaches the cluster's API for this native host: through this host's own server (`k3s kubectl`, as root, since k3s writes the admin kubeconfig for root only) when github_runner_cluster_native_api_self is true, and otherwise through the first other server of the inventory, a Docker server's container or a native server's `k3s kubectl` on its own host. Publishes github_runner_cluster_native_api_host (empty when the inventory names no other server, as for a node that joins a cluster outside it), github_runner_cluster_native_api_argv (the kubectl command, to which a task appends its arguments), github_runner_cluster_native_api_become and github_runner_cluster_native_api_environment. Only works things out; reads nothing. + +- name: Work out which server's API this host's checks go through + vars: + github_runner_cluster_native_api_other: >- + {{ github_runner_cluster_topology.servers | reject('equalto', inventory_hostname) | first | default('') }} + github_runner_cluster_native_api_chosen: >- + {{ inventory_hostname if github_runner_cluster_native_api_self | bool else github_runner_cluster_native_api_other }} + github_runner_cluster_native_api_chosen_native: >- + {{ github_runner_cluster_native_api_chosen | length > 0 and github_runner_cluster_topology.hosts[github_runner_cluster_native_api_chosen].native }} + github_runner_cluster_native_api_context: >- + {{ (hostvars[github_runner_cluster_native_api_chosen].github_runner_cluster_docker_context | default('') | trim) + if github_runner_cluster_native_api_chosen | length > 0 else '' }} + ansible.builtin.set_fact: + github_runner_cluster_native_api_host: "{{ github_runner_cluster_native_api_chosen }}" + github_runner_cluster_native_api_become: "{{ github_runner_cluster_native_api_chosen_native | bool }}" + # A native server's k3s is where the inventory says that host keeps it, or the role's own place, the same constant on every host. + github_runner_cluster_native_api_argv: >- + {{ [(hostvars[github_runner_cluster_native_api_chosen].github_runner_cluster_native_bin_dir | default(github_runner_cluster_native_bin_dir)) ~ '/k3s', 'kubectl'] + if github_runner_cluster_native_api_chosen_native | bool + else (['docker', 'exec', hostvars[github_runner_cluster_native_api_chosen].github_runner_cluster_container_name | default('github-runner-k3s'), 'kubectl'] + if github_runner_cluster_native_api_chosen | length > 0 else []) }} + github_runner_cluster_native_api_environment: >- + {{ {'DOCKER_CONTEXT': github_runner_cluster_native_api_context} if github_runner_cluster_native_api_context | length > 0 else {} }} diff --git a/roles/github_runner_cluster/tasks/native/config.yml b/roles/github_runner_cluster/tasks/native/config.yml index b3b5bbfb..12fd0f7a 100644 --- a/roles/github_runner_cluster/tasks/native/config.yml +++ b/roles/github_runner_cluster/tasks/native/config.yml @@ -1,5 +1,5 @@ --- -# Writes everything the k3s-agent service reads: the two helper scripts, the configuration, the two secrets and the unit. Registers github_runner_cluster_native_changed, whether any of them (or the binary k3s_binary.yml installed) changed, which is what restarts a running service. The token and the --vpn-auth value are secrets: each is in a file of its own, mode 0600, written by a task that logs nothing, and the configuration only names the files. +# Writes everything the native units read: the helper scripts, the agent's configuration, the server's (on a server, or on a host marked eligible, which is staged as a standby), the two secrets, both units, and the role file when the host has none. Registers github_runner_cluster_native_agent_changed and github_runner_cluster_native_server_changed, whether anything the agent or the server unit reads changed (the binary k3s_binary.yml installed and the shared helpers and secrets count for both), which is what restarts a running unit. The token and the --vpn-auth value are secrets: each is in a file of its own, mode 0600, written by a task that logs nothing, and the configurations only name the files. Needs role.yml. - name: Create the directories the node's files go in become: true @@ -11,11 +11,12 @@ - {path: "{{ github_runner_cluster_native_lib_dir }}", mode: "0755"} - {path: "{{ github_runner_cluster_native_lib_dir }}/bin", mode: "0755"} - {path: "{{ github_runner_cluster_native_config_dir }}", mode: "0755"} + - {path: "{{ github_runner_cluster_native_data_dir }}", mode: "0755"} loop_control: label: "{{ item.path }}" -- name: Place the Tailscale wrapper and the resolver helper - # See the two files for what each does. Copied on every run, so a collection upgrade reaches the host. +- name: Place the Tailscale wrapper and the helpers the units run + # See each file for what it does. Copied on every run, so a collection upgrade reaches the host. become: true ansible.builtin.copy: src: "native/{{ item.src }}" @@ -24,17 +25,37 @@ loop: - {src: tailscale-up-wait.sh, dest: "{{ github_runner_cluster_native_lib_dir }}/bin/tailscale"} - {src: write-resolv-conf.sh, dest: "{{ github_runner_cluster_native_lib_dir }}/write-resolv-conf"} + - {src: write-node-ip.sh, dest: "{{ github_runner_cluster_native_lib_dir }}/write-node-ip"} + - {src: fleet-role-is.sh, dest: "{{ github_runner_cluster_native_lib_dir }}/fleet-role-is"} loop_control: label: "{{ item.dest }}" register: github_runner_cluster_native_helpers +- name: Record the role a host set up before the role file existed runs + # Before either unit is written: the new units start only while this file names their role, so a host that runs the agent unit from an earlier release keeps running it across the restart below. A host that has the file keeps it; changing it is a role switch (service.yml), never a run's side effect. A first install records the role it is to run, which nothing runs yet. + become: true + ansible.builtin.copy: + content: >- + {{ github_runner_cluster_native_current_role if github_runner_cluster_native_current_role | length > 0 else github_runner_cluster_native_role }} + dest: "{{ github_runner_cluster_native_role_file }}" + mode: "0644" + force: false + - name: Write the k3s agent configuration become: true ansible.builtin.template: - src: native/config.yaml.j2 - dest: "{{ github_runner_cluster_native_config_dir }}/config.yaml" + src: native/agent.yaml.j2 + dest: "{{ github_runner_cluster_native_config_dir }}/agent.yaml" mode: "0600" - register: github_runner_cluster_native_config + register: github_runner_cluster_native_agent_config + +- name: Remove the agent configuration of an earlier release + # Earlier releases wrote the agent's flags to k3s's default config.yaml, which a k3s started without --config would still read; both units now pass their own file. + become: true + ansible.builtin.file: + path: "{{ github_runner_cluster_native_config_dir }}/config.yaml" + state: absent + register: github_runner_cluster_native_old_config - name: Write the node's token # No trailing newline: k3s trims the file, and the bytes then are exactly the token. @@ -55,20 +76,86 @@ no_log: true register: github_runner_cluster_native_vpn_auth -- name: Write the k3s-agent unit +- name: Write the native units + # Both, on every native host: each starts only while the role file names its role, so the unit of the role the host does not run stays inert. become: true ansible.builtin.template: - src: native/k3s-agent.service.j2 - dest: "{{ github_runner_cluster_native_unit_dir }}/{{ github_runner_cluster_native_service }}.service" + src: "native/{{ item }}.service.j2" + dest: "{{ github_runner_cluster_native_unit_dir }}/{{ item }}.service" mode: "0644" - register: github_runner_cluster_native_unit + loop: + - "{{ github_runner_cluster_native_agent_service }}" + - "{{ github_runner_cluster_native_server_service }}" + register: github_runner_cluster_native_units -- name: Record whether anything the running service reads changed - ansible.builtin.set_fact: - github_runner_cluster_native_changed: >- +- name: Write or remove the server's configuration + when: github_runner_cluster_native_stage_server | bool + block: + - name: Create the server configuration's drop-in directory + become: true + ansible.builtin.file: + path: "{{ github_runner_cluster_native_config_dir }}/server.yaml.d" + state: directory + mode: "0755" + + - name: Write the k3s server configuration and whether it bootstraps or joins + become: true + ansible.builtin.template: + src: "native/{{ item.src }}" + dest: "{{ github_runner_cluster_native_config_dir }}/{{ item.dest }}" + mode: "0600" + loop: + - {src: server.yaml.j2, dest: server.yaml} + - {src: join.yaml.j2, dest: server.yaml.d/10-join.yaml} + loop_control: + label: "{{ item.dest }}" + register: github_runner_cluster_native_server_config + +- name: Remove the server's configuration from a host that neither is nor may become a server + # Its unit stays, inert while the role file names the agent. + become: true + ansible.builtin.file: + path: "{{ github_runner_cluster_native_config_dir }}/{{ item }}" + state: absent + loop: [server.yaml, server.yaml.d] + when: not github_runner_cluster_native_stage_server | bool + +- name: Read which units a run before this one changed and did not restart + # A restart this run makes can be refused (a server's, while another voter is not Ready) or fail after the files are written, and the next run would then find nothing changed and never restart the unit onto them. The marker is written whenever a unit's files change and removed once the unit has started on them. + ansible.builtin.stat: + path: "{{ github_runner_cluster_native_restart_marker }}-{{ item }}" + loop: + - "{{ github_runner_cluster_native_agent_service }}" + - "{{ github_runner_cluster_native_server_service }}" + register: github_runner_cluster_native_pending + +- name: Record whether anything a running unit reads changed + vars: + github_runner_cluster_native_shared_changed: >- {{ github_runner_cluster_native_binary is changed or github_runner_cluster_native_helpers is changed - or github_runner_cluster_native_config is changed or github_runner_cluster_native_token is changed - or github_runner_cluster_native_vpn_auth is changed - or github_runner_cluster_native_unit is changed }} + or github_runner_cluster_native_vpn_auth is changed }} + ansible.builtin.set_fact: + github_runner_cluster_native_agent_changed: >- + {{ github_runner_cluster_native_shared_changed | bool + or github_runner_cluster_native_agent_config is changed + or github_runner_cluster_native_old_config is changed + or github_runner_cluster_native_units.results[0] is changed + or github_runner_cluster_native_pending.results[0].stat.exists }} + github_runner_cluster_native_server_changed: >- + {{ github_runner_cluster_native_shared_changed | bool + or github_runner_cluster_native_server_config is changed + or github_runner_cluster_native_units.results[1] is changed + or github_runner_cluster_native_pending.results[1].stat.exists }} + +- name: Mark the unit of the host's role as waiting for a restart when its files changed + # Only that unit: the other one, when the host switches to it, starts afresh on whatever its files hold. + become: true + ansible.builtin.copy: + content: "" + dest: "{{ github_runner_cluster_native_restart_marker }}-{{ github_runner_cluster_native_service }}" + mode: "0644" + when: >- + github_runner_cluster_native_server_changed | bool if github_runner_cluster_native_role == 'server' + else github_runner_cluster_native_agent_changed | bool diff --git a/roles/github_runner_cluster/tasks/native/demote.yml b/roles/github_runner_cluster/tasks/native/demote.yml new file mode 100644 index 00000000..85d805f6 --- /dev/null +++ b/roles/github_runner_cluster/tasks/native/demote.yml @@ -0,0 +1,154 @@ +--- +# Switches a native server to the agent unit, keeping its node name, node password and Tailscale device. Make-before-break: the server's etcd member is removed only while at least three other voters, every one Ready, remain to hold quorum without it (etcd calls a two-member cluster unsafe, https://etcd.io/docs/v3.6/op-guide/runtime-configuration/), so a cluster grows a voter before it loses one. In order: the node is emptied (drain.yml); the role file is switched to agent, so that neither Restart=always nor a reboot can start the server unit again (k3s marks a removed member's datastore with a tombstone and, started again, would rejoin it as a learner; Register in pkg/etcd/etcd.go at v1.34.1+k3s1); the member is removed through k3s's own controller, which acts on the etcd.k3s.cattle.io/remove annotation and records etcd.k3s.cattle.io/removed-node-name once etcd has removed it (pkg/etcd/member_controller.go); the server unit is stopped and disabled; the server's datastore is moved aside, because k3s reuses any datastore it finds and would otherwise start the old member's etcd again on a later promotion (https://docs.k3s.io/datastore/ha-embedded); the server role labels and k3s's etcd annotations are taken off the Node, which k3s does not do for a server that runs as an agent, so the cluster reads the host's live role right; and the agent unit is enabled and started, the node waited for and uncordoned. Needs service.yml's unit state. Under --check every read runs, the refusal included, and the switch is reported, not made. + +- name: Reach the cluster through another server, since this one is to stop + ansible.builtin.include_tasks: native/api.yml + vars: + github_runner_cluster_native_api_self: false + +- name: Fail if no other server of the inventory can carry the switch + ansible.builtin.fail: + msg: >- + Not demoting {{ inventory_hostname }}: the inventory names no other server to reach the cluster through once this server stops. + Nothing was changed. + when: github_runner_cluster_native_api_host | length == 0 + +- name: Read which nodes are etcd voters and whether each is Ready + become: "{{ github_runner_cluster_native_api_become }}" + ansible.builtin.command: + argv: >- + {{ github_runner_cluster_native_api_argv + ['get', 'nodes', '--request-timeout=' ~ github_runner_cluster_cluster_read_timeout ~ 's', '-o', + 'jsonpath={range .items[*]}{.metadata.name}{","}{.status.conditions[?(@.type=="Ready")].status}{","}{.metadata.labels.node-role\.kubernetes\.io/etcd}{"\n"}{end}'] }} + environment: "{{ github_runner_cluster_native_api_environment }}" + delegate_to: "{{ github_runner_cluster_native_api_host }}" + register: github_runner_cluster_native_voters + changed_when: false + failed_when: false + check_mode: false + timeout: "{{ github_runner_cluster_cluster_read_timeout | int }}" + +- name: Refuse unless at least three other voters, every one Ready, hold quorum without this server + vars: + github_runner_cluster_native_voter_problems: >- + {{ github_runner_cluster_native_voters | exadev.github_runner.cluster_native_voter_blockers(github_runner_cluster_effective_node_name, + github_runner_cluster_native_demotion_min_voters) }} + ansible.builtin.fail: + msg: >- + Not demoting {{ inventory_hostname }}: {{ github_runner_cluster_native_voter_problems | join('; ') }}. A demotion takes a vote away only + while enough Ready voters remain for etcd to keep a quorum that tolerates a failure, so add a server first, then demote this one. + Nothing was changed. + when: github_runner_cluster_native_voter_problems | length > 0 + +- name: Report the demotion a real run would make + ansible.builtin.debug: + msg: "demote {{ inventory_hostname }}: stop {{ github_runner_cluster_native_server_service }}, move its datastore aside, start {{ github_runner_cluster_native_agent_service }}" + changed_when: true + when: ansible_check_mode + +- name: Switch this server to the agent unit + when: not ansible_check_mode + block: + - name: Empty the node before its server stops + ansible.builtin.include_tasks: native/drain.yml + + - name: Record that the host is now an agent + become: true + ansible.builtin.copy: + content: agent + dest: "{{ github_runner_cluster_native_role_file }}" + mode: "0644" + + - name: Ask k3s to remove this server's etcd member + # Asking again is harmless: once the member is removed, the controller sees removed-node-name naming it and does nothing (sync in pkg/etcd/member_controller.go). + become: "{{ github_runner_cluster_native_api_become }}" + ansible.builtin.command: + argv: >- + {{ github_runner_cluster_native_api_argv + ['annotate', '--overwrite', 'node', github_runner_cluster_effective_node_name, + 'etcd.k3s.cattle.io/remove=true'] }} + environment: "{{ github_runner_cluster_native_api_environment }}" + delegate_to: "{{ github_runner_cluster_native_api_host }}" + changed_when: true + + - name: Wait for k3s to record that the member was removed + # The member controller writes removed-node-name and deletes node-name in the same update, once etcd has removed the member. + vars: + github_runner_cluster_native_poll: 5 + github_runner_cluster_native_member_read: "{{ github_runner_cluster_native_member.stdout | default('') | trim | split(',') }}" + become: "{{ github_runner_cluster_native_api_become }}" + ansible.builtin.command: + argv: >- + {{ github_runner_cluster_native_api_argv + ['get', 'node', github_runner_cluster_effective_node_name, + '--request-timeout=' ~ github_runner_cluster_cluster_read_timeout ~ 's', '-o', + 'jsonpath={.metadata.annotations.etcd\.k3s\.cattle\.io/removed-node-name},{.metadata.annotations.etcd\.k3s\.cattle\.io/node-name}'] }} + environment: "{{ github_runner_cluster_native_api_environment }}" + delegate_to: "{{ github_runner_cluster_native_api_host }}" + register: github_runner_cluster_native_member + until: + - github_runner_cluster_native_member.rc == 0 + - github_runner_cluster_native_member_read | length == 2 + - github_runner_cluster_native_member_read[0] | length > 0 + - github_runner_cluster_native_member_read[1] | length == 0 + retries: "{{ ((github_runner_cluster_native_ready_timeout | int) / github_runner_cluster_native_poll) | round(0, 'ceil') | int }}" + delay: "{{ github_runner_cluster_native_poll }}" + changed_when: false + + - name: Stop and disable the server unit + become: true + ansible.builtin.command: + argv: [systemctl, disable, --now, "{{ github_runner_cluster_native_server_service }}"] + changed_when: true + + - name: Move the server's datastore aside, keeping only the newest copy + # A copy, not a deletion: it holds the snapshots this server took (server/db/snapshots). One copy only, the one from this demotion, so repeated switches do not fill the disk. + become: true + ansible.builtin.shell: | + set -eu + rm -rf "$server/db.demoted" + mv "$server/db" "$server/db.demoted" + environment: + server: "{{ github_runner_cluster_native_data_dir }}/server" + changed_when: true + when: github_runner_cluster_native_has_datastore | bool + + - name: Take the server role labels and k3s's etcd annotations off the Node + # A trailing "-" removes a key; one that is not there is left alone. + become: "{{ github_runner_cluster_native_api_become }}" + ansible.builtin.command: + argv: "{{ github_runner_cluster_native_api_argv + [item, 'node', github_runner_cluster_effective_node_name] + github_runner_cluster_native_keys[item] }}" + vars: + github_runner_cluster_native_keys: + label: [node-role.kubernetes.io/control-plane-, node-role.kubernetes.io/etcd-, node-role.kubernetes.io/master-] + annotate: [etcd.k3s.cattle.io/remove-, etcd.k3s.cattle.io/removed-node-name-, etcd.k3s.cattle.io/node-name-, etcd.k3s.cattle.io/node-address-] + loop: [label, annotate] + environment: "{{ github_runner_cluster_native_api_environment }}" + delegate_to: "{{ github_runner_cluster_native_api_host }}" + changed_when: true + + - name: Read the time on the host, before the agent is started + ansible.builtin.command: + argv: [date, -u, "+%Y-%m-%dT%H:%M:%SZ"] + register: github_runner_cluster_native_started + changed_when: false + + - name: Enable and start the agent unit + become: true + ansible.builtin.command: + argv: [systemctl, enable, --now, "{{ github_runner_cluster_native_agent_service }}"] + changed_when: true + + - name: Clear the agent's waiting restart, since it has started on its files + become: true + ansible.builtin.file: + path: "{{ github_runner_cluster_native_restart_marker }}-{{ github_runner_cluster_native_agent_service }}" + state: absent + + - name: Wait for a server to see this node Ready on a heartbeat from the agent just started + ansible.builtin.include_tasks: native/server_ready.yml + + - name: Let the node run workloads again + become: "{{ github_runner_cluster_native_api_become }}" + ansible.builtin.command: + argv: "{{ github_runner_cluster_native_api_argv + ['uncordon', github_runner_cluster_effective_node_name] }}" + environment: "{{ github_runner_cluster_native_api_environment }}" + delegate_to: "{{ github_runner_cluster_native_api_host }}" + changed_when: true diff --git a/roles/github_runner_cluster/tasks/native/drain.yml b/roles/github_runner_cluster/tasks/native/drain.yml new file mode 100644 index 00000000..d0f84d04 --- /dev/null +++ b/roles/github_runner_cluster/tasks/native/drain.yml @@ -0,0 +1,57 @@ +--- +# Empties this host's node before its role switches: cordons it, waits for the ARC runner pods on it to finish (a planned switch never cuts a job), then drains the rest. Past github_runner_cluster_native_runner_wait_timeout, or when the drain fails, it uncordons the node and fails the run with the host's units as they were. Needs api.yml to have chosen another server's API, since this host's own k3s is about to stop. A node the cluster does not list (a first install) has nothing to empty. + +- name: Empty the node before its role switches + vars: + github_runner_cluster_native_poll: 5 + # The label ARC puts on every ephemeral runner pod, as plugins/filter/cluster_guard.py's RUNNER_POD_LABEL names it. + github_runner_cluster_native_runner_selector: actions-ephemeral-runner=True + block: + - name: Cordon the node + become: "{{ github_runner_cluster_native_api_become }}" + ansible.builtin.command: + argv: "{{ github_runner_cluster_native_api_argv + ['cordon', github_runner_cluster_effective_node_name] }}" + environment: "{{ github_runner_cluster_native_api_environment }}" + delegate_to: "{{ github_runner_cluster_native_api_host }}" + changed_when: true + + - name: Wait for the runner pods on the node to finish + become: "{{ github_runner_cluster_native_api_become }}" + ansible.builtin.command: + argv: >- + {{ github_runner_cluster_native_api_argv + ['get', 'pods', '--all-namespaces', '--selector', github_runner_cluster_native_runner_selector, + '--field-selector', 'spec.nodeName=' ~ github_runner_cluster_effective_node_name ~ ',status.phase!=Succeeded,status.phase!=Failed', + '--output', 'name', '--request-timeout=' ~ github_runner_cluster_cluster_read_timeout ~ 's'] }} + environment: "{{ github_runner_cluster_native_api_environment }}" + delegate_to: "{{ github_runner_cluster_native_api_host }}" + register: github_runner_cluster_native_runner_pods + until: github_runner_cluster_native_runner_pods.rc == 0 and github_runner_cluster_native_runner_pods.stdout | trim | length == 0 + retries: "{{ ((github_runner_cluster_native_runner_wait_timeout | int) / github_runner_cluster_native_poll) | round(0, 'ceil') | int }}" + delay: "{{ github_runner_cluster_native_poll }}" + changed_when: false + + - name: Drain the node + become: "{{ github_runner_cluster_native_api_become }}" + ansible.builtin.command: + argv: >- + {{ github_runner_cluster_native_api_argv + ['drain', github_runner_cluster_effective_node_name, '--ignore-daemonsets', '--delete-emptydir-data', + '--timeout=' ~ github_runner_cluster_native_runner_wait_timeout ~ 's'] }} + environment: "{{ github_runner_cluster_native_api_environment }}" + delegate_to: "{{ github_runner_cluster_native_api_host }}" + changed_when: true + rescue: + - name: Uncordon the node again, since the switch is abandoned + become: "{{ github_runner_cluster_native_api_become }}" + ansible.builtin.command: + argv: "{{ github_runner_cluster_native_api_argv + ['uncordon', github_runner_cluster_effective_node_name] }}" + environment: "{{ github_runner_cluster_native_api_environment }}" + delegate_to: "{{ github_runner_cluster_native_api_host }}" + changed_when: true + + - name: Fail, with the node as it was + ansible.builtin.fail: + msg: >- + Not switching {{ inventory_hostname }} from {{ github_runner_cluster_native_current_role }} to {{ github_runner_cluster_native_role }}: + its node could not be emptied ({{ ansible_failed_result.msg | default('') | trim }}{{ ' ' ~ (ansible_failed_result.stdout | default('') | trim) if ansible_failed_result.stdout | default('') | trim | length > 0 else '' }}). + Runner pods still running after about {{ github_runner_cluster_native_runner_wait_timeout }} seconds are left to finish. The node + is uncordoned and its units are as they were; run the role again once its jobs have finished. diff --git a/roles/github_runner_cluster/tasks/native/end_checks.yml b/roles/github_runner_cluster/tasks/native/end_checks.yml index c4e1c09d..7f5119bf 100644 --- a/roles/github_runner_cluster/tasks/native/end_checks.yml +++ b/roles/github_runner_cluster/tasks/native/end_checks.yml @@ -1,12 +1,14 @@ --- -# The checks that close a native node's run, so that a converge that leaves the node down or on the wrong tailnet fails here instead of reporting success. Needs service.yml to have started the service and recorded the time. A failure changes nothing on the host. +# The checks that close a native node's run, so that a converge that leaves the node down or on the wrong tailnet fails here instead of reporting success: the unit of the host's role is running, the other one is not, and the host's Tailscale client is on the cluster's tailnet; for an agent that was started or restarted, a server sees its node Ready on a heartbeat from the k3s just started (a server's turn has already waited for that, and for its etcd vote). Needs service.yml. A failure changes nothing on the host. -- name: Check the service and the node +- name: Check the units and the node vars: # Seconds between reads, as tasks/server_ready.yml polls. github_runner_cluster_native_poll: 5 + github_runner_cluster_native_other_service: >- + {{ github_runner_cluster_native_agent_service if github_runner_cluster_native_role == 'server' else github_runner_cluster_native_server_service }} block: - - name: Read whether the service is running + - name: Read whether the unit of the host's role is running ansible.builtin.command: argv: [systemctl, is-active, "{{ github_runner_cluster_native_service }}"] register: github_runner_cluster_native_running @@ -16,17 +18,33 @@ changed_when: false failed_when: false - - name: Fail unless the service is running + - name: Fail unless the unit of the host's role is running ansible.builtin.fail: msg: "{{ github_runner_cluster_native_service }} is {{ github_runner_cluster_native_running.stdout | trim }} on {{ inventory_hostname }}." when: github_runner_cluster_native_running.rc != 0 + - name: Read whether the other unit is running + ansible.builtin.command: + argv: [systemctl, is-active, "{{ github_runner_cluster_native_other_service }}"] + register: github_runner_cluster_native_other_running + changed_when: false + failed_when: false + + - name: Fail if the other unit is running too + # Both units' conditions read the one role file, so this means something started a unit around them. + ansible.builtin.fail: + msg: >- + {{ github_runner_cluster_native_other_service }} is running on {{ inventory_hostname }} beside {{ github_runner_cluster_native_service }}, + though the host's role file names {{ github_runner_cluster_native_role }}. Stop it with `systemctl disable --now + {{ github_runner_cluster_native_other_service }}`. + when: github_runner_cluster_native_other_running.rc == 0 + - name: Read which tailnet the host's Tailscale client is logged in to now become: true ansible.builtin.command: argv: ["{{ github_runner_cluster_native_tailscale_cli }}", status, --json, --peers=false] register: github_runner_cluster_native_tailscale_after - # k3s's own `tailscale up` runs as the service starts, so the client reaches the cluster's tailnet a moment after the service is active. + # k3s's own `tailscale up` runs as the unit starts, so the client reaches the cluster's tailnet a moment after the unit is active. until: >- github_runner_cluster_native_tailscale_after.rc == 0 and (github_runner_cluster_native_tailscale_after.stdout | exadev.github_runner.cluster_native_tailnet_login(github_runner_cluster_tailnet_domain)).state == 'cluster' @@ -49,40 +67,30 @@ github_runner_cluster_native_tailscale_after.rc != 0 or (github_runner_cluster_native_tailscale_after.stdout | exadev.github_runner.cluster_native_tailnet_login(github_runner_cluster_tailnet_domain)).state != 'cluster' + - name: Reach the cluster through a server of the inventory + ansible.builtin.include_tasks: native/api.yml + vars: + github_runner_cluster_native_api_self: false + when: + - github_runner_cluster_native_role == 'agent' + - github_runner_cluster_native_agent_started | bool + - name: Say that no server of this inventory can confirm the node ansible.builtin.debug: msg: >- {{ github_runner_cluster_group }} bootstraps no cluster of its own, so no server of this inventory can confirm that {{ github_runner_cluster_effective_node_name }} is Ready; check it from the cluster that owns the servers. - when: github_runner_cluster_topology.bootstrap | length == 0 + when: + - github_runner_cluster_native_role == 'agent' + - github_runner_cluster_native_agent_started | bool + - github_runner_cluster_native_api_host | length == 0 - name: Wait for a server to see this node Ready on a heartbeat from the k3s just started - # Through the bootstrap server's container on its own host with its own pinned Docker context, as the version read does. Ready alone is not enough: the API keeps the old k3s's Ready condition for a while, so the node's lease must have been renewed after the time recorded before the start (the same rule as a recreated server's, tasks/server_ready.yml). - ansible.builtin.shell: | - set -eu - ready=$(docker exec "$container" kubectl get node "$node" --request-timeout=30s -o 'jsonpath={.status.conditions[?(@.type=="Ready")].status}') - renewed=$(docker exec "$container" kubectl get lease --namespace kube-node-lease "$node" --request-timeout=30s -o 'jsonpath={.spec.renewTime}') - echo "${ready:-unknown} ${renewed:-none}" - environment: >- - {{ ({'DOCKER_CONTEXT': github_runner_cluster_native_bootstrap_context} if github_runner_cluster_native_bootstrap_context | length > 0 else {}) - | combine({'container': github_runner_cluster_native_bootstrap_container, 'node': github_runner_cluster_effective_node_name}) }} - # delegate_to is templated before the condition below is read, and may not be empty: a cluster outside the inventory has no bootstrap host, and then the task is skipped on this host. - delegate_to: "{{ github_runner_cluster_topology.bootstrap | default(inventory_hostname, true) }}" - vars: - github_runner_cluster_native_bootstrap_container: "{{ hostvars[github_runner_cluster_topology.bootstrap].github_runner_cluster_container_name | default('github-runner-k3s') }}" - github_runner_cluster_native_bootstrap_context: "{{ hostvars[github_runner_cluster_topology.bootstrap].github_runner_cluster_docker_context | default('') | trim }}" - github_runner_cluster_native_ready_read: "{{ github_runner_cluster_native_ready.stdout | default('') | trim | split(' ') }}" - register: github_runner_cluster_native_ready - until: - - github_runner_cluster_native_ready.rc == 0 - - github_runner_cluster_native_ready_read | length == 2 - - github_runner_cluster_native_ready_read[0] == 'True' - - github_runner_cluster_native_ready_read[1] is match('[0-9]{4}-[0-9]{2}-[0-9]{2}T') - - github_runner_cluster_native_ready_read[1][:19] > (github_runner_cluster_native_started.stdout | trim)[:19] - retries: "{{ ((github_runner_cluster_native_ready_timeout | int) / (2 * github_runner_cluster_native_poll)) | round(0, 'ceil') | int }}" - delay: "{{ github_runner_cluster_native_poll }}" - changed_when: false - when: github_runner_cluster_topology.bootstrap | length > 0 + ansible.builtin.include_tasks: native/server_ready.yml + when: + - github_runner_cluster_native_role == 'agent' + - github_runner_cluster_native_agent_started | bool + - github_runner_cluster_native_api_host | length > 0 rescue: - name: Fail with where to look ansible.builtin.fail: diff --git a/roles/github_runner_cluster/tasks/native/main.yml b/roles/github_runner_cluster/tasks/native/main.yml index de2a7350..bb1ef1b5 100644 --- a/roles/github_runner_cluster/tasks/native/main.yml +++ b/roles/github_runner_cluster/tasks/native/main.yml @@ -1,5 +1,5 @@ --- -# Brings this host up as a native k3s node: the agent runs directly on the host as the systemd service k3s-agent, with no Docker or Compose layer, and the host's own Tailscale client is the node's device on the cluster's tailnet (k3s's --vpn-auth runs `tailscale up` against it). Selected by github_runner_cluster_node_mode: native, from tasks/main.yml, which ends the role after this. The topology, the versions, the API port, the token and the mesh are worked out by the same task files as the Docker path, so a native agent joins the same cluster with the same secrets, node name rules and API port. Safe to rerun: each step converges, and the service is restarted only when its binary, configuration or secrets changed. Every task that changes the host uses become (sudo) on its own; under --check nothing is changed and each change a real run would make is reported. +# Brings this host up as a native k3s node: k3s runs directly on the host as the systemd service k3s-agent (an agent) or k3s (a server), with no Docker or Compose layer, and the host's own Tailscale client is the node's device on the cluster's tailnet (k3s's --vpn-auth runs `tailscale up` against it). Selected by github_runner_cluster_node_mode: native, from tasks/main.yml, which ends the role after this. The topology, the versions, the API port, the token and the mesh are worked out by the same task files as the Docker path, so a native node joins the same cluster with the same secrets, node name rules and API port, and a native server is configured as every Docker server is. Safe to rerun: each step converges, a unit is restarted only when its binary, configuration or secrets changed, and the host switches between its server and agent units only when its role changed (role.yml). Every task that changes the host uses become (sudo) on its own; under --check nothing is changed and each change a real run would make is reported. - name: Read the platform the native node needs # Only the subsets that hold the kernel, architecture, distribution and init system, and only when the caller did not gather them: a play with gather_facts: false leaves ansible_facts without them. Read-only, so it runs under --check too. @@ -54,6 +54,11 @@ host's own network with the host's own Tailscale client, and has no Docker network to choose. Remove github_runner_cluster_network_mode (or set it to bridge) for this host. bad: "{{ github_runner_cluster_network_mode == 'host' }}" + - msg: >- + github_runner_cluster_regrade is true for {{ inventory_hostname }}, which sets github_runner_cluster_node_mode: native. Memory-pressure + regrading steps a server down through its actuator, and the only one there is drives a Docker server's container; a native server has + none yet. Remove github_runner_cluster_regrade for this host. + bad: "{{ github_runner_cluster_regrade | bool }}" loop_control: label: "{{ item.msg | split(' ') | first }}" when: item.bad | bool @@ -61,6 +66,9 @@ - name: Decide this host's part in the cluster ansible.builtin.include_tasks: topology.yml +- name: Check etcd's heartbeat interval and election timeout, which a native server passes to its etcd as every server does + ansible.builtin.include_tasks: etcd_timing.yml + - name: Check the versions the node is pinned to ansible.builtin.include_tasks: versions.yml @@ -77,14 +85,17 @@ - name: Check the k3s version against what this node and the cluster run now ansible.builtin.include_tasks: native/version_check.yml +- name: Decide which role the host runs, and whether a server bootstraps a new cluster + ansible.builtin.include_tasks: native/role.yml + - name: Install the pinned Tailscale client and check the tailnet it is logged in to ansible.builtin.include_tasks: native/tailscale.yml - name: Install the pinned k3s binary ansible.builtin.include_tasks: native/k3s_binary.yml -- name: Write the node's helpers, configuration, secrets and unit +- name: Write the node's helpers, configurations, secrets and units ansible.builtin.include_tasks: native/config.yml -- name: Start the node and check it joined +- name: Run the unit of the host's role and check the node joined ansible.builtin.include_tasks: native/service.yml diff --git a/roles/github_runner_cluster/tasks/native/role.yml b/roles/github_runner_cluster/tasks/native/role.yml new file mode 100644 index 00000000..9b18a9d9 --- /dev/null +++ b/roles/github_runner_cluster/tasks/native/role.yml @@ -0,0 +1,102 @@ +--- +# Decides which role this native host runs, server or agent, and, for a server, whether it bootstraps a new cluster or joins one. Publishes github_runner_cluster_native_current_role (what the host runs now: its role file, or agent for a host set up before the file existed, or empty on a first install), github_runner_cluster_native_role (what it is to run after this run; plugins/filter/cluster_native.py's cluster_native_role says how the two are decided), github_runner_cluster_native_stage_server (whether the server's files are written: a server, or a host marked eligible, staged as a standby) and github_runner_cluster_native_cluster_init. Needs topology.yml. Every task only reads, and runs under --check too. + +- name: Read the role file, the agent unit and the datastores on the host + # The server's directory under k3s's data directory is root's alone (k3s makes it 0700), so the reads take become. + become: true + ansible.builtin.stat: + path: "{{ item }}" + loop: + - "{{ github_runner_cluster_native_role_file }}" + - "{{ github_runner_cluster_native_unit_dir }}/{{ github_runner_cluster_native_agent_service }}.service" + - "{{ github_runner_cluster_native_data_dir }}/server/db" + - "{{ github_runner_cluster_native_data_dir }}/server/db.demoted" + register: github_runner_cluster_native_role_paths + +- name: Read the role the host's role file names + become: true + ansible.builtin.slurp: + src: "{{ github_runner_cluster_native_role_file }}" + register: github_runner_cluster_native_role_recorded + when: github_runner_cluster_native_role_paths.results[0].stat.exists + +- name: Read which of the two units is running + ansible.builtin.command: + argv: [systemctl, is-active, --quiet, "{{ item }}"] + loop: + - "{{ github_runner_cluster_native_server_service }}" + - "{{ github_runner_cluster_native_agent_service }}" + register: github_runner_cluster_native_role_units + changed_when: false + failed_when: false + check_mode: false + +- name: Work out the role the host runs now and the role it is to run + vars: + github_runner_cluster_native_roles: >- + {{ (github_runner_cluster_native_role_recorded.content | b64decode if github_runner_cluster_native_role_recorded is not skipped else '') + | exadev.github_runner.cluster_native_role(github_runner_cluster_native_role_paths.results[1].stat.exists, + github_runner_cluster_topology.hosts[inventory_hostname].eligible, + github_runner_cluster_effective_node_role, + (['server'] if github_runner_cluster_native_role_units.results[0].rc == 0 else []) + + (['agent'] if github_runner_cluster_native_role_units.results[1].rc == 0 else [])) }} + ansible.builtin.set_fact: + github_runner_cluster_native_current_role: "{{ github_runner_cluster_native_roles.current }}" + github_runner_cluster_native_role: "{{ github_runner_cluster_native_roles.desired }}" + github_runner_cluster_native_stage_server: "{{ github_runner_cluster_native_roles.desired == 'server' or github_runner_cluster_topology.hosts[inventory_hostname].eligible }}" + github_runner_cluster_native_has_datastore: "{{ github_runner_cluster_native_role_paths.results[2].stat.exists }}" + github_runner_cluster_native_was_member: >- + {{ github_runner_cluster_native_role_paths.results[2].stat.exists or github_runner_cluster_native_role_paths.results[3].stat.exists }} + +- name: Read whether another server of the inventory runs a cluster already + # Only for the bootstrap host about to run as a server that has never been a member: such a host has nothing on disk to say a cluster exists, so the other servers are asked. A host that has a datastore, or one a demotion moved aside, has been a member of a cluster and always joins it. + ansible.builtin.include_tasks: cluster_nodes_read.yml + vars: + github_runner_cluster_running_read_always: true + when: + - github_runner_cluster_native_role == 'server' + - github_runner_cluster_effective_bootstrap | bool + - not github_runner_cluster_native_was_member | bool + +- name: Work out whether this server bootstraps a new cluster + vars: + github_runner_cluster_native_bootstrap_read: >- + {{ github_runner_cluster_running_reads.results | exadev.github_runner.cluster_native_cluster_init(inventory_hostname) }} + ansible.builtin.set_fact: + github_runner_cluster_native_cluster_init: >- + {{ github_runner_cluster_native_role == 'server' and github_runner_cluster_effective_bootstrap | bool + and not github_runner_cluster_native_was_member | bool and github_runner_cluster_native_bootstrap_read.cluster_init }} + github_runner_cluster_native_bootstrap_unread: >- + {{ github_runner_cluster_native_bootstrap_read.unread + if github_runner_cluster_native_role == 'server' and github_runner_cluster_effective_bootstrap | bool and not github_runner_cluster_native_was_member | bool + else [] }} + +- name: Join the first server that runs the cluster, for a bootstrap host that finds one + # The bootstrap host's own join URL is empty, and the first other server of the inventory may be one that is down; one that answered is up. + ansible.builtin.set_fact: + github_runner_cluster_native_join_url: >- + https://{{ github_runner_cluster_topology.hosts[(github_runner_cluster_running_reads.results | exadev.github_runner.cluster_native_cluster_init(inventory_hostname)).running | first].address }}:{{ github_runner_cluster_api_port }} + when: + - github_runner_cluster_native_role == 'server' + - github_runner_cluster_effective_bootstrap | bool + - not github_runner_cluster_native_was_member | bool + - (github_runner_cluster_running_reads.results | exadev.github_runner.cluster_native_cluster_init(inventory_hostname)).running | length > 0 + +- name: Fail if whether a cluster already exists cannot be told + # Starting a new cluster beside one that exists would split the fleet in two, so a server that could not be read counts against it, as it does in the version check. + ansible.builtin.fail: + msg: >- + {{ inventory_hostname }} is the bootstrap host and has no datastore, so it would start a new cluster (cluster-init) unless another + server already runs one, and that cannot be told: {{ github_runner_cluster_native_bootstrap_unread | join('; ') }}. Bring the other + servers' hosts back within reach, or, if no cluster exists, stop their servers. Nothing was changed. + when: github_runner_cluster_native_bootstrap_unread | length > 0 + +- name: Fail if this host would join no server + ansible.builtin.fail: + msg: >- + {{ inventory_hostname }} is to run as a {{ github_runner_cluster_native_role }} with no server to join: it does not bootstrap a new + cluster, the inventory names no other server, and github_runner_cluster_server_url is not set. Nothing was changed. + when: + - github_runner_cluster_native_join_url | length == 0 + - not github_runner_cluster_native_cluster_init | bool + - not (github_runner_cluster_native_role == 'server' and github_runner_cluster_native_has_datastore | bool) diff --git a/roles/github_runner_cluster/tasks/native/server_ready.yml b/roles/github_runner_cluster/tasks/native/server_ready.yml new file mode 100644 index 00000000..8af1907f --- /dev/null +++ b/roles/github_runner_cluster/tasks/native/server_ready.yml @@ -0,0 +1,30 @@ +--- +# Waits, through the API api.yml chose, until this host's node reports Ready on a heartbeat later than github_runner_cluster_native_started and, on a server, k3s reports it a voting member of etcd, for about github_runner_cluster_native_ready_timeout seconds. Ready alone is not enough: the API keeps a stopped kubelet's Ready condition until the node controller's grace period runs out, so the node's lease (kube-node-lease/), which the running kubelet renews, must have been renewed after the start. A voter, because k3s adds a new server as a learner, which does not count towards quorum until the etcd leader promotes it: k3s sets the EtcdIsVoter condition True on a voting member's Node (setEtcdStatusCondition in pkg/etcd/etcd.go at v1.34.1+k3s1), and an agent has no such condition. The run fails when the node does not come back, before the next server's turn. + +- name: Wait for this node to be Ready on a heartbeat from the k3s just started{{ ' and a voting member of etcd' if github_runner_cluster_native_role == 'server' else '' }} + vars: + github_runner_cluster_native_poll: 5 + github_runner_cluster_native_ready_read: "{{ github_runner_cluster_native_ready.stdout | default('') | trim | split(',') }}" + become: "{{ github_runner_cluster_native_api_become }}" + # The kubectl command api.yml chose is written into the script word by word, each word quoted, so a command of several words (docker exec kubectl) runs as it is. + ansible.builtin.shell: | + set -eu + node=$({{ github_runner_cluster_native_api_argv | map('quote') | join(' ') }} get node "$GITHUB_RUNNER_NODE" --request-timeout="${GITHUB_RUNNER_TIMEOUT}s" \ + -o 'jsonpath={.status.conditions[?(@.type=="Ready")].status},{.status.conditions[?(@.type=="EtcdIsVoter")].status}') + renewed=$({{ github_runner_cluster_native_api_argv | map('quote') | join(' ') }} get lease --namespace kube-node-lease "$GITHUB_RUNNER_NODE" --request-timeout="${GITHUB_RUNNER_TIMEOUT}s" -o 'jsonpath={.spec.renewTime}') + echo "${node},${renewed}" + environment: >- + {{ github_runner_cluster_native_api_environment + | combine({'GITHUB_RUNNER_NODE': github_runner_cluster_effective_node_name, 'GITHUB_RUNNER_TIMEOUT': github_runner_cluster_cluster_read_timeout | string}) }} + delegate_to: "{{ github_runner_cluster_native_api_host }}" + register: github_runner_cluster_native_ready + until: + - github_runner_cluster_native_ready.rc == 0 + - github_runner_cluster_native_ready_read | length == 3 + - github_runner_cluster_native_ready_read[0] == 'True' + - github_runner_cluster_native_role != 'server' or github_runner_cluster_native_ready_read[1] == 'True' + - github_runner_cluster_native_ready_read[2] is match('[0-9]{4}-[0-9]{2}-[0-9]{2}T') + - github_runner_cluster_native_ready_read[2][:19] > (github_runner_cluster_native_started.stdout | trim)[:19] + retries: "{{ ((github_runner_cluster_native_ready_timeout | int) / (2 * github_runner_cluster_native_poll)) | round(0, 'ceil') | int }}" + delay: "{{ github_runner_cluster_native_poll }}" + changed_when: false diff --git a/roles/github_runner_cluster/tasks/native/server_turn.yml b/roles/github_runner_cluster/tasks/native/server_turn.yml new file mode 100644 index 00000000..23d8f3d7 --- /dev/null +++ b/roles/github_runner_cluster/tasks/native/server_turn.yml @@ -0,0 +1,131 @@ +--- +# One native server's turn (see service.yml, which gives each native server of the play its turn, one at a time, as tasks/start_servers.yml does for the Docker servers): works out what the server unit needs, refuses when that would take quorum away or add a member beside an unhealthy one, does it, and waits for the node to be Ready on a heartbeat from the k3s just started and a voting member of etcd. What a turn does: +# +# - promote: the host runs the agent unit and is to be a server. Its node is emptied (drain.yml), the role file is switched to server, the agent unit is stopped and disabled and the server unit enabled and started. k3s joins it as an etcd learner (MemberAddAsLearner in join, pkg/etcd/etcd.go), which the etcd leader promotes once it has caught up; a learner does not count towards quorum, so the add costs none. Same node name, node password (/etc/rancher/node/password, which neither unit's k3s is told to change) and Tailscale device, so the cluster's node-password Secret stays valid. +# - first start: nothing runs on the host yet; the server unit is enabled and started, bootstrapping a new cluster or joining one as 10-join.yaml says. +# - restart: the server runs and something it reads changed. Its etcd member is away while k3s restarts. +# - start: the server unit is installed and stopped; it is started, and rejoins with its own datastore. +# +# Every one but a start, and a first start that bootstraps, goes ahead only while every other etcd voter the cluster lists is Ready, read from each node's etcd role label (plugins/filter/cluster_native.py's cluster_native_voter_blockers), never from the inventory, since a host's role can change at run time. Under --check the guard still reads and refuses, and the change is reported, not made. + +- name: Work out what this server's unit needs + ansible.builtin.set_fact: + github_runner_cluster_native_turn: >- + {{ 'promote' if github_runner_cluster_native_current_role == 'agent' + else ('first start' if github_runner_cluster_native_current_role | length == 0 + else ('restart' if github_runner_cluster_native_server_active | bool and github_runner_cluster_native_server_changed | bool + else ('start' if not github_runner_cluster_native_server_active | bool else ''))) }} + +- name: Reach the cluster through this server when it runs, and through another server otherwise + ansible.builtin.include_tasks: native/api.yml + vars: + github_runner_cluster_native_api_self: "{{ github_runner_cluster_native_server_active | bool }}" + +- name: Read which nodes are etcd voters and whether each is Ready + # A voter is a node with k3s's etcd role label, which k3s puts on every server whose etcd is a member. Comma-separated, since either of the last two fields can be empty. + become: "{{ github_runner_cluster_native_api_become }}" + ansible.builtin.command: + argv: >- + {{ github_runner_cluster_native_api_argv + ['get', 'nodes', '--request-timeout=' ~ github_runner_cluster_cluster_read_timeout ~ 's', '-o', + 'jsonpath={range .items[*]}{.metadata.name}{","}{.status.conditions[?(@.type=="Ready")].status}{","}{.metadata.labels.node-role\.kubernetes\.io/etcd}{"\n"}{end}'] }} + environment: "{{ github_runner_cluster_native_api_environment }}" + # Ansible resolves delegate_to before it evaluates when, so a task that is skipped because there is no other server still needs a host here. + delegate_to: "{{ github_runner_cluster_native_api_host if github_runner_cluster_native_api_host | length > 0 else inventory_hostname }}" + register: github_runner_cluster_native_voters + changed_when: false + failed_when: false + check_mode: false + timeout: "{{ github_runner_cluster_cluster_read_timeout | int }}" + when: + - github_runner_cluster_native_turn in ['promote', 'restart'] or (github_runner_cluster_native_turn == 'first start' and not github_runner_cluster_native_cluster_init | bool) + - github_runner_cluster_native_api_host | length > 0 + +- name: Refuse while another etcd voter is not Ready + vars: + github_runner_cluster_native_voter_problems: >- + {{ github_runner_cluster_native_voters | exadev.github_runner.cluster_native_voter_blockers(github_runner_cluster_effective_node_name) }} + ansible.builtin.fail: + msg: >- + Not going ahead with the {{ github_runner_cluster_native_turn }} of {{ github_runner_cluster_native_server_service }} on {{ inventory_hostname }}: + {{ github_runner_cluster_native_voter_problems | join('; ') }}. A native server is restarted, started for the first time beside a + cluster, or promoted from an agent only while every other voter is Ready, so that the cluster keeps its quorum and a learner is only + added beside a healthy membership. Bring the other voters back and run again. Nothing was changed. + when: + - github_runner_cluster_native_voters is not skipped + - github_runner_cluster_native_voter_problems | length > 0 + +- name: Report what a real run would do to this server + ansible.builtin.debug: + msg: "{{ github_runner_cluster_native_turn }} {{ github_runner_cluster_native_server_service }} on {{ inventory_hostname }}" + changed_when: true + when: + - ansible_check_mode + - github_runner_cluster_native_turn | length > 0 + +- name: Switch, start or restart this server + when: + - not ansible_check_mode + - github_runner_cluster_native_turn | length > 0 + block: + - name: Empty the node before it stops running the agent + ansible.builtin.include_tasks: native/drain.yml + when: github_runner_cluster_native_turn == 'promote' + + - name: Record that the host is now a server + # Before the agent unit stops, so that Restart=always cannot start it again: from here on only the server unit's condition holds. + become: true + ansible.builtin.copy: + content: server + dest: "{{ github_runner_cluster_native_role_file }}" + mode: "0644" + when: github_runner_cluster_native_turn == 'promote' + + - name: Stop and disable the agent unit + become: true + ansible.builtin.command: + argv: [systemctl, disable, --now, "{{ github_runner_cluster_native_agent_service }}"] + changed_when: true + when: github_runner_cluster_native_turn == 'promote' + + - name: Read the time on the host, before the server is started or restarted + # The node's Ready condition outlives a stopped k3s for a while, so the wait below is for a heartbeat later than this. The host's own clock, which the kubelet's heartbeat is stamped with. + ansible.builtin.command: + argv: [date, -u, "+%Y-%m-%dT%H:%M:%SZ"] + register: github_runner_cluster_native_started + changed_when: false + + - name: Enable the server unit + become: true + ansible.builtin.command: + argv: [systemctl, enable, "{{ github_runner_cluster_native_server_service }}"] + changed_when: true + when: not github_runner_cluster_native_server_enabled | bool + + - name: Start or restart the server unit + # A restart leaves the pods running: the unit's KillMode=process leaves their containerd shims alone. + become: true + ansible.builtin.command: + argv: [systemctl, "{{ 'restart' if github_runner_cluster_native_turn == 'restart' else 'start' }}", "{{ github_runner_cluster_native_server_service }}"] + changed_when: true + + - name: Clear the server's waiting restart, since it has started on its files + become: true + ansible.builtin.file: + path: "{{ github_runner_cluster_native_restart_marker }}-{{ github_runner_cluster_native_server_service }}" + state: absent + + - name: Reach the cluster through this server from now on + ansible.builtin.include_tasks: native/api.yml + vars: + github_runner_cluster_native_api_self: true + + - name: Wait for this server's node to be Ready on a new heartbeat and an etcd voter + # Ready alone is not enough, as for a recreated Docker server (tasks/server_ready.yml): the node's lease must have been renewed after the start. A voter, because k3s adds a new server as a learner and the turn is over only once etcd counts it: k3s's EtcdIsVoter condition is True on a voting member's Node (setEtcdStatusCondition in pkg/etcd/etcd.go). + ansible.builtin.include_tasks: native/server_ready.yml + + - name: Let the node run workloads again + become: true + ansible.builtin.command: + argv: "{{ github_runner_cluster_native_api_argv + ['uncordon', github_runner_cluster_effective_node_name] }}" + changed_when: true + when: github_runner_cluster_native_turn == 'promote' diff --git a/roles/github_runner_cluster/tasks/native/service.yml b/roles/github_runner_cluster/tasks/native/service.yml index 258af919..cb5372df 100644 --- a/roles/github_runner_cluster/tasks/native/service.yml +++ b/roles/github_runner_cluster/tasks/native/service.yml @@ -1,76 +1,104 @@ --- -# Enables and starts the k3s-agent service, restarts it when anything it reads changed, and then checks the node: the service is running, the host's Tailscale client is logged in to the cluster's tailnet, and a server of the cluster sees the node Ready on a heartbeat from the k3s that was just started. Nothing here happens while github_runner_cluster_native_start is false, which leaves the service as it is: a first install stays disabled and stopped. +# Runs the unit of the host's role and checks the node. Servers go first, one at a time (server_turn.yml, given each native server of the play in turn as tasks/start_servers.yml does for the Docker servers): a promotion from the agent, a first start, a restart when something it reads changed, or a start of a stopped server. Then a host that is to be an agent: a server is demoted (demote.yml), and an agent is enabled and started, or restarted when something it reads changed. Then the end checks. Nothing here happens while github_runner_cluster_native_start is false, which leaves the units as they are: a first install stays disabled and stopped, and a host whose role is to change keeps running its old unit. Needs role.yml and config.yml. -- name: Reload systemd when the unit changed +- name: Reload systemd when a unit changed become: true ansible.builtin.command: argv: [systemctl, daemon-reload] changed_when: true when: - - github_runner_cluster_native_unit is changed + - github_runner_cluster_native_units is changed - not ansible_check_mode -- name: Start the node and check it +- name: Run the unit of the host's role and check the node when: github_runner_cluster_native_start | bool block: - - name: Read whether the service is enabled and running + - name: Read whether each unit is enabled and running ansible.builtin.command: - argv: [systemctl, "{{ item }}", "{{ github_runner_cluster_native_service }}"] - loop: [is-enabled, is-active] - register: github_runner_cluster_native_service_state + argv: [systemctl, "{{ item[1] }}", "{{ item[0] }}"] + loop: "{{ [github_runner_cluster_native_agent_service, github_runner_cluster_native_server_service] | product(['is-enabled', 'is-active']) | list }}" + register: github_runner_cluster_native_unit_state changed_when: false failed_when: false check_mode: false - - name: Work out what the service needs + - name: Work out what the units need ansible.builtin.set_fact: - github_runner_cluster_native_enabled: "{{ github_runner_cluster_native_service_state.results[0].rc == 0 }}" - github_runner_cluster_native_active: "{{ github_runner_cluster_native_service_state.results[1].rc == 0 }}" + github_runner_cluster_native_agent_enabled: "{{ github_runner_cluster_native_unit_state.results[0].rc == 0 }}" + github_runner_cluster_native_agent_active: "{{ github_runner_cluster_native_unit_state.results[1].rc == 0 }}" + github_runner_cluster_native_server_enabled: "{{ github_runner_cluster_native_unit_state.results[2].rc == 0 }}" + github_runner_cluster_native_server_active: "{{ github_runner_cluster_native_unit_state.results[3].rc == 0 }}" + github_runner_cluster_native_agent_started: false - - name: Read the time on the host, before the service is started or restarted - # The node's Ready condition outlives a stopped k3s for a while, so the check below waits for a heartbeat later than this. The host's own clock, which the kubelet's heartbeat is stamped with. - ansible.builtin.command: - argv: [date, -u, "+%Y-%m-%dT%H:%M:%SZ"] - register: github_runner_cluster_native_started - changed_when: false - check_mode: false - - - name: Enable the service - become: true - ansible.builtin.command: - argv: [systemctl, enable, "{{ github_runner_cluster_native_service }}"] - changed_when: true + - name: Give each native server its turn + # One at a time and in no fixed order, under Ansible's linear strategy (the default), where the tasks a host includes dynamically are its own copies: one turn's tasks run for that host alone while the others wait, so no two servers are ever away together. Hosts outside the play (--limit) take no turn. + ansible.builtin.include_tasks: native/server_turn.yml + loop: "{{ ansible_play_hosts_all | select('in', github_runner_cluster_topology.hosts | dict2items | selectattr('value.native') | map(attribute='key') | list) | list }}" + loop_control: + loop_var: github_runner_cluster_native_turn_host when: - - not github_runner_cluster_native_enabled | bool - - not ansible_check_mode + - github_runner_cluster_native_turn_host == inventory_hostname + - github_runner_cluster_native_role == 'server' - - name: Start the service - become: true - ansible.builtin.command: - argv: [systemctl, start, "{{ github_runner_cluster_native_service }}"] - changed_when: true + - name: Demote this server to an agent + ansible.builtin.include_tasks: native/demote.yml when: - - not github_runner_cluster_native_active | bool - - not ansible_check_mode + - github_runner_cluster_native_role == 'agent' + - github_runner_cluster_native_current_role == 'server' - - name: Restart the service, since its binary, configuration or secrets changed - # Pods keep running: the unit's KillMode=process leaves their containerd shims alone. - become: true - ansible.builtin.command: - argv: [systemctl, restart, "{{ github_runner_cluster_native_service }}"] - changed_when: true + - name: Run the agent unit when: - - github_runner_cluster_native_active | bool - - github_runner_cluster_native_changed | bool - - not ansible_check_mode + - github_runner_cluster_native_role == 'agent' + - github_runner_cluster_native_current_role != 'server' + block: + - name: Read the time on the host, before the agent is started or restarted + # The node's Ready condition outlives a stopped k3s for a while, so the end check waits for a heartbeat later than this. The host's own clock, which the kubelet's heartbeat is stamped with. + ansible.builtin.command: + argv: [date, -u, "+%Y-%m-%dT%H:%M:%SZ"] + register: github_runner_cluster_native_started + changed_when: false + check_mode: false + + - name: Enable the agent unit + become: true + ansible.builtin.command: + argv: [systemctl, enable, "{{ github_runner_cluster_native_agent_service }}"] + changed_when: true + when: + - not github_runner_cluster_native_agent_enabled | bool + - not ansible_check_mode + + - name: Start the agent unit, or restart it since something it reads changed + # Pods keep running across a restart: the unit's KillMode=process leaves their containerd shims alone. + become: true + ansible.builtin.command: + argv: [systemctl, "{{ 'restart' if github_runner_cluster_native_agent_active | bool else 'start' }}", "{{ github_runner_cluster_native_agent_service }}"] + changed_when: true + register: github_runner_cluster_native_agent_start + when: + - not github_runner_cluster_native_agent_active | bool or github_runner_cluster_native_agent_changed | bool + - not ansible_check_mode + + - name: Clear the agent's waiting restart, since it has started on its files + become: true + ansible.builtin.file: + path: "{{ github_runner_cluster_native_restart_marker }}-{{ github_runner_cluster_native_agent_service }}" + state: absent + when: github_runner_cluster_native_agent_start is not skipped + + - name: Record whether the agent was started + ansible.builtin.set_fact: + github_runner_cluster_native_agent_started: "{{ github_runner_cluster_native_agent_start is not skipped }}" - - name: Report what a real run would do to the service - ansible.builtin.debug: - msg: >- - {{ 'enable ' if not github_runner_cluster_native_enabled | bool else '' }}{{ 'start' if not github_runner_cluster_native_active | bool - else ('restart' if github_runner_cluster_native_changed | bool else 'leave running') }} {{ github_runner_cluster_native_service }} - changed_when: not github_runner_cluster_native_enabled | bool or not github_runner_cluster_native_active | bool or github_runner_cluster_native_changed | bool - when: ansible_check_mode + - name: Report what a real run would do to the agent + ansible.builtin.debug: + msg: >- + {{ 'enable ' if not github_runner_cluster_native_agent_enabled | bool else '' }}{{ 'start' if not github_runner_cluster_native_agent_active | bool + else ('restart' if github_runner_cluster_native_agent_changed | bool else 'leave running') }} {{ github_runner_cluster_native_agent_service }} + changed_when: >- + not github_runner_cluster_native_agent_enabled | bool or not github_runner_cluster_native_agent_active | bool + or github_runner_cluster_native_agent_changed | bool + when: ansible_check_mode - name: Check the node ansible.builtin.include_tasks: native/end_checks.yml diff --git a/roles/github_runner_cluster/templates/native/config.yaml.j2 b/roles/github_runner_cluster/templates/native/agent.yaml.j2 similarity index 75% rename from roles/github_runner_cluster/templates/native/config.yaml.j2 rename to roles/github_runner_cluster/templates/native/agent.yaml.j2 index 799db226..a1f55ad9 100644 --- a/roles/github_runner_cluster/templates/native/config.yaml.j2 +++ b/roles/github_runner_cluster/templates/native/agent.yaml.j2 @@ -1,6 +1,6 @@ # Managed by the exadev.github_runner collection's github_runner_cluster role (node mode native). Do not edit by hand: change this host's variables and run the role again. -# k3s reads this file as the flags of `k3s agent`. The token and the --vpn-auth value are secrets and are in files of their own, mode 0600. Values are written as JSON strings, which YAML reads as written. -server: {{ github_runner_cluster_effective_server_url | to_json }} +# k3s reads this file as the flags of `k3s agent` (the k3s-agent unit passes it as --config, so k3s reads neither its default /etc/rancher/k3s/config.yaml nor the server's file). The token and the --vpn-auth value are secrets and are in files of their own, mode 0600. Values are written as JSON strings, which YAML reads as written. +server: {{ github_runner_cluster_native_join_url | to_json }} token-file: {{ (github_runner_cluster_native_config_dir ~ '/token') | to_json }} node-name: {{ github_runner_cluster_effective_node_name | to_json }} vpn-auth-file: {{ (github_runner_cluster_native_config_dir ~ '/vpn-auth') | to_json }} diff --git a/roles/github_runner_cluster/templates/native/join.yaml.j2 b/roles/github_runner_cluster/templates/native/join.yaml.j2 new file mode 100644 index 00000000..0303b284 --- /dev/null +++ b/roles/github_runner_cluster/templates/native/join.yaml.j2 @@ -0,0 +1,10 @@ +# Managed by the exadev.github_runner collection's github_runner_cluster role (node mode native). Whether this server bootstraps a new cluster or joins an existing one, read by k3s after server.yaml. +# +# cluster-init only when the role found no cluster to join: this host is the bootstrap host of the inventory, has no datastore of its own (server/db) and none moved aside by a demotion (server/db.demoted), and no other server of the inventory answers. A host that was ever a member therefore joins the cluster that exists rather than starting a second one. k3s reuses a datastore it finds on disk and then ignores both keys (https://docs.k3s.io/datastore/ha-embedded). +{% if github_runner_cluster_native_cluster_init | bool %} +cluster-init: true +{% elif github_runner_cluster_native_join_url | length > 0 %} +server: {{ github_runner_cluster_native_join_url | to_json }} +{% else %} +{} +{% endif %} diff --git a/roles/github_runner_cluster/templates/native/k3s-agent.service.j2 b/roles/github_runner_cluster/templates/native/k3s-agent.service.j2 index 0bec5809..02bbc8a2 100644 --- a/roles/github_runner_cluster/templates/native/k3s-agent.service.j2 +++ b/roles/github_runner_cluster/templates/native/k3s-agent.service.j2 @@ -29,10 +29,12 @@ RestartSec=5s # Kept across restarts, so the file the kubelet reads is there while k3s starts again; removed when the service stops. RuntimeDirectory=github-runner-k3s RuntimeDirectoryPreserve=restart +# Starts only while the role file names this host an agent (files/native/fleet-role-is.sh), so this unit and the k3s server unit never run together and Restart=always never starts the agent on a host that is now a server. +ExecCondition={{ github_runner_cluster_native_lib_dir }}/fleet-role-is agent {{ github_runner_cluster_native_role_file }} ExecStartPre=-/sbin/modprobe br_netfilter ExecStartPre=-/sbin/modprobe overlay ExecStartPre={{ github_runner_cluster_native_lib_dir }}/write-resolv-conf /run/github-runner-k3s/resolv.conf -ExecStart={{ github_runner_cluster_native_bin_dir }}/k3s agent +ExecStart={{ github_runner_cluster_native_bin_dir }}/k3s agent --config {{ github_runner_cluster_native_config_dir }}/agent.yaml [Install] WantedBy=multi-user.target diff --git a/roles/github_runner_cluster/templates/native/k3s.service.j2 b/roles/github_runner_cluster/templates/native/k3s.service.j2 new file mode 100644 index 00000000..46d60d13 --- /dev/null +++ b/roles/github_runner_cluster/templates/native/k3s.service.j2 @@ -0,0 +1,42 @@ +# Managed by the exadev.github_runner collection's github_runner_cluster role (node mode native). Do not edit by hand: change this host's variables and run the role again. +# Modelled on the unit that k3s's own install script writes for a server (install.sh in the k3s repository), with the changes noted below, and kept like the native agent's unit (k3s-agent.service.j2) in everything but the role it runs. +[Unit] +Description=k3s server (a native node of the GitHub Actions runner fleet) +Documentation=https://docs.k3s.io +# tailscaled is wanted rather than required: restarting it (a package upgrade does) must not restart this service, since Tailscale keeps the node's routes across its own restart. +Wants=network-online.target tailscaled.service +After=network-online.target tailscaled.service + +[Service] +# exec rather than the installer's notify: a server that cannot reach the voter it joins would otherwise leave `systemctl start` waiting for ever (TimeoutStartSec=0), and the role checks the node through the API instead. +Type=exec +# The wrapper directory comes first, so k3s's own `tailscale up` finds files/native/tailscale-up-wait.sh before the real command; see that file. +Environment="PATH={{ github_runner_cluster_native_lib_dir }}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +Environment="GITHUB_RUNNER_TAILSCALE={{ github_runner_cluster_native_tailscale_cli }}" +Environment="GITHUB_RUNNER_JOIN_HOST={{ github_runner_cluster_native_join_host }}" +# Pods keep running while this service restarts or stops (their containerd shims stay), as with the installer's unit. +KillMode=process +Delegate=yes +User=root +# Zero limits cost accounting overhead in the kernel; cgroups account for containers. +LimitNOFILE=1048576 +LimitNPROC=infinity +LimitCORE=infinity +TasksMax=infinity +TimeoutStartSec=0 +Restart=always +RestartSec=5s +# Kept across restarts, so the file the kubelet reads is there while k3s starts again; removed when the service stops. +RuntimeDirectory=github-runner-k3s +RuntimeDirectoryPreserve=restart +# Starts only while the role file names this host a server (files/native/fleet-role-is.sh), so this unit and the k3s agent unit never run together and Restart=always never starts a server that has been demoted, which would join the cluster again as a learner. +ExecCondition={{ github_runner_cluster_native_lib_dir }}/fleet-role-is server {{ github_runner_cluster_native_role_file }} +ExecStartPre=-/sbin/modprobe br_netfilter +ExecStartPre=-/sbin/modprobe overlay +ExecStartPre={{ github_runner_cluster_native_lib_dir }}/write-resolv-conf /run/github-runner-k3s/resolv.conf +# etcd's peer URL is the host's Tailscale address, written as node-ip into the configuration's drop-in directory before each start (files/native/write-node-ip.sh). +ExecStartPre={{ github_runner_cluster_native_lib_dir }}/write-node-ip {{ github_runner_cluster_native_config_dir }}/server.yaml.d/00-node-ip.yaml {{ github_runner_cluster_native_config_dir }}/vpn-auth +ExecStart={{ github_runner_cluster_native_bin_dir }}/k3s server --config {{ github_runner_cluster_native_config_dir }}/server.yaml + +[Install] +WantedBy=multi-user.target diff --git a/roles/github_runner_cluster/templates/native/server.yaml.j2 b/roles/github_runner_cluster/templates/native/server.yaml.j2 new file mode 100644 index 00000000..e05ec3c0 --- /dev/null +++ b/roles/github_runner_cluster/templates/native/server.yaml.j2 @@ -0,0 +1,31 @@ +# Managed by the exadev.github_runner collection's github_runner_cluster role (node mode native). Do not edit by hand: change this host's variables and run the role again. +# k3s reads this file as the flags of `k3s server` (the k3s unit passes it as --config), then every *.yaml in server.yaml.d beside it in name order (dotDFiles in pkg/configfilearg/parser.go of k3s), a later file's key replacing an earlier one's: 00-node-ip.yaml, which the unit writes before every start (write-node-ip), and 10-join.yaml, which says whether this server bootstraps a new cluster or joins one, and which a role change rewrites without touching this file. The token and the --vpn-auth value are secrets and are in files of their own, mode 0600. Values are written as JSON strings, which YAML reads as written. +# +# These match what every Docker server passes (k3s/node-entrypoint.sh's mesh_server_args): k3s refuses a server whose critical configuration differs from the cluster's ("critical configuration value mismatch between servers", pkg/cluster/bootstrap.go; the fields are CriticalControlArgs in pkg/daemons/config/types.go, disable-network-policy and the cluster and service networks, which every server leaves at k3s's own, among them), and a server that did not disable traefik would deploy it again. +token-file: {{ (github_runner_cluster_native_config_dir ~ '/token') | to_json }} +vpn-auth-file: {{ (github_runner_cluster_native_config_dir ~ '/vpn-auth') | to_json }} +node-name: {{ github_runner_cluster_effective_node_name | to_json }} +disable: + - "traefik" +disable-network-policy: true +# Only root reads the admin kubeconfig k3s writes to /etc/rancher/k3s/k3s.yaml: the role reads it through `k3s kubectl` with become, and nothing on the host needs it as another user. +write-kubeconfig-mode: "0600" +{% if github_runner_cluster_api_port | int != github_runner_cluster_k3s_default_api_port | int %} +https-listen-port: {{ github_runner_cluster_api_port | int }} +{% endif %} +tls-san: +{% for san in github_runner_cluster_effective_tls_sans %} + - {{ san | to_json }} +{% endfor %} +{% if github_runner_cluster_etcd_heartbeat_interval | string | length > 0 %} +etcd-arg: + - {{ ('heartbeat-interval=' ~ github_runner_cluster_etcd_heartbeat_interval) | to_json }} + - {{ ('election-timeout=' ~ github_runner_cluster_etcd_election_timeout) | to_json }} +{% endif %} +resolv-conf: "/run/github-runner-k3s/resolv.conf" +{% if github_runner_cluster_node_taints | length > 0 %} +node-taint: +{% for taint in github_runner_cluster_node_taints %} + - {{ taint | to_json }} +{% endfor %} +{% endif %} diff --git a/tests/unit/test_native_node.py b/tests/unit/test_native_node.py index d7810bf7..fb4c3ee6 100644 --- a/tests/unit/test_native_node.py +++ b/tests/unit/test_native_node.py @@ -1,4 +1,4 @@ -"""Tests the github_runner_cluster role's native node mode: the filters (plugins/filter/cluster_native.py), the two helper scripts the k3s-agent service runs, the unit and configuration templates, the defaults, argument specs and README, and the role's tasks (tests/native/run.yml) against stand-ins for systemctl, tailscale, dpkg-query, apt-get and docker on PATH and a local server for the k3s release, so that no real service, package or tailnet is touched.""" +"""Tests the github_runner_cluster role's native node mode: the filters (plugins/filter/cluster_native.py), the helper scripts the k3s-agent and k3s units run, the unit and configuration templates, the defaults, argument specs and README, and the role's tasks (tests/native/run.yml) against stand-ins for systemctl, tailscale, dpkg-query, apt-get, docker and the kubectl that `k3s kubectl` and `docker exec kubectl` reach on PATH, a local server for the k3s release, and a k3s data directory with a server's real layout, so that no real service, package, tailnet or cluster is touched.""" from __future__ import annotations @@ -105,6 +105,56 @@ def test_a_status_that_is_not_the_clients_is_refused(self) -> None: with self.assertRaisesRegex(Exception, "tailnet domain must be set"): cluster_native.cluster_native_tailnet_login(tailscale_status("Running", DOMAIN), "") + def test_a_role_file_names_the_role_now_and_a_legacy_agent_counts_as_one(self) -> None: + role = cluster_native.cluster_native_role + self.assertEqual(role("", False, False, "server"), {"current": "", "desired": "server"}) + self.assertEqual(role("", True, False, "agent"), {"current": "agent", "desired": "agent"}) + self.assertEqual(role("agent\n", True, False, "server"), {"current": "agent", "desired": "server"}) + self.assertEqual(role("server", False, False, "agent"), {"current": "server", "desired": "agent"}) + + def test_an_eligible_host_keeps_the_role_its_file_names(self) -> None: + role = cluster_native.cluster_native_role + self.assertEqual(role("agent", True, True, "server"), {"current": "agent", "desired": "agent"}) + self.assertEqual(role("server", False, True, "agent"), {"current": "server", "desired": "server"}) + # The inventory gives an eligible host the role it starts with. + self.assertEqual(role("", False, True, "server"), {"current": "", "desired": "server"}) + + def test_while_exactly_one_unit_runs_its_role_is_the_role_now(self) -> None: + role = cluster_native.cluster_native_role + # A demotion that stopped after the role file and before the server unit: the next run takes it up again. + self.assertEqual(role("agent", True, False, "agent", ["server"]), {"current": "server", "desired": "agent"}) + # A promotion that stopped after the role file and before the agent unit. + self.assertEqual(role("server", True, False, "server", ["agent"]), {"current": "agent", "desired": "server"}) + # Both or neither running leaves the file to say. + self.assertEqual(role("agent", True, False, "agent", ["server", "agent"]), {"current": "agent", "desired": "agent"}) + with self.assertRaisesRegex(Exception, "are not node roles"): + role("agent", True, False, "agent", ["controller"]) + + def test_a_role_file_holding_anything_else_is_refused(self) -> None: + with self.assertRaisesRegex(Exception, "must be 'server' or 'agent'"): + cluster_native.cluster_native_role("controller", False, False, "agent") + + def test_a_cluster_is_started_only_when_no_other_server_runs_one(self) -> None: + init = cluster_native.cluster_native_cluster_init + stopped = {"item": "server2", "rc": 0, "stdout": "stopped"} + running = {"item": "server2", "rc": 0, "stdout": "running\nserver2 v1.34.1+k3s1 true"} + own = {"item": "laptop", "rc": 0, "stdout": "running\nlaptop v1.34.1+k3s1 true"} + self.assertEqual(init([], "laptop"), {"cluster_init": True, "running": [], "unread": []}) + self.assertEqual(init([own, stopped], "laptop"), {"cluster_init": True, "running": [], "unread": []}) + self.assertEqual(init([own, running], "laptop"), {"cluster_init": False, "running": ["server2"], "unread": []}) + unread = init([{"item": "server2", "rc": 1, "stderr": "Error: no route to host\n"}, {"item": "server3", "unreachable": True, "msg": "ssh: connect"}], "laptop") + self.assertEqual(unread, {"cluster_init": False, "running": [], "unread": ["server2 could not be read (Error: no route to host)", "server3 could not be read (ssh: connect)"]}) + + def test_a_server_waits_for_every_other_live_voter_to_be_ready(self) -> None: + blockers = cluster_native.cluster_native_voter_blockers + read = {"rc": 0, "stdout": "server1,True,true\nserver2,False,true\nagent1,False,\nlaptop,Unknown,true\nserver3,,true\n"} + self.assertEqual(blockers(read, "laptop"), ["voter server2 is not Ready (its Ready condition is False)", "voter server3 is not Ready (its Ready condition is unknown)"]) + self.assertEqual(blockers({"rc": 0, "stdout": "server1,True,true\nagent1,False,\n"}, "laptop"), []) + self.assertEqual(blockers({"rc": 0, "stdout": ""}, "laptop"), []) + self.assertEqual(blockers({"rc": 0, "stdout": "server1,True,true\nserver2,True,true\nlaptop,True,true\n"}, "laptop", 3), ["only 2 other voters (server1, server2) would remain, fewer than the 3 a change that takes this server's vote away leaves"]) + self.assertEqual(blockers({"rc": 0, "stdout": "server1,True,true\nserver2,True,true\nserver3,True,true\nlaptop,True,true\n"}, "laptop", 3), []) + self.assertIn("could not be listed (The connection to the server was refused)", blockers({"rc": 1, "stderr": "The connection to the server was refused"}, "laptop")[0]) + def test_the_filters_are_registered_under_the_names_the_tasks_use(self) -> None: names = set(cluster_native.FilterModule().filters()) used = set(re.findall(r"exadev\.github_runner\.(cluster_native_\w+)", "".join(path.read_text() for path in (ROLE / "tasks" / "native").glob("*.yml")))) @@ -210,9 +260,75 @@ def test_a_replaced_file_is_replaced_in_one_rename(self) -> None: self.resolv(resolved="nameserver 10.0.0.2\n") self.assertEqual(self.written(), "nameserver 10.0.0.2\n") + def role_is(self, role: str, recorded: str | None) -> subprocess.CompletedProcess[str]: + file = self.tmp / "fleet-role" + if recorded is not None: + file.write_text(recorded) + return subprocess.run(["sh", str(ROLE / "files" / "native" / "fleet-role-is.sh"), role, str(file)], env=self.env(), capture_output=True, text=True, timeout=60, check=False) + + def test_the_role_condition_holds_only_for_the_role_the_file_names(self) -> None: + self.assertEqual(self.role_is("server", "server").returncode, 0) + self.assertEqual(self.role_is("agent", "agent\n").returncode, 0) + refused = self.role_is("agent", "server") + # 1 to 254 is what systemd reads as a skipped start rather than a failure (systemd.service(5), ExecCondition=). + self.assertEqual(refused.returncode, 1) + self.assertIn("names 'server', not agent", refused.stderr) + + def test_without_a_role_file_neither_unit_starts(self) -> None: + for role in ("server", "agent"): + with self.subTest(role=role): + result = self.role_is(role, None) + self.assertEqual(result.returncode, 1) + self.assertIn("names 'no role'", result.stderr) + + def node_ip(self, vpn_auth: str, address: str | None) -> subprocess.CompletedProcess[str]: + """Run write-node-ip.sh with a stand-in Tailscale whose client has ``address`` (none when it is logged out, until `tailscale up` logs it in).""" + (self.tmp / "vpn-auth").write_text(vpn_auth) + if address is not None: + (self.tmp / "address").write_text(address) + stand_in(self.bin / "tailscale", 'echo "tailscale $*" >> "$LOG"\ncase "$1" in\n ip) [ -e "$DIR/address" ] && cat "$DIR/address" ;;\n up) echo 100.64.0.7 > "$DIR/address" ;;\nesac\n') + output = self.tmp / "server.yaml.d" / "00-node-ip.yaml" + return subprocess.run(["sh", str(ROLE / "files" / "native" / "write-node-ip.sh"), str(output), str(self.tmp / "vpn-auth")], env=self.env(GITHUB_RUNNER_TAILSCALE=str(self.bin / "tailscale"), DIR=str(self.tmp)), capture_output=True, text=True, timeout=60, check=False) + + def written_node_ip(self) -> Any: + return yaml.safe_load((self.tmp / "server.yaml.d" / "00-node-ip.yaml").read_text()) + + def test_the_node_ip_is_the_clients_tailscale_address_and_nothing_logs_in(self) -> None: + result = self.node_ip(f"name=tailscale,joinKey={JOIN_KEY}", "100.64.0.5\n") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.written_node_ip(), {"node-ip": "100.64.0.5"}) + self.assertEqual(self.log.read_text(), "tailscale ip -4\n") + self.assertEqual(sorted(path.name for path in (self.tmp / "server.yaml.d").iterdir()), ["00-node-ip.yaml"]) + + def test_a_client_with_no_address_is_logged_in_as_k3s_would_log_it_in(self) -> None: + result = self.node_ip(f"name=tailscale,joinKey={JOIN_KEY},controlServerURL=https://headscale.example.com,extraArgs=--accept-dns=false --netfilter-mode=off", None) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.written_node_ip(), {"node-ip": "100.64.0.7"}) + self.assertEqual(self.log.read_text().splitlines(), [ + "tailscale ip -4", + f"tailscale up --authkey {JOIN_KEY} --timeout=30s --reset --login-server https://headscale.example.com --accept-dns=false --netfilter-mode=off", + "tailscale ip -4", + ]) + self.assertNotIn(JOIN_KEY, result.stdout + result.stderr) + + def test_tailscales_own_control_plane_takes_no_login_server(self) -> None: + self.assertEqual(self.node_ip(f"name=tailscale,joinKey={JOIN_KEY}", None).returncode, 0) + self.assertIn(f"tailscale up --authkey {JOIN_KEY} --timeout=30s --reset\n", self.log.read_text()) + + def test_no_address_and_no_join_key_fails_and_writes_nothing(self) -> None: + result = self.node_ip("name=tailscale", None) + self.assertNotEqual(result.returncode, 0) + self.assertIn("names no joinKey", result.stderr) + self.assertFalse((self.tmp / "server.yaml.d" / "00-node-ip.yaml").exists()) + + def test_an_address_that_is_not_ipv4_fails(self) -> None: + result = self.node_ip(f"name=tailscale,joinKey={JOIN_KEY}", "fd7a:115c:a1e0::7\n") + self.assertNotEqual(result.returncode, 0) + self.assertIn("reports no IPv4 address", result.stderr) + def test_both_scripts_are_executable_and_shellcheck_clean(self) -> None: scripts = sorted((ROLE / "files" / "native").glob("*.sh")) - self.assertEqual(len(scripts), 2) + self.assertEqual([script.name for script in scripts], ["fleet-role-is.sh", "tailscale-up-wait.sh", "write-node-ip.sh", "write-resolv-conf.sh"]) for script in scripts: self.assertTrue(os.access(script, os.X_OK), script) if shutil.which("shellcheck"): @@ -242,7 +358,15 @@ def test_the_download_is_checked_against_the_releases_own_sums_unless_a_sum_is_s def test_every_native_default_is_declared_with_the_same_default_and_described_in_the_readme(self) -> None: native = {name for name in DEFAULTS if name.startswith("github_runner_cluster_native_") or name == "github_runner_cluster_node_mode"} - derived = {"github_runner_cluster_native_service", "github_runner_cluster_native_join_host"} + derived = { + "github_runner_cluster_native_service", + "github_runner_cluster_native_agent_service", + "github_runner_cluster_native_server_service", + "github_runner_cluster_native_join_url", + "github_runner_cluster_native_join_host", + "github_runner_cluster_native_role_file", + "github_runner_cluster_native_restart_marker", + } self.assertEqual(set(SPECS), native - derived) for name, spec in SPECS.items(): with self.subTest(name=name): @@ -263,12 +387,13 @@ def test_the_mode_choices_are_the_ones_the_topology_filter_accepts(self) -> None class TemplateTest(unittest.TestCase): """Reads the unit and the configuration the role writes, from a run of the role (see NativeRoleTest).""" - def test_the_unit_and_configuration_are_templates_the_role_writes(self) -> None: - config = yaml.safe_load((ROLE / "tasks" / "native" / "config.yml").read_text()) - written = {task["ansible.builtin.template"]["src"] for task in config if "ansible.builtin.template" in task} - self.assertEqual(written, {"native/config.yaml.j2", "native/k3s-agent.service.j2"}) - for name in written: - self.assertTrue((ROLE / "templates" / name).is_file(), name) + def test_the_units_and_configurations_are_templates_the_role_has(self) -> None: + templates = sorted(path.name for path in (ROLE / "templates" / "native").iterdir()) + self.assertEqual(templates, ["agent.yaml.j2", "join.yaml.j2", "k3s-agent.service.j2", "k3s.service.j2", "server.yaml.j2"]) + config = (ROLE / "tasks" / "native" / "config.yml").read_text() + for name in ("agent.yaml.j2", "server.yaml.j2", "join.yaml.j2", "native/{{ item }}.service.j2"): + self.assertIn(name, config) + self.assertEqual((DEFAULTS["github_runner_cluster_native_agent_service"], DEFAULTS["github_runner_cluster_native_server_service"]), ("k3s-agent", "k3s")) def test_every_secret_task_logs_nothing(self) -> None: for path in sorted((ROLE / "tasks" / "native").glob("*.yml")): @@ -283,7 +408,8 @@ def test_become_is_used_by_the_tasks_that_change_the_host_and_by_nothing_wider(s for task in yaml.safe_load(path.read_text()): with self.subTest(path=path.name, task=task.get("name")): self.assertNotIn("block", task) if "become" in task else None - self.assertIn(task.get("become"), (None, True)) + # A command run through another server's API takes become exactly when that server is native (api.yml). + self.assertIn(task.get("become"), (None, True, "{{ github_runner_cluster_native_api_become }}")) self.assertNotIn("become", (ROLE / "tasks" / "main.yml").read_text()) def test_main_hands_native_hosts_to_the_native_tasks_and_ends_the_role_before_any_docker_task(self) -> None: @@ -304,7 +430,8 @@ def setUpClass(cls) -> None: cls.mirror = Path(tempfile.mkdtemp(prefix="native-mirror-")) release = cls.mirror / K3S_RELEASE release.mkdir() - cls.binary = f'#!/bin/sh\necho "k3s version {K3S_RELEASE} (stand-in)"\n' + # `k3s kubectl` reaches the test's kubectl stand-in, which answers for the cluster as the stand-in for `docker exec kubectl` does. + cls.binary = f'#!/bin/sh\nif [ "$1" = kubectl ]; then shift; exec "$STUB/bin/kubectl" "$@"; fi\necho "k3s version {K3S_RELEASE} (stand-in)"\n' (release / "k3s").write_text(cls.binary) cls.digest = hashlib.sha256(cls.binary.encode()).hexdigest() (release / "sha256sum-amd64.txt").write_text(f"{cls.digest} k3s\n") @@ -334,6 +461,7 @@ def setUp(self) -> None: "github_runner_cluster_native_lib_dir": self.root / "usr" / "local" / "lib" / "github-runner-k3s", "github_runner_cluster_native_config_dir": self.root / "etc" / "rancher" / "k3s", "github_runner_cluster_native_unit_dir": self.root / "etc" / "systemd" / "system", + "github_runner_cluster_native_data_dir": self.root / "var" / "lib" / "rancher" / "k3s", "github_runner_cluster_native_apt_keyring": self.root / "usr" / "share" / "keyrings" / "tailscale-archive-keyring.gpg", "github_runner_cluster_native_apt_source": self.root / "etc" / "apt" / "sources.list.d" / "tailscale.list", } @@ -351,16 +479,36 @@ def make_stand_ins(self) -> None: bin_dir.mkdir() log = 'echo "$(basename "$0") $*" >> "$STUB/calls.log"\n' stand_in(bin_dir / "systemctl", log + r"""state="$STUB/state" -case "$1" in - is-enabled) if [ -e "$state/enabled-$2" ]; then echo enabled; exit 0; fi; echo disabled; exit 1 ;; - is-active) if [ -e "$state/active-$2" ]; then echo active; exit 0; fi; echo inactive; exit 3 ;; - enable) if [ "$2" = --now ]; then touch "$state/enabled-$3" "$state/active-$3"; else touch "$state/enabled-$2"; fi ;; - start|restart) - touch "$state/active-$2" - # k3s runs `tailscale up` with the join key as it starts, which logs the host's client in to the cluster's tailnet. - if [ "$2" = k3s-agent ] && [ ! -e "$state/no-join" ]; then - printf '%s' '{"BackendState":"Running","CurrentTailnet":{"MagicDNSSuffix":"example.ts.net"}}' > "$state/tailscale.json" - fi ;; +command=$1 +shift +now="" +quiet="" +for argument in "$@"; do + case "$argument" in + --quiet) quiet=1 ;; + --now) now=1 ;; + *) unit=$argument ;; + esac +done +join() { + # k3s runs `tailscale up` with the join key as it starts, which logs the host's client in to the cluster's tailnet. + if [ ! -e "$state/no-join" ]; then + printf '%s' '{"BackendState":"Running","CurrentTailnet":{"MagicDNSSuffix":"example.ts.net"}}' > "$state/tailscale.json" + fi +} +case "$command" in + # --quiet prints nothing, as systemctl's does. + is-enabled) if [ -e "$state/enabled-$unit" ]; then [ -n "$quiet" ] || echo enabled; exit 0; fi; [ -n "$quiet" ] || echo disabled; exit 1 ;; + is-active) if [ -e "$state/active-$unit" ]; then [ -n "$quiet" ] || echo active; exit 0; fi; [ -n "$quiet" ] || echo inactive; exit 3 ;; + enable) + touch "$state/enabled-$unit" + if [ -n "$now" ]; then touch "$state/active-$unit"; case "$unit" in k3s|k3s-agent) join ;; esac; fi ;; + disable) rm -f "$state/enabled-$unit"; if [ -n "$now" ]; then rm -f "$state/active-$unit"; fi ;; + start|restart) touch "$state/active-$unit"; case "$unit" in k3s|k3s-agent) join ;; esac ;; +esac +# What the role file named when a k3s unit was started, which is what its ExecCondition would have read. +case "$command:$unit" in + enable:k3s*|start:k3s*|restart:k3s*) if [ "$command" != enable ] || [ -n "$now" ]; then echo "$command $unit $(cat "$ROLE_FILE" 2>/dev/null || echo none)" >> "$STUB/starts.log"; fi ;; esac """) stand_in(bin_dir / "tailscale", log + r"""case "$1" in @@ -379,13 +527,28 @@ def make_stand_ins(self) -> None: esac done """) + # `docker exec kubectl ...` reaches the same kubectl stand-in as `k3s kubectl ...`. stand_in(bin_dir / "docker", log + r"""case "$*" in - "ps "*) echo running ;; - *"kubectl get nodes"*) printf 'server1 v1.34.1+k3s1 true\n' ;; - *"kubectl get node "*) echo True ;; - *"kubectl get lease"*) echo 2099-01-01T00:00:00Z ;; + "ps "*) if [ -e "$STUB/state/docker-stopped" ]; then echo exited; else echo running; fi ;; + "exec "*" kubectl "*) shift 3; exec "$STUB/bin/kubectl" "$@" ;; *) echo "docker stand-in: unexpected $*" >&2; exit 1 ;; esac +""") + # The cluster as the API reports it, from files in the state directory the tests write: the nodes with their versions and live roles, the voters' readiness, this node's readiness and etcd vote, its member annotations, and the runner pods on it. + stand_in(bin_dir / "kubectl", log + r"""state="$STUB/state" +case "$*" in + "get nodes "*kubeletVersion*) + if [ -e "$state/nodes-unreadable" ]; then echo "The connection to the server was refused" >&2; exit 1; fi + if [ -e "$state/nodes" ]; then cat "$state/nodes"; else printf 'server1 v1.34.1+k3s1 true\n'; fi ;; + "get nodes "*) if [ -e "$state/voters" ]; then cat "$state/voters"; else printf 'server1,True,true\n'; fi ;; + "get node "*removed-node-name*) if [ -e "$state/member-removed" ]; then printf 'laptop-0a1b2c3d,'; else printf ',laptop-0a1b2c3d'; fi ;; + "annotate --overwrite node "*"etcd.k3s.cattle.io/remove=true"*) [ -e "$state/member-stays" ] || touch "$state/member-removed" ;; + "get node "*) if [ -e "$state/node-ready" ]; then cat "$state/node-ready"; else printf 'True,True'; fi ;; + "get lease "*) if [ -e "$state/lease" ]; then cat "$state/lease"; else echo 2099-01-01T00:00:00Z; fi ;; + "get pods "*) [ -e "$state/runner-pods" ] && cat "$state/runner-pods"; exit 0 ;; + cordon*|uncordon*|drain*|label*|annotate*) ;; + *) echo "kubectl stand-in: unexpected $*" >&2; exit 1 ;; +esac """) def set_tailscale(self, backend: str | None, domain: str | None) -> None: @@ -407,7 +570,7 @@ def set_package(self, version: str | None) -> None: (self.state / "enabled-tailscaled").touch() (self.state / "active-tailscaled").touch() - def inventory(self, only_the_node: bool = False, **host_vars: Any) -> Path: + def inventory(self, only_the_node: bool = False, second_server: bool = False, **host_vars: Any) -> Path: shared = { "ansible_connection": "local", "ansible_become": False, @@ -422,17 +585,18 @@ def inventory(self, only_the_node: bool = False, **host_vars: Any) -> Path: **{name: str(path) for name, path in self.paths.items()}, } node = {**shared, "github_runner_cluster_node_mode": "native", **host_vars} - hosts = {"laptop": node} if only_the_node else {"server1": dict(shared), "laptop": node} + hosts = {"laptop": node} if only_the_node else {"server1": dict(shared), **({"server2": dict(shared)} if second_server else {}), "laptop": node} inventory = {"all": {"children": {"github_runner_cluster": {"hosts": hosts}}}} path = self.tmp / "inventory.yml" path.write_text(yaml.safe_dump(inventory)) return path - def run_role(self, *args: str, host_vars: dict[str, Any] | None = None, check: bool = False, only_the_node: bool = False, **extra: Any) -> subprocess.CompletedProcess[str]: + def run_role(self, *args: str, host_vars: dict[str, Any] | None = None, check: bool = False, only_the_node: bool = False, second_server: bool = False, **extra: Any) -> subprocess.CompletedProcess[str]: env = { **os.environ, "PATH": f"{self.stubs / 'bin'}:{os.environ['PATH']}", "STUB": str(self.stubs), + "ROLE_FILE": str(self.file("github_runner_cluster_native_data_dir", "fleet-role")), "ANSIBLE_COLLECTIONS_PATH": str(ROOT / "playbooks" / "collections"), "ANSIBLE_NOCOLOR": "1", "ANSIBLE_FORCE_COLOR": "0", @@ -440,7 +604,7 @@ def run_role(self, *args: str, host_vars: dict[str, Any] | None = None, check: b "ANSIBLE_INVENTORY_UNPARSED_WARNING": "false", "ANSIBLE_RETRY_FILES_ENABLED": "false", } - command = [ANSIBLE_PLAYBOOK, "-i", str(self.inventory(only_the_node, **(host_vars or {}))), str(RUN), "-e", json.dumps({"host_facts": self.facts, **extra}), *args] + command = [ANSIBLE_PLAYBOOK, "-i", str(self.inventory(only_the_node, second_server, **(host_vars or {}))), str(RUN), "-e", json.dumps({"host_facts": self.facts, **extra}), *args] if check: command.append("--check") return subprocess.run(command, env=env, capture_output=True, text=True, timeout=300, check=False, cwd=ROOT / "ansible") @@ -448,10 +612,11 @@ def run_role(self, *args: str, host_vars: dict[str, Any] | None = None, check: b def calls(self) -> list[str]: """The commands the stand-ins were run with, one per call (a call whose arguments hold a line break logs more lines, which are not calls).""" log = self.stubs / "calls.log" - return [line for line in log.read_text().splitlines() if re.match(r"^(systemctl|tailscale|apt-get|dpkg-query|docker) ", line)] if log.exists() else [] + return [line for line in log.read_text().splitlines() if re.match(r"^(systemctl|tailscale|apt-get|dpkg-query|docker|kubectl) ", line)] if log.exists() else [] - def mutations(self) -> list[str]: - return [call for call in self.calls() if call.split()[0] in ("apt-get", "systemctl") and call.split()[1] not in ("is-enabled", "is-active")] + [call for call in self.calls() if call in ("tailscale logout",)] + def mutations(self, since: int = 0) -> list[str]: + """The calls that change something, from the ``since``-th call on.""" + return [call for call in self.calls()[since:] if call.split()[0] in ("apt-get", "systemctl") and call.split()[1] not in ("is-enabled", "is-active")] + [call for call in self.calls()[since:] if call in ("tailscale logout",) or (call.startswith("kubectl ") and call.split()[1] in ("cordon", "uncordon", "drain", "label", "annotate"))] def file(self, key: str, name: str) -> Path: return self.paths[key] / name @@ -471,7 +636,7 @@ def test_a_first_run_installs_configures_starts_and_checks_the_node(self) -> Non self.assertEqual(stat.S_IMODE(binary.stat().st_mode), 0o755) config_dir = "github_runner_cluster_native_config_dir" - config = yaml.safe_load(self.file(config_dir, "config.yaml").read_text()) + config = yaml.safe_load(self.file(config_dir, "agent.yaml").read_text()) self.assertEqual(config, { "server": f"https://server1.{DOMAIN}:6443", "token-file": str(self.file(config_dir, "token")), @@ -481,7 +646,9 @@ def test_a_first_run_installs_configures_starts_and_checks_the_node(self) -> Non }) self.assertEqual(self.file(config_dir, "token").read_text(), TOKEN) self.assertEqual(self.file(config_dir, "vpn-auth").read_text(), f"name=tailscale,joinKey={JOIN_KEY}") - for name in ("config.yaml", "token", "vpn-auth"): + self.assertFalse(self.file(config_dir, "server.yaml").exists()) + self.assertEqual(self.file("github_runner_cluster_native_data_dir", "fleet-role").read_text(), "agent") + for name in ("agent.yaml", "token", "vpn-auth"): self.assertEqual(stat.S_IMODE(self.file(config_dir, name).stat().st_mode), 0o600, name) lib = "github_runner_cluster_native_lib_dir" @@ -509,7 +676,8 @@ def test_the_unit_runs_the_installed_binary_under_the_wrapper_with_the_hosts_joi self.assertEqual(set(sections), {"Unit", "Service", "Install"}) lib, bin_dir = self.paths["github_runner_cluster_native_lib_dir"], self.paths["github_runner_cluster_native_bin_dir"] service = sections["Service"] - self.assertIn(f"ExecStart={bin_dir}/k3s agent", service) + self.assertIn(f"ExecStart={bin_dir}/k3s agent --config {self.paths['github_runner_cluster_native_config_dir']}/agent.yaml", service) + self.assertIn(f"ExecCondition={lib}/fleet-role-is agent {self.paths['github_runner_cluster_native_data_dir']}/fleet-role", service) self.assertIn(f"ExecStartPre={lib}/write-resolv-conf /run/github-runner-k3s/resolv.conf", service) self.assertIn(f'Environment="PATH={lib}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"', service) self.assertIn(f'Environment="GITHUB_RUNNER_TAILSCALE={self.stubs / "bin" / "tailscale"}"', service) @@ -523,12 +691,12 @@ def test_the_unit_runs_the_installed_binary_under_the_wrapper_with_the_hosts_joi def test_node_taints_are_written_as_k3s_node_taints(self) -> None: taints = ["github-runner.example.com/excluded=true:NoExecute", "dedicated=ci:NoSchedule"] self.assertEqual(self.run_role(host_vars={"github_runner_cluster_node_taints": taints}).returncode, 0) - config = yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "config.yaml").read_text()) + config = yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "agent.yaml").read_text()) self.assertEqual(config["node-taint"], taints) def test_a_host_that_sets_its_node_name_joins_under_it(self) -> None: self.assertEqual(self.run_role(host_vars={"github_runner_cluster_node_name": "laptop-renamed"}).returncode, 0) - config = yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "config.yaml").read_text()) + config = yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "agent.yaml").read_text()) self.assertEqual(config["node-name"], "laptop-renamed") def test_a_second_run_changes_nothing(self) -> None: @@ -537,7 +705,7 @@ def test_a_second_run_changes_nothing(self) -> None: result = self.run_role() self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertEqual(self.recap(result)["changed"], 0, result.stdout) - self.assertEqual([call for call in self.calls()[before:] if "is-" not in call and not call.startswith(("docker", "tailscale status"))], []) + self.assertEqual([call for call in self.calls()[before:] if "is-" not in call and not call.startswith(("docker", "tailscale status", "kubectl get"))], []) def test_a_changed_setting_restarts_the_running_service_once_and_enables_nothing_again(self) -> None: self.assertEqual(self.run_role().returncode, 0) @@ -565,7 +733,7 @@ def test_start_false_writes_everything_and_leaves_the_service_alone(self) -> Non self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertTrue(self.file("github_runner_cluster_native_unit_dir", "k3s-agent.service").is_file()) self.assertTrue(self.file("github_runner_cluster_native_config_dir", "token").is_file()) - self.assertEqual([call for call in self.calls() if "k3s-agent" in call], []) + self.assertEqual([call for call in self.calls() if "k3s-agent" in call and " is-active " not in call], []) self.assertEqual([call for call in self.calls() if call.startswith("docker exec") and "get node " in call], []) def test_a_check_run_changes_nothing_and_says_what_a_real_run_would_do(self) -> None: @@ -676,7 +844,7 @@ def test_settings_a_native_node_needs_are_checked_first(self) -> None: ({"github_runner_cluster_native_k3s_sha256": "ABC"}, "must be empty or 64 lower-case hexadecimal digits"), ({"github_runner_cluster_k3s_token": ""}, "github_runner_cluster_k3s_token is not set"), ({"github_runner_cluster_k3s_version": "latest"}, "must be a pinned rancher/k3s tag"), - ({"github_runner_cluster_node_role": "server"}, "cannot be a server or bootstrap the cluster"), + ({"github_runner_cluster_regrade": True}, "a native server has none yet"), ({"github_runner_cluster_node_mode": "container"}, "must be one of: compose, native"), ({"github_runner_cluster_network_mode": "host"}, "which sets github_runner_cluster_node_mode: native"), ] @@ -717,9 +885,10 @@ def test_a_deliberate_jump_is_allowed_when_asked_for(self) -> None: def test_the_run_ends_with_a_node_that_a_server_sees_ready_and_a_node_that_is_not_is_reported(self) -> None: self.assertEqual(self.run_role().returncode, 0) - reads = [call for call in self.calls() if call.startswith("docker exec") and "kubectl get node " in call or "kubectl get lease" in call] + reads = [call for call in self.calls() if call.startswith("docker exec") and (" kubectl get node " in call or " kubectl get lease " in call)] self.assertEqual(len(reads), 2) - stand_in(self.stubs / "bin" / "docker", 'case "$*" in\n "ps "*) echo running ;;\n *"kubectl get nodes"*) printf "server1 v1.34.1+k3s1 true\\n" ;;\n *"kubectl get node "*) echo False ;;\n *"kubectl get lease"*) echo 2000-01-01T00:00:00Z ;;\nesac\n') + (self.state / "node-ready").write_text("False,") + (self.state / "lease").write_text("2000-01-01T00:00:00Z") result = self.run_role(host_vars={"github_runner_cluster_native_ready_timeout": 1}, github_runner_cluster_k3s_token="another-token-0123456789") self.assertNotEqual(result.returncode, 0) self.assertIn("did not come up as a node of the cluster", re.sub(r"\s+", " ", result.stdout)) @@ -743,10 +912,335 @@ def test_a_node_joining_a_cluster_outside_the_inventory_cannot_be_confirmed_by_a self.assertEqual(result.returncode, 0, result.stdout + result.stderr) text = re.sub(r"\s+", " ", result.stdout) self.assertIn("bootstraps no cluster of its own, so no server of this inventory can confirm that laptop is Ready", text) - self.assertEqual([call for call in self.calls() if call.startswith("docker")], []) - config = yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "config.yaml").read_text()) + self.assertEqual([call for call in self.calls() if call.startswith(("docker", "kubectl"))], []) + config = yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "agent.yaml").read_text()) self.assertEqual(config["server"], "https://elsewhere.example.ts.net:6443") + # --- A native server --- + + def server_tree(self) -> Path: + """Lay out a k3s server's data directory as k3s leaves it once its etcd has run: the datastore under server/db (etcd's member, its name and config files and the scheduled snapshots) beside the server's own certificates and token, which stay where they are. Returns the WAL file, whose bytes a test can follow.""" + data = self.paths["github_runner_cluster_native_data_dir"] + files = { + "server/db/etcd/config": "heartbeat-interval: 500\nelection-timeout: 5000\n", + "server/db/etcd/name": "laptop-0a1b2c3d", + "server/db/etcd/member/snap/db": "bolt", + "server/db/etcd/member/wal/0000000000000000-0000000000000000.wal": "wal-bytes", + "server/db/snapshots/etcd-snapshot-laptop-1767225600": "snapshot", + "server/tls/server-ca.crt": "ca", + "server/token": "K10secure::server:token", + "server/cred/admin.kubeconfig": "kubeconfig", + "agent/etc/containerd/config.toml": "containerd", + } + for name, content in files.items(): + path = data / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + (data / "server").chmod(0o700) + return data / "server/db/etcd/member/wal/0000000000000000-0000000000000000.wal" + + def starts(self) -> list[str]: + log = self.stubs / "starts.log" + return log.read_text().splitlines() if log.exists() else [] + + def unit_calls(self, since: int = 0) -> list[str]: + return [call for call in self.calls()[since:] if call.startswith("systemctl") and " is-" not in call and "daemon-reload" not in call] + + def role_file(self) -> str: + return self.file("github_runner_cluster_native_data_dir", "fleet-role").read_text() + + def test_a_native_bootstrap_server_starts_a_new_cluster_with_the_docker_servers_flags(self) -> None: + result = self.run_role(host_vars={"github_runner_cluster_bootstrap": True, "github_runner_cluster_etcd_heartbeat_interval": 500, "github_runner_cluster_etcd_election_timeout": 5000}, only_the_node=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + config_dir = "github_runner_cluster_native_config_dir" + server = yaml.safe_load(self.file(config_dir, "server.yaml").read_text()) + self.assertEqual(server, { + "token-file": str(self.file(config_dir, "token")), + "vpn-auth-file": str(self.file(config_dir, "vpn-auth")), + "node-name": "laptop", + "disable": ["traefik"], + "disable-network-policy": True, + "write-kubeconfig-mode": "0600", + "tls-san": [f"laptop.{DOMAIN}"], + "etcd-arg": ["heartbeat-interval=500", "election-timeout=5000"], + "resolv-conf": "/run/github-runner-k3s/resolv.conf", + }) + self.assertEqual(yaml.safe_load(self.file(config_dir, "server.yaml.d/10-join.yaml").read_text()), {"cluster-init": True}) + for name in ("server.yaml", "server.yaml.d/10-join.yaml"): + self.assertEqual(stat.S_IMODE(self.file(config_dir, name).stat().st_mode), 0o600, name) + self.assertEqual(self.role_file(), "server") + self.assertEqual(self.unit_calls(), ["systemctl enable --now tailscaled", "systemctl enable k3s", "systemctl start k3s"]) + self.assertEqual(self.starts(), ["start k3s server"]) + # The node is waited for through its own server, as root. + self.assertTrue(any(call.startswith("kubectl get node laptop") for call in self.calls())) + self.assertTrue(any(call.startswith("kubectl get lease") for call in self.calls())) + self.assertFalse(any(call.startswith("docker") for call in self.calls())) + + def test_the_server_unit_runs_k3s_server_under_its_role_condition_with_the_mesh_address(self) -> None: + self.assertEqual(self.run_role(host_vars={"github_runner_cluster_bootstrap": True}, only_the_node=True).returncode, 0) + text = self.file("github_runner_cluster_native_unit_dir", "k3s.service").read_text() + service = [line for line in text.split("[Service]")[1].split("[Install]")[0].splitlines() if line and not line.startswith("#")] + lib, config = self.paths["github_runner_cluster_native_lib_dir"], self.paths["github_runner_cluster_native_config_dir"] + self.assertIn(f"ExecCondition={lib}/fleet-role-is server {self.paths['github_runner_cluster_native_data_dir']}/fleet-role", service) + self.assertIn(f"ExecStartPre={lib}/write-node-ip {config}/server.yaml.d/00-node-ip.yaml {config}/vpn-auth", service) + self.assertIn(f"ExecStart={self.paths['github_runner_cluster_native_bin_dir']}/k3s server --config {config}/server.yaml", service) + # Everything else is the agent unit's: one runs at a time, under the same limits. + agent = [line for line in self.file("github_runner_cluster_native_unit_dir", "k3s-agent.service").read_text().split("[Service]")[1].split("[Install]")[0].splitlines() if line and not line.startswith("#")] + self.assertEqual([line for line in service if not line.startswith(("ExecCondition=", "ExecStart"))], [line for line in agent if not line.startswith(("ExecCondition=", "ExecStart"))]) + for helper in ("write-node-ip", "fleet-role-is"): + self.assertEqual(self.file("github_runner_cluster_native_lib_dir", helper).read_text(), (ROLE / "files" / "native" / f"{helper}.sh").read_text()) + + def run_three(self, *args: str, laptop: dict[str, Any] | None = None, check: bool = False, **extra: Any) -> subprocess.CompletedProcess[str]: + """Run the role on laptop in a group of the Docker servers server1 and server2 and laptop, a native server unless ``laptop`` says otherwise.""" + return self.run_role(*args, host_vars={"github_runner_cluster_node_role": "server", **(laptop or {})}, check=check, second_server=True, **extra) + + def test_a_native_server_joins_a_cluster_of_docker_servers_as_they_do(self) -> None: + (self.state / "voters").write_text("server1,True,true\nserver2,True,true\n") + result = self.run_three() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + config_dir = "github_runner_cluster_native_config_dir" + self.assertEqual(yaml.safe_load(self.file(config_dir, "server.yaml.d/10-join.yaml").read_text()), {"server": f"https://server1.{DOMAIN}:6443"}) + # The test inventory lists laptop first (yaml.safe_dump sorts the hosts), and the SANs are in inventory order. + self.assertEqual(yaml.safe_load(self.file(config_dir, "server.yaml").read_text())["tls-san"], [f"laptop.{DOMAIN}", f"server1.{DOMAIN}", f"server2.{DOMAIN}"]) + unit = self.file("github_runner_cluster_native_unit_dir", "k3s.service").read_text() + self.assertIn(f'Environment="GITHUB_RUNNER_JOIN_HOST=server1.{DOMAIN}"', unit) + # The voters were read through the bootstrap server's container before the server started. + calls = self.calls() + read = next(index for index, call in enumerate(calls) if call.startswith("docker exec github-runner-k3s kubectl get nodes") and "etcd}" in call) + self.assertLess(read, calls.index("systemctl start k3s")) + self.assertEqual(self.starts(), ["start k3s server"]) + + def test_a_native_server_does_not_join_while_another_voter_is_not_ready(self) -> None: + (self.state / "voters").write_text("server1,True,true\nserver2,False,true\n") + result = self.run_three() + self.assertNotEqual(result.returncode, 0) + self.assertIn("voter server2 is not Ready (its Ready condition is False)", re.sub(r"\s+", " ", result.stdout)) + self.assertEqual(self.starts(), []) + self.assertNotIn("systemctl start k3s", self.calls()) + + def test_an_agent_is_promoted_in_place_once_its_node_is_empty(self) -> None: + self.assertEqual(self.run_three(laptop={"github_runner_cluster_node_role": "agent", "github_runner_cluster_voter_eligible": False}).returncode, 0) + # A three-host group with laptop as an agent has one Docker server: the first run is the laptop as an agent of server1, the second makes it the third server. + self.assertEqual(self.role_file(), "agent") + password = self.root / "etc" / "rancher" / "node" / "password" + password.parent.mkdir(parents=True) + password.write_text("node-password") + before = len(self.calls()) + result = self.run_three() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + calls = self.calls()[before:] + order = [ + "kubectl cordon laptop", + next(call for call in calls if call.startswith("kubectl get pods --all-namespaces --selector actions-ephemeral-runner=True --field-selector spec.nodeName=laptop,status.phase!=Succeeded,status.phase!=Failed")), + next(call for call in calls if call.startswith("kubectl drain laptop --ignore-daemonsets --delete-emptydir-data")), + "systemctl disable --now k3s-agent", + "systemctl start k3s", + "kubectl uncordon laptop", + ] + self.assertEqual([call for call in calls if call in order], order) + self.assertEqual(self.role_file(), "server") + self.assertEqual(self.starts()[-1], "start k3s server") + self.assertFalse((self.state / "active-k3s-agent").exists()) + self.assertFalse((self.state / "enabled-k3s-agent").exists()) + self.assertEqual(password.read_text(), "node-password") + # The cordon and drain went through another server, and the uncordon through this one, which now serves the API. + self.assertIn("docker exec github-runner-k3s kubectl cordon laptop", calls) + self.assertNotIn("docker exec github-runner-k3s kubectl uncordon laptop", calls) + + def test_a_promotion_whose_runner_pods_do_not_finish_is_abandoned_with_the_node_uncordoned(self) -> None: + self.assertEqual(self.run_three(laptop={"github_runner_cluster_node_role": "agent"}).returncode, 0) + (self.state / "runner-pods").write_text("pod/runner-abc12\n") + before = len(self.calls()) + result = self.run_three(laptop={"github_runner_cluster_native_runner_wait_timeout": 1}) + self.assertNotEqual(result.returncode, 0) + self.assertIn("its node could not be emptied", re.sub(r"\s+", " ", result.stdout)) + calls = self.calls()[before:] + self.assertIn("kubectl uncordon laptop", calls) + self.assertNotIn("systemctl disable --now k3s-agent", calls) + self.assertEqual(self.role_file(), "agent") + self.assertTrue((self.state / "active-k3s-agent").exists()) + + def test_a_running_server_restarts_only_while_every_other_voter_is_ready_and_a_refused_restart_is_made_later(self) -> None: + self.assertEqual(self.run_three().returncode, 0) + (self.state / "voters").write_text("server1,True,true\nserver2,Unknown,true\nlaptop,True,true\n") + taints = {"github_runner_cluster_node_taints": ["dedicated=ci:NoSchedule"]} + before = len(self.calls()) + refused = self.run_three(laptop=taints) + self.assertNotEqual(refused.returncode, 0) + self.assertIn("Not going ahead with the restart of k3s on laptop: voter server2 is not Ready", re.sub(r"\s+", " ", refused.stdout)) + self.assertEqual(self.unit_calls(before), []) + # The voters were read through this server's own k3s, which is running. + self.assertTrue(any(call.startswith("kubectl get nodes") and "etcd}" in call for call in self.calls()[before:])) + self.assertFalse(any(call.startswith("docker exec") and "etcd}" in call for call in self.calls()[before:])) + (self.state / "voters").write_text("server1,True,true\nserver2,True,true\nlaptop,True,true\n") + before = len(self.calls()) + result = self.run_three(laptop=taints) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.unit_calls(before), ["systemctl restart k3s"]) + before = len(self.calls()) + self.assertEqual(self.run_three(laptop=taints).returncode, 0) + self.assertEqual(self.unit_calls(before), []) + + def test_a_check_run_reports_a_promotion_and_makes_none(self) -> None: + self.assertEqual(self.run_three(laptop={"github_runner_cluster_node_role": "agent"}).returncode, 0) + before = len(self.calls()) + result = self.run_three(check=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("promote k3s on laptop", result.stdout) + self.assertEqual([call for call in self.mutations(before) if not call.startswith(("apt-get", "systemctl enable --now tailscaled"))], []) + self.assertEqual(self.role_file(), "agent") + + def demote(self, voters: str = "server1,True,true\nserver2,True,true\nserver3,True,true\nlaptop,True,true\n") -> subprocess.CompletedProcess[str]: + """Run laptop as a server, lay out its datastore as k3s leaves it, then run it as an agent with the cluster's voters read as ``voters``.""" + (self.state / "voters").write_text("server1,True,true\nserver2,True,true\n") + self.assertEqual(self.run_three().returncode, 0) + self.wal = self.server_tree() + older = self.paths["github_runner_cluster_native_data_dir"] / "server" / "db.demoted" / "etcd" / "name" + older.parent.mkdir(parents=True) + older.write_text("laptop-from-an-earlier-demotion") + (self.state / "voters").write_text(voters) + self.demote_before = len(self.calls()) + return self.run_three(laptop={"github_runner_cluster_node_role": "agent"}) + + def assert_demotion_refused(self, voters: str, message: str) -> None: + result = self.demote(voters) + before = self.demote_before + self.assertNotEqual(result.returncode, 0) + self.assertIn(message, re.sub(r"\s+", " ", result.stdout)) + self.assertEqual(self.role_file(), "server") + self.assertTrue((self.state / "active-k3s").exists()) + self.assertTrue(self.wal.exists()) + # The Tailscale package is installed on every run of this test's host, whose dpkg-query stand-in reports none. + self.assertEqual([call for call in self.mutations(before) if not call.startswith(("apt-get", "systemctl enable --now tailscaled"))], []) + + def test_a_demotion_that_would_leave_two_voters_is_refused(self) -> None: + self.assert_demotion_refused("server1,True,true\nserver2,True,true\nlaptop,True,true\n", "only 2 other voters (server1, server2) would remain, fewer than the 3") + + def test_a_demotion_while_another_voter_is_not_ready_is_refused(self) -> None: + self.assert_demotion_refused("server1,True,true\nserver2,True,true\nserver3,False,true\nlaptop,True,true\n", "voter server3 is not Ready") + + def test_a_server_is_demoted_once_its_member_is_removed_with_its_datastore_moved_aside_and_its_identity_kept(self) -> None: + before = len(self.calls()) + result = self.demote() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + data = self.paths["github_runner_cluster_native_data_dir"] + self.assertEqual(self.role_file(), "agent") + self.assertFalse((data / "server" / "db").exists()) + moved = data / "server" / "db.demoted" + self.assertEqual((moved / "etcd/member/wal/0000000000000000-0000000000000000.wal").read_text(), "wal-bytes") + self.assertEqual((moved / "snapshots/etcd-snapshot-laptop-1767225600").read_text(), "snapshot") + # The newest copy replaces the one an earlier demotion left. + self.assertEqual((moved / "etcd" / "name").read_text(), "laptop-0a1b2c3d") + for kept in ("server/tls/server-ca.crt", "server/token", "agent/etc/containerd/config.toml"): + self.assertTrue((data / kept).is_file(), kept) + calls = self.calls()[before:] + order = [ + "kubectl cordon laptop", + "kubectl annotate --overwrite node laptop etcd.k3s.cattle.io/remove=true", + "systemctl disable --now k3s", + "kubectl label node laptop node-role.kubernetes.io/control-plane- node-role.kubernetes.io/etcd- node-role.kubernetes.io/master-", + "kubectl annotate node laptop etcd.k3s.cattle.io/remove- etcd.k3s.cattle.io/removed-node-name- etcd.k3s.cattle.io/node-name- etcd.k3s.cattle.io/node-address-", + "systemctl enable --now k3s-agent", + "kubectl uncordon laptop", + ] + self.assertEqual([call for call in calls if call in order], order) + # Everything went through another server's API, since this one stops. + for call in order: + if call.startswith("kubectl"): + self.assertIn("docker exec github-runner-k3s " + call, calls) + self.assertEqual(self.starts()[-1], "enable k3s-agent agent") + self.assertFalse((self.state / "active-k3s").exists()) + + def test_a_server_whose_member_is_not_removed_is_not_stopped(self) -> None: + (self.state / "member-stays").touch() + result = self.demote_quickly() + self.assertNotEqual(result.returncode, 0) + self.assertTrue((self.state / "active-k3s").exists()) + self.assertTrue(self.wal.exists()) + # The role file already names the agent, so the server is not started again by Restart=always or a reboot; a rerun finishes the demotion. + self.assertEqual(self.role_file(), "agent") + (self.state / "member-stays").unlink() + result = self.run_three(laptop={"github_runner_cluster_node_role": "agent"}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertFalse((self.state / "active-k3s").exists()) + self.assertFalse(self.wal.exists()) + + def demote_quickly(self) -> subprocess.CompletedProcess[str]: + (self.state / "voters").write_text("server1,True,true\nserver2,True,true\n") + self.assertEqual(self.run_three().returncode, 0) + self.wal = self.server_tree() + (self.state / "voters").write_text("server1,True,true\nserver2,True,true\nserver3,True,true\nlaptop,True,true\n") + return self.run_three(laptop={"github_runner_cluster_node_role": "agent", "github_runner_cluster_native_ready_timeout": 1}) + + def test_a_former_member_never_starts_a_second_cluster(self) -> None: + moved = self.paths["github_runner_cluster_native_data_dir"] / "server" / "db.demoted" / "etcd" + moved.mkdir(parents=True) + (moved / "name").write_text("laptop-0a1b2c3d") + alone = self.run_role(host_vars={"github_runner_cluster_bootstrap": True}, only_the_node=True) + self.assertNotEqual(alone.returncode, 0) + self.assertIn("with no server to join", re.sub(r"\s+", " ", alone.stdout)) + self.assertEqual(self.starts(), []) + (self.state / "voters").write_text("server1,True,true\nserver2,True,true\n") + result = self.run_three(laptop={"github_runner_cluster_bootstrap": True}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "server.yaml.d/10-join.yaml").read_text()), {"server": f"https://server1.{DOMAIN}:6443"}) + + def test_a_bootstrap_host_with_no_datastore_joins_a_cluster_another_server_runs(self) -> None: + (self.state / "voters").write_text("server1,True,true\nserver2,True,true\n") + result = self.run_three(laptop={"github_runner_cluster_bootstrap": True}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "server.yaml.d/10-join.yaml").read_text()), {"server": f"https://server1.{DOMAIN}:6443"}) + + def test_a_bootstrap_host_with_no_datastore_starts_a_cluster_only_when_every_other_server_is_stopped(self) -> None: + (self.state / "docker-stopped").touch() + result = self.run_three(laptop={"github_runner_cluster_bootstrap": True}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "server.yaml.d/10-join.yaml").read_text()), {"cluster-init": True}) + + def test_a_bootstrap_host_that_cannot_tell_whether_a_cluster_exists_is_refused(self) -> None: + (self.state / "nodes-unreadable").touch() + result = self.run_three(laptop={"github_runner_cluster_bootstrap": True}, github_runner_cluster_allow_k3s_version_jump=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("would start a new cluster (cluster-init) unless another server already runs one, and that cannot be told", re.sub(r"\s+", " ", result.stdout)) + self.assertEqual(self.mutations(), []) + + def test_an_agent_from_before_the_role_file_keeps_running_as_an_agent(self) -> None: + unit = self.file("github_runner_cluster_native_unit_dir", "k3s-agent.service") + unit.write_text("[Service]\nExecStart=/usr/local/bin/k3s agent\n") + old = self.file("github_runner_cluster_native_config_dir", "config.yaml") + old.parent.mkdir(parents=True) + old.write_text("server: https://server1.example.ts.net:6443\n") + for name in ("enabled-k3s-agent", "active-k3s-agent"): + (self.state / name).touch() + self.set_package(TAILSCALE_VERSION) + self.set_tailscale("Running", DOMAIN) + result = self.run_role() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.role_file(), "agent") + self.assertFalse(old.exists()) + self.assertEqual(self.starts(), ["restart k3s-agent agent"]) + + def test_an_eligible_agent_is_staged_as_a_standby_server_and_keeps_its_role(self) -> None: + result = self.run_role(host_vars={"github_runner_cluster_voter_eligible": True}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.role_file(), "agent") + server = yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "server.yaml").read_text()) + self.assertEqual(server["tls-san"], [f"laptop.{DOMAIN}", f"server1.{DOMAIN}"]) + self.assertNotIn("systemctl start k3s", self.calls()) + # The inventory gives an eligible host only the role it starts with. + before = len(self.calls()) + result = self.run_three(laptop={"github_runner_cluster_voter_eligible": True}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.role_file(), "agent") + self.assertEqual(self.unit_calls(before), []) + + def test_a_native_bootstrap_host_is_refused_under_headscale_in_cluster(self) -> None: + result = self.run_role(host_vars={"github_runner_cluster_bootstrap": True, "github_runner_cluster_mesh": "headscale_in_cluster"}) + self.assertNotEqual(result.returncode, 0) + self.assertIn("whose bootstrap server runs the Headscale server as a static pod", re.sub(r"\s+", " ", result.stdout)) + self.assertEqual(self.calls(), []) + + if __name__ == "__main__": unittest.main() From 33afffd9ef44a55673dbeb385dd4688b1a7ab17c Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 3 Nov 2026 08:42:28 +0000 Subject: [PATCH 04/10] test(mesh): promote a native server in place beside the Docker servers The native_server scenario runs a native host as a privileged systemd container that the role manages over Ansible's docker connection. It joins three Docker servers as an agent, then the inventory makes it a server: its k3s must join etcd as a learner and be promoted to a voter with its mesh address as its peer URL, while writes through another server keep succeeding, and a rerun must not restart it. --- .github/workflows/mesh-integration.yml | 3 +- tests/mesh/native/Dockerfile | 9 ++ tests/mesh/run.sh | 149 ++++++++++++++++++++++++- 3 files changed, 157 insertions(+), 4 deletions(-) create mode 100644 tests/mesh/native/Dockerfile diff --git a/.github/workflows/mesh-integration.yml b/.github/workflows/mesh-integration.yml index 88375cca..d0525494 100644 --- a/.github/workflows/mesh-integration.yml +++ b/.github/workflows/mesh-integration.yml @@ -1,6 +1,6 @@ name: Mesh integration -# Brings up three k3s nodes in Docker and joins them through Headscale with the cluster role (tests/mesh/run.sh), once per scenario: headscale_hosted, headscale_in_cluster, and the headscale_existing policy check, plus the role's refusal of a Docker Compose release that recreates the containers it has just built. headscale_in_cluster also runs with github_runner_cluster_docker_context pinned while the current Docker context points at nothing, which covers the cluster role's own docker calls and the recovery playbook's, and context_checks checks the role refuses a pin it cannot honour. regrade turns memory-pressure regrading on and steps a server down to etcd-only under (simulated) pressure and back up, checking that it keeps its etcd seat and vote and that no other server steps meanwhile. Only on changes that can affect how nodes join the mesh. +# Brings up three k3s nodes in Docker and joins them through Headscale with the cluster role (tests/mesh/run.sh), once per scenario: headscale_hosted, headscale_in_cluster, and the headscale_existing policy check, plus the role's refusal of a Docker Compose release that recreates the containers it has just built. headscale_in_cluster also runs with github_runner_cluster_docker_context pinned while the current Docker context points at nothing, which covers the cluster role's own docker calls and the recovery playbook's, and context_checks checks the role refuses a pin it cannot honour. regrade turns memory-pressure regrading on and steps a server down to etcd-only under (simulated) pressure and back up, checking that it keeps its etcd seat and vote and that no other server steps meanwhile. native_server joins a native host (a systemd container) as an agent beside three Docker servers, then promotes it to a server in place: it must join etcd as a learner, be promoted to a voter, and keep the cluster writable throughout. Only on changes that can affect how nodes join the mesh. # # Runs on GitHub-hosted ubuntu-latest, whose Docker daemon runs the privileged containers the test needs (k3s, and Tailscale inside it). The work directory sits under runner.temp, which the job always has write access to. @@ -38,6 +38,7 @@ jobs: - {scenario: hosted, compose: v5.5.1} - {scenario: in_cluster, compose: v5.5.1} - {scenario: regrade, compose: v5.5.1} + - {scenario: native_server, compose: v5.5.1} - {scenario: hosted, compose: v2.39.0} - {scenario: compose_faulty, compose: v2.38.2} - {scenario: in_cluster, compose: v5.5.1, pin: true} diff --git a/tests/mesh/native/Dockerfile b/tests/mesh/native/Dockerfile new file mode 100644 index 00000000..eed08f83 --- /dev/null +++ b/tests/mesh/native/Dockerfile @@ -0,0 +1,9 @@ +# A stand-in for a Linux host that runs a native k3s node (github_runner_cluster_node_mode: native), for tests/mesh/run.sh native_server: Ubuntu 24.04 with systemd as PID 1, as a dedicated host boots, and what the cluster role needs to manage it over Ansible's docker connection (python3, sudo for become). The role installs Tailscale and k3s itself, as on a real host. Run privileged, with tmpfs on /run and /run/lock and a volume for k3s's data directory, so containerd's overlay snapshots sit on the volume's filesystem rather than on this container's own overlay, which overlayfs cannot stack on. +FROM ubuntu:24.04 +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends systemd systemd-sysv dbus python3 sudo ca-certificates curl gnupg iproute2 iptables kmod \ + && rm -rf /var/lib/apt/lists/* \ + && systemctl mask getty.target console-getty.service systemd-firstboot.service systemd-udevd.service systemd-modules-load.service +# The signal systemd shuts down on, which docker stop sends instead of SIGTERM. +STOPSIGNAL SIGRTMIN+3 +CMD ["/sbin/init"] diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index b3b7a411..8714ab71 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Mesh integration test for the github_runner_cluster role: stands up three k3s nodes in Docker on one machine, each in its own copy of the Compose project, and has the role join them through Headscale. Scenarios: hosted headscale_hosted: the role runs Headscale under Compose beside the first node. in_cluster headscale_in_cluster: the role runs Headscale inside the cluster, on the bootstrap server. existing headscale_existing: the policy check fails, with the entry to add, against a server whose policy lacks autoApprovers, and passes once it has one. Every scenario uses automatic server selection (three hosts, no overrides, so three servers). compose_faulty the role refuses a Docker Compose release that recreates the containers it has just built, before writing anything (run it under such a release; it is not in the default set). host_network each node runs in a Docker daemon of its own (a docker:dind container standing in for a VM with an address on the LAN, on the shared test network) with github_runner_cluster_network_mode host, joined through a Headscale server the test runs (headscale_existing); the nodes' Tailscale clients must then reach each other directly at those daemons' addresses, and the role must not publish any port; then the role must refuse to move two running nodes to the bridge network in one run, and move one there and back through its guard, the node keeping its mesh address and etcd membership. Linux only, since the control node reaches each daemon at its address on the test network. The cluster scenarios assert that etcd has three members, that every node is Ready, and that pods on different nodes reach each other over the mesh. +# Mesh integration test for the github_runner_cluster role: stands up three k3s nodes in Docker on one machine, each in its own copy of the Compose project, and has the role join them through Headscale. Scenarios: hosted headscale_hosted: the role runs Headscale under Compose beside the first node. in_cluster headscale_in_cluster: the role runs Headscale inside the cluster, on the bootstrap server. existing headscale_existing: the policy check fails, with the entry to add, against a server whose policy lacks autoApprovers, and passes once it has one. Every scenario uses automatic server selection (three hosts, no overrides, so three servers). native_server three Docker servers and a native host (a privileged systemd container, tests/mesh/native/Dockerfile, managed over Ansible's docker connection) that joins as an agent, is then made a server by the inventory and promoted in place: it must join etcd as a learner, be promoted to a voter with its mesh address as its peer URL, and keep the cluster writable throughout, and a rerun must not restart it. compose_faulty the role refuses a Docker Compose release that recreates the containers it has just built, before writing anything (run it under such a release; it is not in the default set). host_network each node runs in a Docker daemon of its own (a docker:dind container standing in for a VM with an address on the LAN, on the shared test network) with github_runner_cluster_network_mode host, joined through a Headscale server the test runs (headscale_existing); the nodes' Tailscale clients must then reach each other directly at those daemons' addresses, and the role must not publish any port; then the role must refuse to move two running nodes to the bridge network in one run, and move one there and back through its guard, the node keeping its mesh address and etcd membership. Linux only, since the control node reaches each daemon at its address on the test network. The cluster scenarios assert that etcd has three members, that every node is Ready, and that pods on different nodes reach each other over the mesh. # # With GRTEST_PIN_CONTEXT=1, the cluster scenarios run the role, and the in_cluster recovery playbook, with github_runner_cluster_docker_context pinned while the current Docker context they see points at nothing (tests/lib/docker_context.sh), so any docker call that ignored the pin fails the run; afterwards the current context must be unchanged. context_checks the role refuses a pinned context that does not exist, and a pinned context alongside DOCKER_HOST, before writing anything (not in the default set). # @@ -23,6 +23,13 @@ dind_image=docker:27.5.1-dind # Set by scenario_host_network: every node's containers are then on that node's own daemon, reached through a Docker context per node. host_network=0 busybox_image=busybox:1.37 +# The native_server scenario's native host: its container, image, address and data volume. Unset (empty native_role) in every other scenario, which then has no native host. +native_host=grtest-native +native_image=grtest-native-host +native_ip="${subnet_prefix}.20" +native_role="" +# The Docker hosts the inventory lists, by index; all of them unless a scenario leaves one out (native_server's second run, where node 3 keeps running as a voter outside the inventory). +inventory_nodes="" # Tries per pod pair, 5 seconds apart plus each try's own timeout: about two minutes for the mesh to converge after a restart. reach_attempts=8 # etcd's timing for every server, in milliseconds: a heartbeat interval and an election timeout of ten times it, above k3s's own defaults and so told apart from them, and above etcd's required five times. @@ -92,7 +99,8 @@ teardown() { if [ -f "${work}/headscale/docker-compose.yml" ]; then docker compose --project-directory "${work}/headscale" -f "${work}/headscale/docker-compose.yml" down -v >/dev/null 2>&1 || true fi - docker rm -f grtest-headscale grtest-hs-existing grtest-etcdctl >/dev/null 2>&1 || true + docker rm -f grtest-headscale grtest-hs-existing grtest-etcdctl "$native_host" >/dev/null 2>&1 || true + docker volume rm "${native_host}-k3s" >/dev/null 2>&1 || true for index in $(seq 1 "$node_count"); do docker rm -f -v "$(vm "$index")" >/dev/null 2>&1 || true docker context rm -f "$(vm "$index")" >/dev/null 2>&1 || true @@ -201,12 +209,20 @@ EOF echo " github_runner_cluster_headscale_accept_node_local_state: true" fi echo " hosts:" - for index in $(seq 1 "$node_count"); do + for index in ${inventory_nodes:-$(seq 1 "$node_count")}; do echo " $(node_name "$index"):" echo " github_runner_cluster_dir: ${work}/$(node_name "$index")" echo " github_runner_cluster_compose_project: $(node_name "$index")" echo " github_runner_cluster_container_name: $(container "$index")" done + if [ -n "$native_role" ]; then + echo " ${native_host}:" + echo " ansible_connection: community.docker.docker" + echo " ansible_host: ${native_host}" + echo " ansible_python_interpreter: /usr/bin/python3" + echo " github_runner_cluster_node_mode: native" + echo " github_runner_cluster_node_role: ${native_role}" + fi } > "${work}/inventory.yml" } @@ -913,6 +929,132 @@ EOF done } +# native_server: the native host is a container running systemd as PID 1 (tests/mesh/native/Dockerfile), on the test network at its own address, with k3s's data directory on a volume. The role reaches it over Ansible's docker connection and installs Tailscale and k3s on it, as on a real host. +start_native_host() { + docker build --quiet --tag "$native_image" "${repo_root}/tests/mesh/native" >/dev/null + docker run -d --privileged --name "$native_host" --hostname "$native_host" --network "$network" --ip "$native_ip" \ + --tmpfs /run --tmpfs /run/lock --volume "${native_host}-k3s:/var/lib/rancher/k3s" --volume /lib/modules:/lib/modules:ro \ + "$native_image" >/dev/null + wait_for "systemd to be running in ${native_host}" 120 native_systemd_up + # Docker answers the container's DNS at 127.0.0.11, which a pod cannot reach and write-resolv-conf.sh therefore leaves out, and the container has no systemd-resolved, so the kubelet's resolver file is pointed at public servers through a drop-in for both units. A real host's systemd-resolved lists its upstream servers instead; nothing else of the role's is changed. + # shellcheck disable=SC2016 # Expanded by the container's shell. + native_exec sh -c 'printf "nameserver 1.1.1.1\nnameserver 8.8.8.8\n" > /etc/grtest-upstream-resolv.conf + for unit in k3s k3s-agent; do + mkdir -p "/etc/systemd/system/${unit}.service.d" + printf "[Service]\nEnvironment=GITHUB_RUNNER_RESOLV_SOURCES=/etc/grtest-upstream-resolv.conf\n" > "/etc/systemd/system/${unit}.service.d/grtest.conf" + done' +} + +native_exec() { docker exec -i "$native_host" "$@"; } +native_systemd_up() { case "$(native_exec systemctl is-system-running 2>/dev/null)" in running | degraded) return 0 ;; *) return 1 ;; esac; } +native_unit_is() { [ "$(native_exec systemctl is-active "$1" /dev/null)" = "$2" ]; } + +# The etcd member list, as assert_cluster reads it, through node 1. +etcd_members() { + docker rm -f grtest-etcdctl >/dev/null 2>&1 || true + docker run --rm --name grtest-etcdctl --network "container:$(container 1)" --volumes-from "$(container 1)" --entrypoint etcdctl "$etcd_image" \ + --endpoints https://127.0.0.1:2379 --cacert /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \ + --cert /var/lib/rancher/k3s/server/tls/etcd/client.crt --key /var/lib/rancher/k3s/server/tls/etcd/client.key \ + member list -w simple /dev/null)" = True ] +} + +# Writes to the cluster through node 1 once a second until the stop file appears, recording each write as ok or failed; a write is a ConfigMap under a name of its own (write_marker). +write_throughout() { + local log="$1" stop="$2" + : > "$log" + until [ -e "$stop" ]; do + if write_marker 2>/dev/null; then echo ok >> "$log"; else echo "failed $(date -u +%H:%M:%S)" >> "$log"; fi + sleep 1 + done +} + +# Watches the member list until the stop file appears, recording every line that names the native host's member, so a learner seen at any point is on record. +watch_native_member() { + local log="$1" stop="$2" + : > "$log" + until [ -e "$stop" ]; do + etcd_members 2>/dev/null | grep -F ", ${native_host}-" >> "$log" || true + sleep 1 + done +} + +scenario_native_server() { + reset_environment + prepare_nodes hosted + start_native_host + native_role=agent + write_inventory hosted + log "Bringing up three Docker servers and the native host as an agent" + run_role + wait_for "${native_host} to be Ready as an agent" 300 node_is 1 "$native_host" Ready + native_unit_is k3s-agent active || fail "${native_host} does not run k3s-agent" + native_unit_is k3s inactive || fail "${native_host} runs k3s as an agent" + [ "$(native_exec cat /var/lib/rancher/k3s/fleet-role)" = agent ] || fail "the role file does not name the agent" + [ "$(kubectl_on 1 get nodes -l node-role.kubernetes.io/etcd=true --no-headers | wc -l | tr -d ' ')" = 3 ] || fail "expected three etcd nodes before the promotion" + local password + password=$(native_exec sha256sum /etc/rancher/node/password | cut -d' ' -f1) + + log "Promoting the native host to a server, with writes throughout; node 3 keeps running as a voter outside the inventory, so the inventory has three servers" + native_role=server + inventory_nodes="1 2" + write_inventory hosted + local writes="${work}/writes.log" members="${work}/native-member.log" stop="${work}/stop-writing" + rm -f "$stop" + write_throughout "$writes" "$stop" & + local writer=$! + watch_native_member "$members" "$stop" & + local watcher=$! + run_role --limit "$native_host" || { touch "$stop"; wait "$writer" "$watcher" || true; fail "the role did not promote ${native_host}"; } + wait_for "${native_host} to be a voting member of etcd" 300 native_is_voter + touch "$stop" + wait "$writer" "$watcher" + + log "Checking it joined etcd as a learner and was promoted, with its mesh address as its peer URL" + local list + list=$(etcd_members) + echo "$list" + [ "$(echo "$list" | grep -c ', started, ')" = 4 ] || fail "expected four started etcd members after the promotion" + echo "$list" | grep -F ", ${native_host}-" | grep -Eq ', https://100\.[0-9.]+:2380, .*, false$' \ + || fail "${native_host}'s member is not a voter with a mesh peer URL" + # k3s adds every joining server as a learner (MemberAddAsLearner) and the etcd leader logs each promotion; the member watch shows the learner unless it was promoted between two reads. + local promoted=0 index + for index in 1 2 3; do + if docker logs "$(container "$index")" 2>&1 | grep -F "Promoted learner ${native_host}-" >/dev/null; then promoted=1; fi + done + [ "$promoted" = 1 ] || fail "no server's k3s logged promoting ${native_host}'s learner" + if grep -Eq ', true$' "$members"; then log "The member watch saw ${native_host} as a learner"; else log "The member watch did not catch ${native_host} as a learner (promoted between reads); the promotion is in the servers' logs"; fi + native_exec journalctl -u k3s --no-pager | grep -F "Adding member ${native_host}-" >/dev/null || fail "${native_host}'s k3s did not add itself to etcd" + + log "Checking every write succeeded throughout" + local ok failed + ok=$(grep -c '^ok$' "$writes" || true) + failed=$(grep -c '^failed' "$writes" || true) + echo " ${ok} writes succeeded, ${failed} failed" + [ "$ok" -gt 0 ] || fail "no write was made during the promotion" + [ "$failed" = 0 ] || { grep '^failed' "$writes"; fail "writes failed during the promotion"; } + + log "Checking the host now runs the server unit only, under the same identity" + native_unit_is k3s active || fail "${native_host} does not run k3s" + native_unit_is k3s-agent inactive || fail "${native_host} still runs k3s-agent" + [ "$(native_exec cat /var/lib/rancher/k3s/fleet-role)" = server ] || fail "the role file does not name the server" + [ "$(native_exec sha256sum /etc/rancher/node/password | cut -d' ' -f1)" = "$password" ] || fail "${native_host}'s node password changed" + [ "$(kubectl_on 1 get nodes --no-headers | grep -c "^${native_host} ")" = 1 ] || fail "${native_host} is not exactly one Node" + native_exec cat /etc/rancher/k3s/server.yaml.d/00-node-ip.yaml | grep -Eq '^node-ip: "100\.' || fail "the node-ip drop-in does not hold the mesh address" + # Restart=always must not start the agent unit again, whose condition now fails. + native_exec systemctl start k3s-agent || true + native_unit_is k3s-agent inactive || fail "k3s-agent started on a server" + + log "Rerunning the role on the native server, which must not restart it" + local before + before=$(native_exec systemctl show k3s --property ActiveEnterTimestampMonotonic) + run_role --limit "$native_host" + [ "$before" = "$(native_exec systemctl show k3s --property ActiveEnterTimestampMonotonic)" ] || fail "rerunning the role restarted ${native_host}'s server" +} + scenarios=("$@") [ "${#scenarios[@]}" -gt 0 ] || scenarios=(existing hosted in_cluster) for scenario in "${scenarios[@]}"; do @@ -921,6 +1063,7 @@ for scenario in "${scenarios[@]}"; do hosted | in_cluster) scenario_cluster "$scenario" ;; existing) scenario_existing ;; regrade) scenario_regrade ;; + native_server) scenario_native_server ;; host_network) scenario_host_network ;; compose_faulty) scenario_compose_faulty ;; context_checks) scenario_context_checks ;; From a5674a35cbebbec5390bd051fa101ead53245ac2 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 3 Nov 2026 08:53:54 +0000 Subject: [PATCH 05/10] test(mesh): bring the Docker servers up before the native host joins them A play runs a native host's tasks before any Docker host's, since the role's first task includes them and the linear strategy runs an include for its hosts while the others wait, so the native agent waited on a bootstrap server that did not exist yet. The scenario now starts the Docker servers in a run of their own, the role README says to do the same, and a failure prints the native host's units, role file, Tailscale status and journal. --- roles/github_runner_cluster/README.md | 2 +- tests/mesh/run.sh | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index aa10b112..14ac59a9 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -76,7 +76,7 @@ The manual Secret deletion is for a node that is genuinely new under an old name ## Native nodes -`github_runner_cluster_node_mode: native` in a host's `host_vars` runs that host's k3s node directly on the host, as the systemd service `k3s-agent` (an agent) or `k3s` (a server), with no Docker or Compose layer. The default, `compose`, is everything else in this README. A node under Compose nests containerd inside a container, so every image layer is overlayfs on overlayfs and pod egress goes through the container's NAT as well; a native node's containerd uses the host's filesystem and network. It is for a Linux host that systemd runs. A native host is a server only when it asks to be: `github_runner_cluster_node_role: server`, or `github_runner_cluster_bootstrap: true` to bootstrap the cluster. `tasks/topology.yml` never gives it a server place automatically and does not count it towards the automatic server number, so an inventory whose native hosts set neither keeps all of them agents whatever `github_runner_cluster_max_servers` allows; the server count with them must still be odd. See [Native servers](#native-servers). It also refuses a native host that sets `github_runner_cluster_network_mode: host`, since that setting puts a Docker node's container on the Docker host's network and a native node has no container (it already runs on the host's own network, with the host's own Tailscale client). Add the host to the cluster group like any other (its place in the group does not matter, since it never takes an automatic server place), set `github_runner_cluster_node_mode: native` and `github_runner_cluster_tailnet_domain` for it, and run the playbook with `--limit` to it. Its tasks that change the host use `become` on their own, so the connecting user needs passwordless sudo or `--ask-become-pass`. `examples/native-agent/inventory.yml` is a starting point, and `examples/native-server/inventory.yml` for a native server and a standby. Preparing the host itself (the kernel modules and sysctls k3s needs, not suspending, a quiet Wi-Fi radio, a headless boot, security updates that never reboot it) is the `github_runner_linux_host` role's, which `playbooks/site.yml` runs first; k3s loads the modules and sets most of the sysctls itself when it starts, so the node does not depend on it. +`github_runner_cluster_node_mode: native` in a host's `host_vars` runs that host's k3s node directly on the host, as the systemd service `k3s-agent` (an agent) or `k3s` (a server), with no Docker or Compose layer. The default, `compose`, is everything else in this README. A node under Compose nests containerd inside a container, so every image layer is overlayfs on overlayfs and pod egress goes through the container's NAT as well; a native node's containerd uses the host's filesystem and network. It is for a Linux host that systemd runs. A native host is a server only when it asks to be: `github_runner_cluster_node_role: server`, or `github_runner_cluster_bootstrap: true` to bootstrap the cluster. `tasks/topology.yml` never gives it a server place automatically and does not count it towards the automatic server number, so an inventory whose native hosts set neither keeps all of them agents whatever `github_runner_cluster_max_servers` allows; the server count with them must still be odd. See [Native servers](#native-servers). It also refuses a native host that sets `github_runner_cluster_network_mode: host`, since that setting puts a Docker node's container on the Docker host's network and a native node has no container (it already runs on the host's own network, with the host's own Tailscale client). Add the host to the cluster group like any other (its place in the group does not matter, since it never takes an automatic server place), set `github_runner_cluster_node_mode: native` and `github_runner_cluster_tailnet_domain` for it, and run the playbook with `--limit` to it. A play runs a native host's tasks before any Docker host's (the role's first task includes them, and Ansible's linear strategy runs an include for the hosts that take it while the others wait), so a native node never waits on a Docker server the same run is still to start: bring the servers it joins up first, in a run of their own. Its tasks that change the host use `become` on their own, so the connecting user needs passwordless sudo or `--ask-become-pass`. `examples/native-agent/inventory.yml` is a starting point, and `examples/native-server/inventory.yml` for a native server and a standby. Preparing the host itself (the kernel modules and sysctls k3s needs, not suspending, a quiet Wi-Fi radio, a headless boot, security updates that never reboot it) is the `github_runner_linux_host` role's, which `playbooks/site.yml` runs first; k3s loads the modules and sets most of the sysctls itself when it starts, so the node does not depend on it. The role reads the same secrets and settings as the Docker path and joins the same cluster: `github_runner_cluster_k3s_token`, the mesh's join key and control server (`tasks/mesh/main.yml`, whichever provider is in use), the node name rules (`github_runner_cluster_node_name`, else the inventory name), the API port (`github_runner_cluster_api_port`, checked to be one port for the whole group by the same `tasks/api_port.yml`), the join URL `tasks/topology.yml` works out, `github_runner_cluster_node_taints` (written as `node-taint` in k3s's configuration, so they are applied each time the node registers, as they are under Compose), and the pins `github_runner_cluster_k3s_version` and `github_runner_cluster_tailscale_version`. Node labels are not set by this role under either mode: the ARC role labels nodes by name from the API (`github_runner_arc_labelled_nodes`), so a native node gets its labels the same way. diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index 8714ab71..a5a019cd 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -64,6 +64,11 @@ diagnose() { docker_on "$index" exec "grtest-node${index}-regrade-1" fleet-regrade status &1 | head -n 5 || true docker_on "$index" logs --tail 40 "grtest-node${index}-regrade-1" 2>&1 | cut -c1-400 || true done + if docker inspect "$native_host" >/dev/null 2>&1; then + echo "----- ${native_host}: units, role file, Tailscale and the last journal lines" + docker exec "$native_host" sh -c 'systemctl is-active k3s k3s-agent tailscaled; cat /var/lib/rancher/k3s/fleet-role; tailscale status | head -n 20' &1 || true + docker exec "$native_host" journalctl -u k3s -u k3s-agent -u tailscaled --no-pager -n 80 &1 | cut -c1-400 || true + fi if docker inspect grtest-headscale >/dev/null 2>&1; then echo "----- grtest-headscale: nodes and routes" docker exec grtest-headscale headscale nodes list &1 || true @@ -988,8 +993,10 @@ scenario_native_server() { start_native_host native_role=agent write_inventory hosted - log "Bringing up three Docker servers and the native host as an agent" - run_role + # A play runs a native host's tasks before any Docker host's (the role's first task includes them, and Ansible's linear strategy runs an include for its hosts while the others wait), so the servers a native node joins are brought up first, as the role README says. + log "Bringing up three Docker servers, then the native host as an agent" + run_role --limit "$(node_name 1),$(node_name 2),$(node_name 3)" + run_role --limit "$native_host" wait_for "${native_host} to be Ready as an agent" 300 node_is 1 "$native_host" Ready native_unit_is k3s-agent active || fail "${native_host} does not run k3s-agent" native_unit_is k3s inactive || fail "${native_host} runs k3s as an agent" From 9179ea224edec77d7f1e2f7722c1bb6660bf42d6 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 3 Nov 2026 09:06:25 +0000 Subject: [PATCH 06/10] test(mesh): read the native host's node by its own name node_status builds a Docker node's name from its index, so the scenario looked for a node that does not exist and timed out with the native agent Ready. --- tests/mesh/run.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index a5a019cd..adbe99ff 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -952,6 +952,8 @@ start_native_host() { native_exec() { docker exec -i "$native_host" "$@"; } native_systemd_up() { case "$(native_exec systemctl is-system-running 2>/dev/null)" in running | degraded) return 0 ;; *) return 1 ;; esac; } +# node_status names a Docker node by its index; the native host's node is named for the host. +native_node_ready() { [ "$(kubectl_on 1 get node "$native_host" --no-headers 2>/dev/null | awk '{print $2}')" = Ready ]; } native_unit_is() { [ "$(native_exec systemctl is-active "$1" /dev/null)" = "$2" ]; } # The etcd member list, as assert_cluster reads it, through node 1. @@ -997,7 +999,7 @@ scenario_native_server() { log "Bringing up three Docker servers, then the native host as an agent" run_role --limit "$(node_name 1),$(node_name 2),$(node_name 3)" run_role --limit "$native_host" - wait_for "${native_host} to be Ready as an agent" 300 node_is 1 "$native_host" Ready + wait_for "${native_host} to be Ready as an agent" 300 native_node_ready native_unit_is k3s-agent active || fail "${native_host} does not run k3s-agent" native_unit_is k3s inactive || fail "${native_host} runs k3s as an agent" [ "$(native_exec cat /var/lib/rancher/k3s/fleet-role)" = agent ] || fail "the role file does not name the agent" From 5a97a5964264c62ce1366f5f7bc153693d7a56e2 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 3 Nov 2026 10:06:31 +0000 Subject: [PATCH 07/10] test(native): split the native server tests into a module of their own CI deals whole test modules out to its shards, and the native node module had grown past one shard's time limit once it ran the role for servers too. The harness the two share moves to native_stand_ins.py, the server tests to test_native_server.py, and the agent tests stay where they were. --- roles/github_runner_cluster/README.md | 2 +- tests/unit/native_stand_ins.py | 322 +++++++++++++++ tests/unit/test_native_node.py | 567 +------------------------- tests/unit/test_native_server.py | 275 +++++++++++++ 4 files changed, 602 insertions(+), 564 deletions(-) create mode 100644 tests/unit/native_stand_ins.py create mode 100644 tests/unit/test_native_server.py diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index 14ac59a9..358ec719 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -140,7 +140,7 @@ A node on Wi-Fi drops off the network whenever the radio does, and it then flaps ### Tests -`tests/unit/test_native_node.py` covers it without a real host: the filters in `plugins/filter/cluster_native.py`, the helper scripts, the defaults, argument specs and this README, and the role's tasks (`tests/native/run.yml`) against stand-ins for `systemctl`, `tailscale`, `dpkg-query`, `apt-get`, `docker` and `kubectl` on `PATH`, a local server for the k3s release, and a server data directory laid out as k3s leaves it. For servers it checks a bootstrap with `cluster-init` and the Docker servers' flags, a join, the voter guard refusing a join and a restart (and the refused restart being made by the next run), a promotion in place, an abandoned promotion whose runner pods did not finish, a demotion refused while the member is a voter and made once it is removed, with the datastore moved aside and the identity kept, and a former member never starting a second cluster. `tests/mesh/run.sh native_server` runs a real native server as a privileged systemd container beside the Docker servers: it joins etcd as a learner, is promoted to a voter, and keeps the cluster writable throughout. It checks a first run, idempotence, `--check`, a changed setting restarting the service once, a client in another tailnet (refused, then logged out first when allowed), a client k3s did not log in, a refused downgrade, a wrong sum, an unsupported host, the secrets staying out of the output, and the node's readiness as a server reports it. +`tests/unit/test_native_node.py` and `tests/unit/test_native_server.py` cover it without a real host: the filters in `plugins/filter/cluster_native.py`, the helper scripts, the defaults, argument specs and this README, and the role's tasks (`tests/native/run.yml`) against stand-ins (`tests/unit/native_stand_ins.py`) for `systemctl`, `tailscale`, `dpkg-query`, `apt-get`, `docker` and `kubectl` on `PATH`, a local server for the k3s release, and a server data directory laid out as k3s leaves it. For servers it checks a bootstrap with `cluster-init` and the Docker servers' flags, a join, the voter guard refusing a join and a restart (and the refused restart being made by the next run), a promotion in place, an abandoned promotion whose runner pods did not finish, a demotion refused while the member is a voter and made once it is removed, with the datastore moved aside and the identity kept, and a former member never starting a second cluster. `tests/mesh/run.sh native_server` runs a real native server as a privileged systemd container beside the Docker servers: it joins etcd as a learner, is promoted to a voter, and keeps the cluster writable throughout. It checks a first run, idempotence, `--check`, a changed setting restarting the service once, a client in another tailnet (refused, then logged out first when allowed), a client k3s did not log in, a refused downgrade, a wrong sum, an unsupported host, the secrets staying out of the output, and the node's readiness as a server reports it. ## Docker host disk diff --git a/tests/unit/native_stand_ins.py b/tests/unit/native_stand_ins.py new file mode 100644 index 00000000..a7ab2bb1 --- /dev/null +++ b/tests/unit/native_stand_ins.py @@ -0,0 +1,322 @@ +"""Shared stand-ins and a base class for the tests that run the github_runner_cluster role against a native node (test_native_node.py, its agent, and test_native_server.py, its server): stand-ins for systemctl, tailscale, dpkg-query, apt-get, docker and kubectl first on the playbook's PATH, a local server for the k3s release, a k3s data directory laid out as a server leaves it, and the helpers that run the role through tests/native/run.yml, so that no real service, package, tailnet or cluster is touched.""" + +from __future__ import annotations + +import hashlib +import http.server +import json +import os +import re +import shutil +import stat +import subprocess +import tempfile +import threading +import unittest +from functools import partial +from pathlib import Path +from typing import Any + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +ROLE = ROOT / "roles" / "github_runner_cluster" +RUN = ROOT / "tests" / "native" / "run.yml" +ANSIBLE_PLAYBOOK = os.environ.get("ANSIBLE_PLAYBOOK", "ansible-playbook") + +TOKEN = "k3s-token-example-0123456789" +JOIN_KEY = "tskey-auth-example-0123456789" +DOMAIN = "example.ts.net" +K3S_TAG = "v1.34.1-k3s1" +K3S_RELEASE = "v1.34.1+k3s1" +TAILSCALE_VERSION = "1.102.4" +FACTS = {"system": "Linux", "architecture": "x86_64", "os_family": "Debian", "distribution": "Ubuntu", "distribution_release": "plucky", "service_mgr": "systemd"} + + +def tailscale_status(backend: str, domain: str | None) -> str: + """Return what `tailscale status --json` prints for a client in ``backend`` that is logged in to ``domain`` (none when it is logged out).""" + return json.dumps({"BackendState": backend, "CurrentTailnet": None if domain is None else {"Name": "x", "MagicDNSSuffix": domain, "MagicDNSEnabled": True}}) + + +def stand_in(path: Path, body: str) -> None: + """Write an executable shell script.""" + path.write_text("#!/bin/sh\n" + body) + path.chmod(path.stat().st_mode | stat.S_IXUSR) + + +class ReleaseServer(http.server.ThreadingHTTPServer): + """Serves a directory as the k3s release download and Tailscale repository.""" + + +class QuietHandler(http.server.SimpleHTTPRequestHandler): + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 - the signature of the method it replaces + return + + +class NativeRoleHarness(unittest.TestCase): + """Runs the role against a native node, stand-ins for the host's commands and a local k3s release. Holds no tests of its own.""" + + @classmethod + def setUpClass(cls) -> None: + cls.mirror = Path(tempfile.mkdtemp(prefix="native-mirror-")) + release = cls.mirror / K3S_RELEASE + release.mkdir() + # `k3s kubectl` reaches the test's kubectl stand-in, which answers for the cluster as the stand-in for `docker exec kubectl` does. + cls.binary = f'#!/bin/sh\nif [ "$1" = kubectl ]; then shift; exec "$STUB/bin/kubectl" "$@"; fi\necho "k3s version {K3S_RELEASE} (stand-in)"\n' + (release / "k3s").write_text(cls.binary) + cls.digest = hashlib.sha256(cls.binary.encode()).hexdigest() + (release / "sha256sum-amd64.txt").write_text(f"{cls.digest} k3s\n") + repository = cls.mirror / "ubuntu" + repository.mkdir() + (repository / "plucky.noarmor.gpg").write_text("key\n") + (repository / "plucky.tailscale-keyring.list").write_text("deb [signed-by=/usr/share/keyrings/tailscale-archive-keyring.gpg] https://pkgs.tailscale.com/stable/ubuntu plucky main\n") + cls.server = ReleaseServer(("127.0.0.1", 0), partial(QuietHandler, directory=str(cls.mirror))) + cls.url = f"http://127.0.0.1:{cls.server.server_address[1]}" + threading.Thread(target=cls.server.serve_forever, daemon=True).start() + + @classmethod + def tearDownClass(cls) -> None: + cls.server.shutdown() + cls.server.server_close() + shutil.rmtree(cls.mirror, True) + + def setUp(self) -> None: + self.tmp = Path(tempfile.mkdtemp(prefix="native-role-")) + self.addCleanup(shutil.rmtree, self.tmp, True) + self.stubs = self.tmp / "stubs" + self.state = self.stubs / "state" + self.state.mkdir(parents=True) + self.root = self.tmp / "host" + self.paths = { + "github_runner_cluster_native_bin_dir": self.root / "usr" / "local" / "bin", + "github_runner_cluster_native_lib_dir": self.root / "usr" / "local" / "lib" / "github-runner-k3s", + "github_runner_cluster_native_config_dir": self.root / "etc" / "rancher" / "k3s", + "github_runner_cluster_native_unit_dir": self.root / "etc" / "systemd" / "system", + "github_runner_cluster_native_data_dir": self.root / "var" / "lib" / "rancher" / "k3s", + "github_runner_cluster_native_apt_keyring": self.root / "usr" / "share" / "keyrings" / "tailscale-archive-keyring.gpg", + "github_runner_cluster_native_apt_source": self.root / "etc" / "apt" / "sources.list.d" / "tailscale.list", + } + for path in self.paths.values(): + path.parent.mkdir(parents=True, exist_ok=True) + # Directories every host has already: the role puts files in them and creates no others of the system's. + self.paths["github_runner_cluster_native_bin_dir"].mkdir(exist_ok=True) + self.paths["github_runner_cluster_native_unit_dir"].mkdir(exist_ok=True) + self.make_stand_ins() + self.set_tailscale(None, None) + self.facts = dict(FACTS) + + def make_stand_ins(self) -> None: + bin_dir = self.stubs / "bin" + bin_dir.mkdir() + log = 'echo "$(basename "$0") $*" >> "$STUB/calls.log"\n' + stand_in(bin_dir / "systemctl", log + r"""state="$STUB/state" +command=$1 +shift +now="" +quiet="" +for argument in "$@"; do + case "$argument" in + --quiet) quiet=1 ;; + --now) now=1 ;; + *) unit=$argument ;; + esac +done +join() { + # k3s runs `tailscale up` with the join key as it starts, which logs the host's client in to the cluster's tailnet. + if [ ! -e "$state/no-join" ]; then + printf '%s' '{"BackendState":"Running","CurrentTailnet":{"MagicDNSSuffix":"example.ts.net"}}' > "$state/tailscale.json" + fi +} +case "$command" in + # --quiet prints nothing, as systemctl's does. + is-enabled) if [ -e "$state/enabled-$unit" ]; then [ -n "$quiet" ] || echo enabled; exit 0; fi; [ -n "$quiet" ] || echo disabled; exit 1 ;; + is-active) if [ -e "$state/active-$unit" ]; then [ -n "$quiet" ] || echo active; exit 0; fi; [ -n "$quiet" ] || echo inactive; exit 3 ;; + enable) + touch "$state/enabled-$unit" + if [ -n "$now" ]; then touch "$state/active-$unit"; case "$unit" in k3s|k3s-agent) join ;; esac; fi ;; + disable) rm -f "$state/enabled-$unit"; if [ -n "$now" ]; then rm -f "$state/active-$unit"; fi ;; + start|restart) touch "$state/active-$unit"; case "$unit" in k3s|k3s-agent) join ;; esac ;; +esac +# What the role file named when a k3s unit was started, which is what its ExecCondition would have read. +case "$command:$unit" in + enable:k3s*|start:k3s*|restart:k3s*) if [ "$command" != enable ] || [ -n "$now" ]; then echo "$command $unit $(cat "$ROLE_FILE" 2>/dev/null || echo none)" >> "$STUB/starts.log"; fi ;; +esac +""") + stand_in(bin_dir / "tailscale", log + r"""case "$1" in + status) [ -e "$STUB/state/tailscale.json" ] || exit 1; cat "$STUB/state/tailscale.json" ;; + logout) printf '%s' '{"BackendState":"NeedsLogin","CurrentTailnet":null}' > "$STUB/state/tailscale.json" ;; +esac +""") + stand_in(bin_dir / "dpkg-query", r"""if [ -e "$STUB/state/tailscale-version" ]; then cat "$STUB/state/tailscale-version"; else exit 1; fi +""") + stand_in(bin_dir / "apt-get", log + r"""for argument in "$@"; do + case "$argument" in + tailscale=*) + printf '%s' "${argument#tailscale=}" > "$STUB/state/tailscale-version" + # The package's scripts start tailscaled, which has no login on a host that never had one. + if [ ! -e "$STUB/state/tailscale.json" ]; then printf '%s' '{"BackendState":"NeedsLogin","CurrentTailnet":null}' > "$STUB/state/tailscale.json"; fi ;; + esac +done +""") + # `docker exec kubectl ...` reaches the same kubectl stand-in as `k3s kubectl ...`. + stand_in(bin_dir / "docker", log + r"""case "$*" in + "ps "*) if [ -e "$STUB/state/docker-stopped" ]; then echo exited; else echo running; fi ;; + "exec "*" kubectl "*) shift 3; exec "$STUB/bin/kubectl" "$@" ;; + *) echo "docker stand-in: unexpected $*" >&2; exit 1 ;; +esac +""") + # The cluster as the API reports it, from files in the state directory the tests write: the nodes with their versions and live roles, the voters' readiness, this node's readiness and etcd vote, its member annotations, and the runner pods on it. + stand_in(bin_dir / "kubectl", log + r"""state="$STUB/state" +case "$*" in + "get nodes "*kubeletVersion*) + if [ -e "$state/nodes-unreadable" ]; then echo "The connection to the server was refused" >&2; exit 1; fi + if [ -e "$state/nodes" ]; then cat "$state/nodes"; else printf 'server1 v1.34.1+k3s1 true\n'; fi ;; + "get nodes "*) if [ -e "$state/voters" ]; then cat "$state/voters"; else printf 'server1,True,true\n'; fi ;; + "get node "*removed-node-name*) if [ -e "$state/member-removed" ]; then printf 'laptop-0a1b2c3d,'; else printf ',laptop-0a1b2c3d'; fi ;; + "annotate --overwrite node "*"etcd.k3s.cattle.io/remove=true"*) [ -e "$state/member-stays" ] || touch "$state/member-removed" ;; + "get node "*) if [ -e "$state/node-ready" ]; then cat "$state/node-ready"; else printf 'True,True'; fi ;; + "get lease "*) if [ -e "$state/lease" ]; then cat "$state/lease"; else echo 2099-01-01T00:00:00Z; fi ;; + "get pods "*) [ -e "$state/runner-pods" ] && cat "$state/runner-pods"; exit 0 ;; + cordon*|uncordon*|drain*|label*|annotate*) ;; + *) echo "kubectl stand-in: unexpected $*" >&2; exit 1 ;; +esac +""") + + def set_tailscale(self, backend: str | None, domain: str | None) -> None: + path = self.state / "tailscale.json" + if backend is None: + path.unlink(missing_ok=True) + else: + path.write_text(tailscale_status(backend, domain)) + + def set_package(self, version: str | None) -> None: + """Install (or remove) the Tailscale package, whose scripts enable and start tailscaled.""" + path = self.state / "tailscale-version" + for name in ("enabled-tailscaled", "active-tailscaled"): + (self.state / name).unlink(missing_ok=True) + if version is None: + path.unlink(missing_ok=True) + else: + path.write_text(version) + (self.state / "enabled-tailscaled").touch() + (self.state / "active-tailscaled").touch() + + def inventory(self, only_the_node: bool = False, second_server: bool = False, **host_vars: Any) -> Path: + shared = { + "ansible_connection": "local", + "ansible_become": False, + "github_runner_cluster_tailnet_domain": DOMAIN, + "github_runner_cluster_k3s_version": K3S_TAG, + "github_runner_cluster_k3s_token": TOKEN, + "github_runner_cluster_tailscale_join_key": JOIN_KEY, + "github_runner_cluster_native_k3s_download_url": self.url, + "github_runner_cluster_native_tailscale_repository_url": self.url, + "github_runner_cluster_native_tailscale_cli": str(self.stubs / "bin" / "tailscale"), + "github_runner_cluster_native_ready_timeout": 20, + **{name: str(path) for name, path in self.paths.items()}, + } + node = {**shared, "github_runner_cluster_node_mode": "native", **host_vars} + hosts = {"laptop": node} if only_the_node else {"server1": dict(shared), **({"server2": dict(shared)} if second_server else {}), "laptop": node} + inventory = {"all": {"children": {"github_runner_cluster": {"hosts": hosts}}}} + path = self.tmp / "inventory.yml" + path.write_text(yaml.safe_dump(inventory)) + return path + + def run_role(self, *args: str, host_vars: dict[str, Any] | None = None, check: bool = False, only_the_node: bool = False, second_server: bool = False, **extra: Any) -> subprocess.CompletedProcess[str]: + env = { + **os.environ, + "PATH": f"{self.stubs / 'bin'}:{os.environ['PATH']}", + "STUB": str(self.stubs), + "ROLE_FILE": str(self.file("github_runner_cluster_native_data_dir", "fleet-role")), + "ANSIBLE_COLLECTIONS_PATH": str(ROOT / "playbooks" / "collections"), + "ANSIBLE_NOCOLOR": "1", + "ANSIBLE_FORCE_COLOR": "0", + "ANSIBLE_LOCALHOST_WARNING": "false", + "ANSIBLE_INVENTORY_UNPARSED_WARNING": "false", + "ANSIBLE_RETRY_FILES_ENABLED": "false", + } + command = [ANSIBLE_PLAYBOOK, "-i", str(self.inventory(only_the_node, second_server, **(host_vars or {}))), str(RUN), "-e", json.dumps({"host_facts": self.facts, **extra}), *args] + if check: + command.append("--check") + return subprocess.run(command, env=env, capture_output=True, text=True, timeout=300, check=False, cwd=ROOT / "ansible") + + def calls(self) -> list[str]: + """The commands the stand-ins were run with, one per call (a call whose arguments hold a line break logs more lines, which are not calls).""" + log = self.stubs / "calls.log" + return [line for line in log.read_text().splitlines() if re.match(r"^(systemctl|tailscale|apt-get|dpkg-query|docker|kubectl) ", line)] if log.exists() else [] + + def mutations(self, since: int = 0) -> list[str]: + """The calls that change something, from the ``since``-th call on.""" + return [call for call in self.calls()[since:] if call.split()[0] in ("apt-get", "systemctl") and call.split()[1] not in ("is-enabled", "is-active")] + [call for call in self.calls()[since:] if call in ("tailscale logout",) or (call.startswith("kubectl ") and call.split()[1] in ("cordon", "uncordon", "drain", "label", "annotate"))] + + def file(self, key: str, name: str) -> Path: + return self.paths[key] / name + + def recap(self, result: subprocess.CompletedProcess[str]) -> dict[str, int]: + line = next(line for line in result.stdout.splitlines() if re.match(r"^laptop\s+:", line)) + return {name: int(count) for name, count in re.findall(r"(\w+)=(\d+)", line)} + + def server_tree(self) -> Path: + """Lay out a k3s server's data directory as k3s leaves it once its etcd has run: the datastore under server/db (etcd's member, its name and config files and the scheduled snapshots) beside the server's own certificates and token, which stay where they are. Returns the WAL file, whose bytes a test can follow.""" + data = self.paths["github_runner_cluster_native_data_dir"] + files = { + "server/db/etcd/config": "heartbeat-interval: 500\nelection-timeout: 5000\n", + "server/db/etcd/name": "laptop-0a1b2c3d", + "server/db/etcd/member/snap/db": "bolt", + "server/db/etcd/member/wal/0000000000000000-0000000000000000.wal": "wal-bytes", + "server/db/snapshots/etcd-snapshot-laptop-1767225600": "snapshot", + "server/tls/server-ca.crt": "ca", + "server/token": "K10secure::server:token", + "server/cred/admin.kubeconfig": "kubeconfig", + "agent/etc/containerd/config.toml": "containerd", + } + for name, content in files.items(): + path = data / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + (data / "server").chmod(0o700) + return data / "server/db/etcd/member/wal/0000000000000000-0000000000000000.wal" + + def starts(self) -> list[str]: + log = self.stubs / "starts.log" + return log.read_text().splitlines() if log.exists() else [] + + def unit_calls(self, since: int = 0) -> list[str]: + return [call for call in self.calls()[since:] if call.startswith("systemctl") and " is-" not in call and "daemon-reload" not in call] + + def role_file(self) -> str: + return self.file("github_runner_cluster_native_data_dir", "fleet-role").read_text() + + def run_three(self, *args: str, laptop: dict[str, Any] | None = None, check: bool = False, **extra: Any) -> subprocess.CompletedProcess[str]: + """Run the role on laptop in a group of the Docker servers server1 and server2 and laptop, a native server unless ``laptop`` says otherwise.""" + return self.run_role(*args, host_vars={"github_runner_cluster_node_role": "server", **(laptop or {})}, check=check, second_server=True, **extra) + + def demote(self, voters: str = "server1,True,true\nserver2,True,true\nserver3,True,true\nlaptop,True,true\n") -> subprocess.CompletedProcess[str]: + """Run laptop as a server, lay out its datastore as k3s leaves it, then run it as an agent with the cluster's voters read as ``voters``.""" + (self.state / "voters").write_text("server1,True,true\nserver2,True,true\n") + self.assertEqual(self.run_three().returncode, 0) + self.wal = self.server_tree() + older = self.paths["github_runner_cluster_native_data_dir"] / "server" / "db.demoted" / "etcd" / "name" + older.parent.mkdir(parents=True) + older.write_text("laptop-from-an-earlier-demotion") + (self.state / "voters").write_text(voters) + self.demote_before = len(self.calls()) + return self.run_three(laptop={"github_runner_cluster_node_role": "agent"}) + + def assert_demotion_refused(self, voters: str, message: str) -> None: + result = self.demote(voters) + before = self.demote_before + self.assertNotEqual(result.returncode, 0) + self.assertIn(message, re.sub(r"\s+", " ", result.stdout)) + self.assertEqual(self.role_file(), "server") + self.assertTrue((self.state / "active-k3s").exists()) + self.assertTrue(self.wal.exists()) + # The Tailscale package is installed on every run of this test's host, whose dpkg-query stand-in reports none. + self.assertEqual([call for call in self.mutations(before) if not call.startswith(("apt-get", "systemctl enable --now tailscaled"))], []) + + def demote_quickly(self) -> subprocess.CompletedProcess[str]: + (self.state / "voters").write_text("server1,True,true\nserver2,True,true\n") + self.assertEqual(self.run_three().returncode, 0) + self.wal = self.server_tree() + (self.state / "voters").write_text("server1,True,true\nserver2,True,true\nserver3,True,true\nlaptop,True,true\n") + return self.run_three(laptop={"github_runner_cluster_node_role": "agent", "github_runner_cluster_native_ready_timeout": 1}) diff --git a/tests/unit/test_native_node.py b/tests/unit/test_native_node.py index fb4c3ee6..755aa2d3 100644 --- a/tests/unit/test_native_node.py +++ b/tests/unit/test_native_node.py @@ -2,8 +2,6 @@ from __future__ import annotations -import hashlib -import http.server import importlib.util import json import os @@ -12,19 +10,15 @@ import stat import subprocess import tempfile -import threading import unittest -from functools import partial from pathlib import Path from typing import Any import yaml -ROOT = Path(__file__).resolve().parents[2] -ROLE = ROOT / "roles" / "github_runner_cluster" -RUN = ROOT / "tests" / "native" / "run.yml" +from native_stand_ins import DOMAIN, FACTS, JOIN_KEY, ROLE, ROOT, TAILSCALE_VERSION, TOKEN, NativeRoleHarness, stand_in, tailscale_status + PLUGIN = ROOT / "plugins" / "filter" / "cluster_native.py" -ANSIBLE_PLAYBOOK = os.environ.get("ANSIBLE_PLAYBOOK", "ansible-playbook") DEFAULTS: dict[str, Any] = yaml.safe_load((ROLE / "defaults" / "main.yml").read_text()) SPECS: dict[str, Any] = yaml.safe_load((ROLE / "meta" / "argument_specs.yml").read_text())["argument_specs"]["main"]["options"] README = (ROLE / "README.md").read_text() @@ -34,19 +28,6 @@ cluster_native = importlib.util.module_from_spec(_spec) _spec.loader.exec_module(cluster_native) -TOKEN = "k3s-token-example-0123456789" -JOIN_KEY = "tskey-auth-example-0123456789" -DOMAIN = "example.ts.net" -K3S_TAG = "v1.34.1-k3s1" -K3S_RELEASE = "v1.34.1+k3s1" -TAILSCALE_VERSION = "1.102.4" -FACTS = {"system": "Linux", "architecture": "x86_64", "os_family": "Debian", "distribution": "Ubuntu", "distribution_release": "plucky", "service_mgr": "systemd"} - - -def tailscale_status(backend: str, domain: str | None) -> str: - """Return what `tailscale status --json` prints for a client in ``backend`` that is logged in to ``domain`` (none when it is logged out).""" - return json.dumps({"BackendState": backend, "CurrentTailnet": None if domain is None else {"Name": "x", "MagicDNSSuffix": domain, "MagicDNSEnabled": True}}) - class FilterTest(unittest.TestCase): def test_each_supported_architecture_names_its_release_asset_and_sum_file(self) -> None: @@ -161,12 +142,6 @@ def test_the_filters_are_registered_under_the_names_the_tasks_use(self) -> None: self.assertEqual(names, used) -def stand_in(path: Path, body: str) -> None: - """Write an executable shell script.""" - path.write_text("#!/bin/sh\n" + body) - path.chmod(path.stat().st_mode | stat.S_IXUSR) - - class ScriptTest(unittest.TestCase): def setUp(self) -> None: self.tmp = Path(tempfile.mkdtemp(prefix="native-scripts-")) @@ -335,15 +310,6 @@ def test_both_scripts_are_executable_and_shellcheck_clean(self) -> None: self.assertEqual(subprocess.run(["shellcheck", *map(str, scripts)], capture_output=True, text=True, check=False).returncode, 0) -class ReleaseServer(http.server.ThreadingHTTPServer): - """Serves a directory as the k3s release download and Tailscale repository.""" - - -class QuietHandler(http.server.SimpleHTTPRequestHandler): - def log_message(self, format: str, *args: Any) -> None: # noqa: A002 - the signature of the method it replaces - return - - class DefaultsTest(unittest.TestCase): def test_native_mode_is_off_by_default_and_the_service_starts_when_it_is_on(self) -> None: self.assertEqual(DEFAULTS["github_runner_cluster_node_mode"], "compose") @@ -422,208 +388,8 @@ def test_main_hands_native_hosts_to_the_native_tasks_and_ends_the_role_before_an self.assertEqual(tasks[2]["ansible.builtin.include_tasks"], "docker_context.yml") -class NativeRoleTest(unittest.TestCase): - """Runs the role against a native node, stand-ins for the host's commands and a local k3s release.""" - - @classmethod - def setUpClass(cls) -> None: - cls.mirror = Path(tempfile.mkdtemp(prefix="native-mirror-")) - release = cls.mirror / K3S_RELEASE - release.mkdir() - # `k3s kubectl` reaches the test's kubectl stand-in, which answers for the cluster as the stand-in for `docker exec kubectl` does. - cls.binary = f'#!/bin/sh\nif [ "$1" = kubectl ]; then shift; exec "$STUB/bin/kubectl" "$@"; fi\necho "k3s version {K3S_RELEASE} (stand-in)"\n' - (release / "k3s").write_text(cls.binary) - cls.digest = hashlib.sha256(cls.binary.encode()).hexdigest() - (release / "sha256sum-amd64.txt").write_text(f"{cls.digest} k3s\n") - repository = cls.mirror / "ubuntu" - repository.mkdir() - (repository / "plucky.noarmor.gpg").write_text("key\n") - (repository / "plucky.tailscale-keyring.list").write_text("deb [signed-by=/usr/share/keyrings/tailscale-archive-keyring.gpg] https://pkgs.tailscale.com/stable/ubuntu plucky main\n") - cls.server = ReleaseServer(("127.0.0.1", 0), partial(QuietHandler, directory=str(cls.mirror))) - cls.url = f"http://127.0.0.1:{cls.server.server_address[1]}" - threading.Thread(target=cls.server.serve_forever, daemon=True).start() - - @classmethod - def tearDownClass(cls) -> None: - cls.server.shutdown() - cls.server.server_close() - shutil.rmtree(cls.mirror, True) - - def setUp(self) -> None: - self.tmp = Path(tempfile.mkdtemp(prefix="native-role-")) - self.addCleanup(shutil.rmtree, self.tmp, True) - self.stubs = self.tmp / "stubs" - self.state = self.stubs / "state" - self.state.mkdir(parents=True) - self.root = self.tmp / "host" - self.paths = { - "github_runner_cluster_native_bin_dir": self.root / "usr" / "local" / "bin", - "github_runner_cluster_native_lib_dir": self.root / "usr" / "local" / "lib" / "github-runner-k3s", - "github_runner_cluster_native_config_dir": self.root / "etc" / "rancher" / "k3s", - "github_runner_cluster_native_unit_dir": self.root / "etc" / "systemd" / "system", - "github_runner_cluster_native_data_dir": self.root / "var" / "lib" / "rancher" / "k3s", - "github_runner_cluster_native_apt_keyring": self.root / "usr" / "share" / "keyrings" / "tailscale-archive-keyring.gpg", - "github_runner_cluster_native_apt_source": self.root / "etc" / "apt" / "sources.list.d" / "tailscale.list", - } - for path in self.paths.values(): - path.parent.mkdir(parents=True, exist_ok=True) - # Directories every host has already: the role puts files in them and creates no others of the system's. - self.paths["github_runner_cluster_native_bin_dir"].mkdir(exist_ok=True) - self.paths["github_runner_cluster_native_unit_dir"].mkdir(exist_ok=True) - self.make_stand_ins() - self.set_tailscale(None, None) - self.facts = dict(FACTS) - - def make_stand_ins(self) -> None: - bin_dir = self.stubs / "bin" - bin_dir.mkdir() - log = 'echo "$(basename "$0") $*" >> "$STUB/calls.log"\n' - stand_in(bin_dir / "systemctl", log + r"""state="$STUB/state" -command=$1 -shift -now="" -quiet="" -for argument in "$@"; do - case "$argument" in - --quiet) quiet=1 ;; - --now) now=1 ;; - *) unit=$argument ;; - esac -done -join() { - # k3s runs `tailscale up` with the join key as it starts, which logs the host's client in to the cluster's tailnet. - if [ ! -e "$state/no-join" ]; then - printf '%s' '{"BackendState":"Running","CurrentTailnet":{"MagicDNSSuffix":"example.ts.net"}}' > "$state/tailscale.json" - fi -} -case "$command" in - # --quiet prints nothing, as systemctl's does. - is-enabled) if [ -e "$state/enabled-$unit" ]; then [ -n "$quiet" ] || echo enabled; exit 0; fi; [ -n "$quiet" ] || echo disabled; exit 1 ;; - is-active) if [ -e "$state/active-$unit" ]; then [ -n "$quiet" ] || echo active; exit 0; fi; [ -n "$quiet" ] || echo inactive; exit 3 ;; - enable) - touch "$state/enabled-$unit" - if [ -n "$now" ]; then touch "$state/active-$unit"; case "$unit" in k3s|k3s-agent) join ;; esac; fi ;; - disable) rm -f "$state/enabled-$unit"; if [ -n "$now" ]; then rm -f "$state/active-$unit"; fi ;; - start|restart) touch "$state/active-$unit"; case "$unit" in k3s|k3s-agent) join ;; esac ;; -esac -# What the role file named when a k3s unit was started, which is what its ExecCondition would have read. -case "$command:$unit" in - enable:k3s*|start:k3s*|restart:k3s*) if [ "$command" != enable ] || [ -n "$now" ]; then echo "$command $unit $(cat "$ROLE_FILE" 2>/dev/null || echo none)" >> "$STUB/starts.log"; fi ;; -esac -""") - stand_in(bin_dir / "tailscale", log + r"""case "$1" in - status) [ -e "$STUB/state/tailscale.json" ] || exit 1; cat "$STUB/state/tailscale.json" ;; - logout) printf '%s' '{"BackendState":"NeedsLogin","CurrentTailnet":null}' > "$STUB/state/tailscale.json" ;; -esac -""") - stand_in(bin_dir / "dpkg-query", r"""if [ -e "$STUB/state/tailscale-version" ]; then cat "$STUB/state/tailscale-version"; else exit 1; fi -""") - stand_in(bin_dir / "apt-get", log + r"""for argument in "$@"; do - case "$argument" in - tailscale=*) - printf '%s' "${argument#tailscale=}" > "$STUB/state/tailscale-version" - # The package's scripts start tailscaled, which has no login on a host that never had one. - if [ ! -e "$STUB/state/tailscale.json" ]; then printf '%s' '{"BackendState":"NeedsLogin","CurrentTailnet":null}' > "$STUB/state/tailscale.json"; fi ;; - esac -done -""") - # `docker exec kubectl ...` reaches the same kubectl stand-in as `k3s kubectl ...`. - stand_in(bin_dir / "docker", log + r"""case "$*" in - "ps "*) if [ -e "$STUB/state/docker-stopped" ]; then echo exited; else echo running; fi ;; - "exec "*" kubectl "*) shift 3; exec "$STUB/bin/kubectl" "$@" ;; - *) echo "docker stand-in: unexpected $*" >&2; exit 1 ;; -esac -""") - # The cluster as the API reports it, from files in the state directory the tests write: the nodes with their versions and live roles, the voters' readiness, this node's readiness and etcd vote, its member annotations, and the runner pods on it. - stand_in(bin_dir / "kubectl", log + r"""state="$STUB/state" -case "$*" in - "get nodes "*kubeletVersion*) - if [ -e "$state/nodes-unreadable" ]; then echo "The connection to the server was refused" >&2; exit 1; fi - if [ -e "$state/nodes" ]; then cat "$state/nodes"; else printf 'server1 v1.34.1+k3s1 true\n'; fi ;; - "get nodes "*) if [ -e "$state/voters" ]; then cat "$state/voters"; else printf 'server1,True,true\n'; fi ;; - "get node "*removed-node-name*) if [ -e "$state/member-removed" ]; then printf 'laptop-0a1b2c3d,'; else printf ',laptop-0a1b2c3d'; fi ;; - "annotate --overwrite node "*"etcd.k3s.cattle.io/remove=true"*) [ -e "$state/member-stays" ] || touch "$state/member-removed" ;; - "get node "*) if [ -e "$state/node-ready" ]; then cat "$state/node-ready"; else printf 'True,True'; fi ;; - "get lease "*) if [ -e "$state/lease" ]; then cat "$state/lease"; else echo 2099-01-01T00:00:00Z; fi ;; - "get pods "*) [ -e "$state/runner-pods" ] && cat "$state/runner-pods"; exit 0 ;; - cordon*|uncordon*|drain*|label*|annotate*) ;; - *) echo "kubectl stand-in: unexpected $*" >&2; exit 1 ;; -esac -""") - - def set_tailscale(self, backend: str | None, domain: str | None) -> None: - path = self.state / "tailscale.json" - if backend is None: - path.unlink(missing_ok=True) - else: - path.write_text(tailscale_status(backend, domain)) - - def set_package(self, version: str | None) -> None: - """Install (or remove) the Tailscale package, whose scripts enable and start tailscaled.""" - path = self.state / "tailscale-version" - for name in ("enabled-tailscaled", "active-tailscaled"): - (self.state / name).unlink(missing_ok=True) - if version is None: - path.unlink(missing_ok=True) - else: - path.write_text(version) - (self.state / "enabled-tailscaled").touch() - (self.state / "active-tailscaled").touch() - - def inventory(self, only_the_node: bool = False, second_server: bool = False, **host_vars: Any) -> Path: - shared = { - "ansible_connection": "local", - "ansible_become": False, - "github_runner_cluster_tailnet_domain": DOMAIN, - "github_runner_cluster_k3s_version": K3S_TAG, - "github_runner_cluster_k3s_token": TOKEN, - "github_runner_cluster_tailscale_join_key": JOIN_KEY, - "github_runner_cluster_native_k3s_download_url": self.url, - "github_runner_cluster_native_tailscale_repository_url": self.url, - "github_runner_cluster_native_tailscale_cli": str(self.stubs / "bin" / "tailscale"), - "github_runner_cluster_native_ready_timeout": 20, - **{name: str(path) for name, path in self.paths.items()}, - } - node = {**shared, "github_runner_cluster_node_mode": "native", **host_vars} - hosts = {"laptop": node} if only_the_node else {"server1": dict(shared), **({"server2": dict(shared)} if second_server else {}), "laptop": node} - inventory = {"all": {"children": {"github_runner_cluster": {"hosts": hosts}}}} - path = self.tmp / "inventory.yml" - path.write_text(yaml.safe_dump(inventory)) - return path - - def run_role(self, *args: str, host_vars: dict[str, Any] | None = None, check: bool = False, only_the_node: bool = False, second_server: bool = False, **extra: Any) -> subprocess.CompletedProcess[str]: - env = { - **os.environ, - "PATH": f"{self.stubs / 'bin'}:{os.environ['PATH']}", - "STUB": str(self.stubs), - "ROLE_FILE": str(self.file("github_runner_cluster_native_data_dir", "fleet-role")), - "ANSIBLE_COLLECTIONS_PATH": str(ROOT / "playbooks" / "collections"), - "ANSIBLE_NOCOLOR": "1", - "ANSIBLE_FORCE_COLOR": "0", - "ANSIBLE_LOCALHOST_WARNING": "false", - "ANSIBLE_INVENTORY_UNPARSED_WARNING": "false", - "ANSIBLE_RETRY_FILES_ENABLED": "false", - } - command = [ANSIBLE_PLAYBOOK, "-i", str(self.inventory(only_the_node, second_server, **(host_vars or {}))), str(RUN), "-e", json.dumps({"host_facts": self.facts, **extra}), *args] - if check: - command.append("--check") - return subprocess.run(command, env=env, capture_output=True, text=True, timeout=300, check=False, cwd=ROOT / "ansible") - - def calls(self) -> list[str]: - """The commands the stand-ins were run with, one per call (a call whose arguments hold a line break logs more lines, which are not calls).""" - log = self.stubs / "calls.log" - return [line for line in log.read_text().splitlines() if re.match(r"^(systemctl|tailscale|apt-get|dpkg-query|docker|kubectl) ", line)] if log.exists() else [] - - def mutations(self, since: int = 0) -> list[str]: - """The calls that change something, from the ``since``-th call on.""" - return [call for call in self.calls()[since:] if call.split()[0] in ("apt-get", "systemctl") and call.split()[1] not in ("is-enabled", "is-active")] + [call for call in self.calls()[since:] if call in ("tailscale logout",) or (call.startswith("kubectl ") and call.split()[1] in ("cordon", "uncordon", "drain", "label", "annotate"))] - - def file(self, key: str, name: str) -> Path: - return self.paths[key] / name - - def recap(self, result: subprocess.CompletedProcess[str]) -> dict[str, int]: - line = next(line for line in result.stdout.splitlines() if re.match(r"^laptop\s+:", line)) - return {name: int(count) for name, count in re.findall(r"(\w+)=(\d+)", line)} +class NativeRoleTest(NativeRoleHarness): + """Runs the role against a native node as an agent, with stand-ins for the host's commands and a local k3s release.""" def test_a_first_run_installs_configures_starts_and_checks_the_node(self) -> None: result = self.run_role() @@ -917,330 +683,5 @@ def test_a_node_joining_a_cluster_outside_the_inventory_cannot_be_confirmed_by_a self.assertEqual(config["server"], "https://elsewhere.example.ts.net:6443") - # --- A native server --- - - def server_tree(self) -> Path: - """Lay out a k3s server's data directory as k3s leaves it once its etcd has run: the datastore under server/db (etcd's member, its name and config files and the scheduled snapshots) beside the server's own certificates and token, which stay where they are. Returns the WAL file, whose bytes a test can follow.""" - data = self.paths["github_runner_cluster_native_data_dir"] - files = { - "server/db/etcd/config": "heartbeat-interval: 500\nelection-timeout: 5000\n", - "server/db/etcd/name": "laptop-0a1b2c3d", - "server/db/etcd/member/snap/db": "bolt", - "server/db/etcd/member/wal/0000000000000000-0000000000000000.wal": "wal-bytes", - "server/db/snapshots/etcd-snapshot-laptop-1767225600": "snapshot", - "server/tls/server-ca.crt": "ca", - "server/token": "K10secure::server:token", - "server/cred/admin.kubeconfig": "kubeconfig", - "agent/etc/containerd/config.toml": "containerd", - } - for name, content in files.items(): - path = data / name - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(content) - (data / "server").chmod(0o700) - return data / "server/db/etcd/member/wal/0000000000000000-0000000000000000.wal" - - def starts(self) -> list[str]: - log = self.stubs / "starts.log" - return log.read_text().splitlines() if log.exists() else [] - - def unit_calls(self, since: int = 0) -> list[str]: - return [call for call in self.calls()[since:] if call.startswith("systemctl") and " is-" not in call and "daemon-reload" not in call] - - def role_file(self) -> str: - return self.file("github_runner_cluster_native_data_dir", "fleet-role").read_text() - - def test_a_native_bootstrap_server_starts_a_new_cluster_with_the_docker_servers_flags(self) -> None: - result = self.run_role(host_vars={"github_runner_cluster_bootstrap": True, "github_runner_cluster_etcd_heartbeat_interval": 500, "github_runner_cluster_etcd_election_timeout": 5000}, only_the_node=True) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - config_dir = "github_runner_cluster_native_config_dir" - server = yaml.safe_load(self.file(config_dir, "server.yaml").read_text()) - self.assertEqual(server, { - "token-file": str(self.file(config_dir, "token")), - "vpn-auth-file": str(self.file(config_dir, "vpn-auth")), - "node-name": "laptop", - "disable": ["traefik"], - "disable-network-policy": True, - "write-kubeconfig-mode": "0600", - "tls-san": [f"laptop.{DOMAIN}"], - "etcd-arg": ["heartbeat-interval=500", "election-timeout=5000"], - "resolv-conf": "/run/github-runner-k3s/resolv.conf", - }) - self.assertEqual(yaml.safe_load(self.file(config_dir, "server.yaml.d/10-join.yaml").read_text()), {"cluster-init": True}) - for name in ("server.yaml", "server.yaml.d/10-join.yaml"): - self.assertEqual(stat.S_IMODE(self.file(config_dir, name).stat().st_mode), 0o600, name) - self.assertEqual(self.role_file(), "server") - self.assertEqual(self.unit_calls(), ["systemctl enable --now tailscaled", "systemctl enable k3s", "systemctl start k3s"]) - self.assertEqual(self.starts(), ["start k3s server"]) - # The node is waited for through its own server, as root. - self.assertTrue(any(call.startswith("kubectl get node laptop") for call in self.calls())) - self.assertTrue(any(call.startswith("kubectl get lease") for call in self.calls())) - self.assertFalse(any(call.startswith("docker") for call in self.calls())) - - def test_the_server_unit_runs_k3s_server_under_its_role_condition_with_the_mesh_address(self) -> None: - self.assertEqual(self.run_role(host_vars={"github_runner_cluster_bootstrap": True}, only_the_node=True).returncode, 0) - text = self.file("github_runner_cluster_native_unit_dir", "k3s.service").read_text() - service = [line for line in text.split("[Service]")[1].split("[Install]")[0].splitlines() if line and not line.startswith("#")] - lib, config = self.paths["github_runner_cluster_native_lib_dir"], self.paths["github_runner_cluster_native_config_dir"] - self.assertIn(f"ExecCondition={lib}/fleet-role-is server {self.paths['github_runner_cluster_native_data_dir']}/fleet-role", service) - self.assertIn(f"ExecStartPre={lib}/write-node-ip {config}/server.yaml.d/00-node-ip.yaml {config}/vpn-auth", service) - self.assertIn(f"ExecStart={self.paths['github_runner_cluster_native_bin_dir']}/k3s server --config {config}/server.yaml", service) - # Everything else is the agent unit's: one runs at a time, under the same limits. - agent = [line for line in self.file("github_runner_cluster_native_unit_dir", "k3s-agent.service").read_text().split("[Service]")[1].split("[Install]")[0].splitlines() if line and not line.startswith("#")] - self.assertEqual([line for line in service if not line.startswith(("ExecCondition=", "ExecStart"))], [line for line in agent if not line.startswith(("ExecCondition=", "ExecStart"))]) - for helper in ("write-node-ip", "fleet-role-is"): - self.assertEqual(self.file("github_runner_cluster_native_lib_dir", helper).read_text(), (ROLE / "files" / "native" / f"{helper}.sh").read_text()) - - def run_three(self, *args: str, laptop: dict[str, Any] | None = None, check: bool = False, **extra: Any) -> subprocess.CompletedProcess[str]: - """Run the role on laptop in a group of the Docker servers server1 and server2 and laptop, a native server unless ``laptop`` says otherwise.""" - return self.run_role(*args, host_vars={"github_runner_cluster_node_role": "server", **(laptop or {})}, check=check, second_server=True, **extra) - - def test_a_native_server_joins_a_cluster_of_docker_servers_as_they_do(self) -> None: - (self.state / "voters").write_text("server1,True,true\nserver2,True,true\n") - result = self.run_three() - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - config_dir = "github_runner_cluster_native_config_dir" - self.assertEqual(yaml.safe_load(self.file(config_dir, "server.yaml.d/10-join.yaml").read_text()), {"server": f"https://server1.{DOMAIN}:6443"}) - # The test inventory lists laptop first (yaml.safe_dump sorts the hosts), and the SANs are in inventory order. - self.assertEqual(yaml.safe_load(self.file(config_dir, "server.yaml").read_text())["tls-san"], [f"laptop.{DOMAIN}", f"server1.{DOMAIN}", f"server2.{DOMAIN}"]) - unit = self.file("github_runner_cluster_native_unit_dir", "k3s.service").read_text() - self.assertIn(f'Environment="GITHUB_RUNNER_JOIN_HOST=server1.{DOMAIN}"', unit) - # The voters were read through the bootstrap server's container before the server started. - calls = self.calls() - read = next(index for index, call in enumerate(calls) if call.startswith("docker exec github-runner-k3s kubectl get nodes") and "etcd}" in call) - self.assertLess(read, calls.index("systemctl start k3s")) - self.assertEqual(self.starts(), ["start k3s server"]) - - def test_a_native_server_does_not_join_while_another_voter_is_not_ready(self) -> None: - (self.state / "voters").write_text("server1,True,true\nserver2,False,true\n") - result = self.run_three() - self.assertNotEqual(result.returncode, 0) - self.assertIn("voter server2 is not Ready (its Ready condition is False)", re.sub(r"\s+", " ", result.stdout)) - self.assertEqual(self.starts(), []) - self.assertNotIn("systemctl start k3s", self.calls()) - - def test_an_agent_is_promoted_in_place_once_its_node_is_empty(self) -> None: - self.assertEqual(self.run_three(laptop={"github_runner_cluster_node_role": "agent", "github_runner_cluster_voter_eligible": False}).returncode, 0) - # A three-host group with laptop as an agent has one Docker server: the first run is the laptop as an agent of server1, the second makes it the third server. - self.assertEqual(self.role_file(), "agent") - password = self.root / "etc" / "rancher" / "node" / "password" - password.parent.mkdir(parents=True) - password.write_text("node-password") - before = len(self.calls()) - result = self.run_three() - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - calls = self.calls()[before:] - order = [ - "kubectl cordon laptop", - next(call for call in calls if call.startswith("kubectl get pods --all-namespaces --selector actions-ephemeral-runner=True --field-selector spec.nodeName=laptop,status.phase!=Succeeded,status.phase!=Failed")), - next(call for call in calls if call.startswith("kubectl drain laptop --ignore-daemonsets --delete-emptydir-data")), - "systemctl disable --now k3s-agent", - "systemctl start k3s", - "kubectl uncordon laptop", - ] - self.assertEqual([call for call in calls if call in order], order) - self.assertEqual(self.role_file(), "server") - self.assertEqual(self.starts()[-1], "start k3s server") - self.assertFalse((self.state / "active-k3s-agent").exists()) - self.assertFalse((self.state / "enabled-k3s-agent").exists()) - self.assertEqual(password.read_text(), "node-password") - # The cordon and drain went through another server, and the uncordon through this one, which now serves the API. - self.assertIn("docker exec github-runner-k3s kubectl cordon laptop", calls) - self.assertNotIn("docker exec github-runner-k3s kubectl uncordon laptop", calls) - - def test_a_promotion_whose_runner_pods_do_not_finish_is_abandoned_with_the_node_uncordoned(self) -> None: - self.assertEqual(self.run_three(laptop={"github_runner_cluster_node_role": "agent"}).returncode, 0) - (self.state / "runner-pods").write_text("pod/runner-abc12\n") - before = len(self.calls()) - result = self.run_three(laptop={"github_runner_cluster_native_runner_wait_timeout": 1}) - self.assertNotEqual(result.returncode, 0) - self.assertIn("its node could not be emptied", re.sub(r"\s+", " ", result.stdout)) - calls = self.calls()[before:] - self.assertIn("kubectl uncordon laptop", calls) - self.assertNotIn("systemctl disable --now k3s-agent", calls) - self.assertEqual(self.role_file(), "agent") - self.assertTrue((self.state / "active-k3s-agent").exists()) - - def test_a_running_server_restarts_only_while_every_other_voter_is_ready_and_a_refused_restart_is_made_later(self) -> None: - self.assertEqual(self.run_three().returncode, 0) - (self.state / "voters").write_text("server1,True,true\nserver2,Unknown,true\nlaptop,True,true\n") - taints = {"github_runner_cluster_node_taints": ["dedicated=ci:NoSchedule"]} - before = len(self.calls()) - refused = self.run_three(laptop=taints) - self.assertNotEqual(refused.returncode, 0) - self.assertIn("Not going ahead with the restart of k3s on laptop: voter server2 is not Ready", re.sub(r"\s+", " ", refused.stdout)) - self.assertEqual(self.unit_calls(before), []) - # The voters were read through this server's own k3s, which is running. - self.assertTrue(any(call.startswith("kubectl get nodes") and "etcd}" in call for call in self.calls()[before:])) - self.assertFalse(any(call.startswith("docker exec") and "etcd}" in call for call in self.calls()[before:])) - (self.state / "voters").write_text("server1,True,true\nserver2,True,true\nlaptop,True,true\n") - before = len(self.calls()) - result = self.run_three(laptop=taints) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertEqual(self.unit_calls(before), ["systemctl restart k3s"]) - before = len(self.calls()) - self.assertEqual(self.run_three(laptop=taints).returncode, 0) - self.assertEqual(self.unit_calls(before), []) - - def test_a_check_run_reports_a_promotion_and_makes_none(self) -> None: - self.assertEqual(self.run_three(laptop={"github_runner_cluster_node_role": "agent"}).returncode, 0) - before = len(self.calls()) - result = self.run_three(check=True) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertIn("promote k3s on laptop", result.stdout) - self.assertEqual([call for call in self.mutations(before) if not call.startswith(("apt-get", "systemctl enable --now tailscaled"))], []) - self.assertEqual(self.role_file(), "agent") - - def demote(self, voters: str = "server1,True,true\nserver2,True,true\nserver3,True,true\nlaptop,True,true\n") -> subprocess.CompletedProcess[str]: - """Run laptop as a server, lay out its datastore as k3s leaves it, then run it as an agent with the cluster's voters read as ``voters``.""" - (self.state / "voters").write_text("server1,True,true\nserver2,True,true\n") - self.assertEqual(self.run_three().returncode, 0) - self.wal = self.server_tree() - older = self.paths["github_runner_cluster_native_data_dir"] / "server" / "db.demoted" / "etcd" / "name" - older.parent.mkdir(parents=True) - older.write_text("laptop-from-an-earlier-demotion") - (self.state / "voters").write_text(voters) - self.demote_before = len(self.calls()) - return self.run_three(laptop={"github_runner_cluster_node_role": "agent"}) - - def assert_demotion_refused(self, voters: str, message: str) -> None: - result = self.demote(voters) - before = self.demote_before - self.assertNotEqual(result.returncode, 0) - self.assertIn(message, re.sub(r"\s+", " ", result.stdout)) - self.assertEqual(self.role_file(), "server") - self.assertTrue((self.state / "active-k3s").exists()) - self.assertTrue(self.wal.exists()) - # The Tailscale package is installed on every run of this test's host, whose dpkg-query stand-in reports none. - self.assertEqual([call for call in self.mutations(before) if not call.startswith(("apt-get", "systemctl enable --now tailscaled"))], []) - - def test_a_demotion_that_would_leave_two_voters_is_refused(self) -> None: - self.assert_demotion_refused("server1,True,true\nserver2,True,true\nlaptop,True,true\n", "only 2 other voters (server1, server2) would remain, fewer than the 3") - - def test_a_demotion_while_another_voter_is_not_ready_is_refused(self) -> None: - self.assert_demotion_refused("server1,True,true\nserver2,True,true\nserver3,False,true\nlaptop,True,true\n", "voter server3 is not Ready") - - def test_a_server_is_demoted_once_its_member_is_removed_with_its_datastore_moved_aside_and_its_identity_kept(self) -> None: - before = len(self.calls()) - result = self.demote() - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - data = self.paths["github_runner_cluster_native_data_dir"] - self.assertEqual(self.role_file(), "agent") - self.assertFalse((data / "server" / "db").exists()) - moved = data / "server" / "db.demoted" - self.assertEqual((moved / "etcd/member/wal/0000000000000000-0000000000000000.wal").read_text(), "wal-bytes") - self.assertEqual((moved / "snapshots/etcd-snapshot-laptop-1767225600").read_text(), "snapshot") - # The newest copy replaces the one an earlier demotion left. - self.assertEqual((moved / "etcd" / "name").read_text(), "laptop-0a1b2c3d") - for kept in ("server/tls/server-ca.crt", "server/token", "agent/etc/containerd/config.toml"): - self.assertTrue((data / kept).is_file(), kept) - calls = self.calls()[before:] - order = [ - "kubectl cordon laptop", - "kubectl annotate --overwrite node laptop etcd.k3s.cattle.io/remove=true", - "systemctl disable --now k3s", - "kubectl label node laptop node-role.kubernetes.io/control-plane- node-role.kubernetes.io/etcd- node-role.kubernetes.io/master-", - "kubectl annotate node laptop etcd.k3s.cattle.io/remove- etcd.k3s.cattle.io/removed-node-name- etcd.k3s.cattle.io/node-name- etcd.k3s.cattle.io/node-address-", - "systemctl enable --now k3s-agent", - "kubectl uncordon laptop", - ] - self.assertEqual([call for call in calls if call in order], order) - # Everything went through another server's API, since this one stops. - for call in order: - if call.startswith("kubectl"): - self.assertIn("docker exec github-runner-k3s " + call, calls) - self.assertEqual(self.starts()[-1], "enable k3s-agent agent") - self.assertFalse((self.state / "active-k3s").exists()) - - def test_a_server_whose_member_is_not_removed_is_not_stopped(self) -> None: - (self.state / "member-stays").touch() - result = self.demote_quickly() - self.assertNotEqual(result.returncode, 0) - self.assertTrue((self.state / "active-k3s").exists()) - self.assertTrue(self.wal.exists()) - # The role file already names the agent, so the server is not started again by Restart=always or a reboot; a rerun finishes the demotion. - self.assertEqual(self.role_file(), "agent") - (self.state / "member-stays").unlink() - result = self.run_three(laptop={"github_runner_cluster_node_role": "agent"}) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertFalse((self.state / "active-k3s").exists()) - self.assertFalse(self.wal.exists()) - - def demote_quickly(self) -> subprocess.CompletedProcess[str]: - (self.state / "voters").write_text("server1,True,true\nserver2,True,true\n") - self.assertEqual(self.run_three().returncode, 0) - self.wal = self.server_tree() - (self.state / "voters").write_text("server1,True,true\nserver2,True,true\nserver3,True,true\nlaptop,True,true\n") - return self.run_three(laptop={"github_runner_cluster_node_role": "agent", "github_runner_cluster_native_ready_timeout": 1}) - - def test_a_former_member_never_starts_a_second_cluster(self) -> None: - moved = self.paths["github_runner_cluster_native_data_dir"] / "server" / "db.demoted" / "etcd" - moved.mkdir(parents=True) - (moved / "name").write_text("laptop-0a1b2c3d") - alone = self.run_role(host_vars={"github_runner_cluster_bootstrap": True}, only_the_node=True) - self.assertNotEqual(alone.returncode, 0) - self.assertIn("with no server to join", re.sub(r"\s+", " ", alone.stdout)) - self.assertEqual(self.starts(), []) - (self.state / "voters").write_text("server1,True,true\nserver2,True,true\n") - result = self.run_three(laptop={"github_runner_cluster_bootstrap": True}) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertEqual(yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "server.yaml.d/10-join.yaml").read_text()), {"server": f"https://server1.{DOMAIN}:6443"}) - - def test_a_bootstrap_host_with_no_datastore_joins_a_cluster_another_server_runs(self) -> None: - (self.state / "voters").write_text("server1,True,true\nserver2,True,true\n") - result = self.run_three(laptop={"github_runner_cluster_bootstrap": True}) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertEqual(yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "server.yaml.d/10-join.yaml").read_text()), {"server": f"https://server1.{DOMAIN}:6443"}) - - def test_a_bootstrap_host_with_no_datastore_starts_a_cluster_only_when_every_other_server_is_stopped(self) -> None: - (self.state / "docker-stopped").touch() - result = self.run_three(laptop={"github_runner_cluster_bootstrap": True}) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertEqual(yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "server.yaml.d/10-join.yaml").read_text()), {"cluster-init": True}) - - def test_a_bootstrap_host_that_cannot_tell_whether_a_cluster_exists_is_refused(self) -> None: - (self.state / "nodes-unreadable").touch() - result = self.run_three(laptop={"github_runner_cluster_bootstrap": True}, github_runner_cluster_allow_k3s_version_jump=True) - self.assertNotEqual(result.returncode, 0) - self.assertIn("would start a new cluster (cluster-init) unless another server already runs one, and that cannot be told", re.sub(r"\s+", " ", result.stdout)) - self.assertEqual(self.mutations(), []) - - def test_an_agent_from_before_the_role_file_keeps_running_as_an_agent(self) -> None: - unit = self.file("github_runner_cluster_native_unit_dir", "k3s-agent.service") - unit.write_text("[Service]\nExecStart=/usr/local/bin/k3s agent\n") - old = self.file("github_runner_cluster_native_config_dir", "config.yaml") - old.parent.mkdir(parents=True) - old.write_text("server: https://server1.example.ts.net:6443\n") - for name in ("enabled-k3s-agent", "active-k3s-agent"): - (self.state / name).touch() - self.set_package(TAILSCALE_VERSION) - self.set_tailscale("Running", DOMAIN) - result = self.run_role() - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertEqual(self.role_file(), "agent") - self.assertFalse(old.exists()) - self.assertEqual(self.starts(), ["restart k3s-agent agent"]) - - def test_an_eligible_agent_is_staged_as_a_standby_server_and_keeps_its_role(self) -> None: - result = self.run_role(host_vars={"github_runner_cluster_voter_eligible": True}) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertEqual(self.role_file(), "agent") - server = yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "server.yaml").read_text()) - self.assertEqual(server["tls-san"], [f"laptop.{DOMAIN}", f"server1.{DOMAIN}"]) - self.assertNotIn("systemctl start k3s", self.calls()) - # The inventory gives an eligible host only the role it starts with. - before = len(self.calls()) - result = self.run_three(laptop={"github_runner_cluster_voter_eligible": True}) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertEqual(self.role_file(), "agent") - self.assertEqual(self.unit_calls(before), []) - - def test_a_native_bootstrap_host_is_refused_under_headscale_in_cluster(self) -> None: - result = self.run_role(host_vars={"github_runner_cluster_bootstrap": True, "github_runner_cluster_mesh": "headscale_in_cluster"}) - self.assertNotEqual(result.returncode, 0) - self.assertIn("whose bootstrap server runs the Headscale server as a static pod", re.sub(r"\s+", " ", result.stdout)) - self.assertEqual(self.calls(), []) - - if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_native_server.py b/tests/unit/test_native_server.py new file mode 100644 index 00000000..1daa8cc3 --- /dev/null +++ b/tests/unit/test_native_server.py @@ -0,0 +1,275 @@ +"""Tests the github_runner_cluster role's native node as a k3s server (tests/native/run.yml) against the stand-ins in native_stand_ins.py: bootstrapping and joining with the Docker servers' flags, the restart guard on the other voters, a switch between the server and agent units in both directions, and a former member never starting a second cluster.""" + +from __future__ import annotations + +import re +import stat +import unittest + +import yaml +from native_stand_ins import DOMAIN, ROLE, TAILSCALE_VERSION, NativeRoleHarness + + +class NativeServerTest(NativeRoleHarness): + """A native host run as a k3s server, and switched between its server and agent units.""" + + def test_a_native_bootstrap_server_starts_a_new_cluster_with_the_docker_servers_flags(self) -> None: + result = self.run_role(host_vars={"github_runner_cluster_bootstrap": True, "github_runner_cluster_etcd_heartbeat_interval": 500, "github_runner_cluster_etcd_election_timeout": 5000}, only_the_node=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + config_dir = "github_runner_cluster_native_config_dir" + server = yaml.safe_load(self.file(config_dir, "server.yaml").read_text()) + self.assertEqual(server, { + "token-file": str(self.file(config_dir, "token")), + "vpn-auth-file": str(self.file(config_dir, "vpn-auth")), + "node-name": "laptop", + "disable": ["traefik"], + "disable-network-policy": True, + "write-kubeconfig-mode": "0600", + "tls-san": [f"laptop.{DOMAIN}"], + "etcd-arg": ["heartbeat-interval=500", "election-timeout=5000"], + "resolv-conf": "/run/github-runner-k3s/resolv.conf", + }) + self.assertEqual(yaml.safe_load(self.file(config_dir, "server.yaml.d/10-join.yaml").read_text()), {"cluster-init": True}) + for name in ("server.yaml", "server.yaml.d/10-join.yaml"): + self.assertEqual(stat.S_IMODE(self.file(config_dir, name).stat().st_mode), 0o600, name) + self.assertEqual(self.role_file(), "server") + self.assertEqual(self.unit_calls(), ["systemctl enable --now tailscaled", "systemctl enable k3s", "systemctl start k3s"]) + self.assertEqual(self.starts(), ["start k3s server"]) + # The node is waited for through its own server, as root. + self.assertTrue(any(call.startswith("kubectl get node laptop") for call in self.calls())) + self.assertTrue(any(call.startswith("kubectl get lease") for call in self.calls())) + self.assertFalse(any(call.startswith("docker") for call in self.calls())) + + def test_the_server_unit_runs_k3s_server_under_its_role_condition_with_the_mesh_address(self) -> None: + self.assertEqual(self.run_role(host_vars={"github_runner_cluster_bootstrap": True}, only_the_node=True).returncode, 0) + text = self.file("github_runner_cluster_native_unit_dir", "k3s.service").read_text() + service = [line for line in text.split("[Service]")[1].split("[Install]")[0].splitlines() if line and not line.startswith("#")] + lib, config = self.paths["github_runner_cluster_native_lib_dir"], self.paths["github_runner_cluster_native_config_dir"] + self.assertIn(f"ExecCondition={lib}/fleet-role-is server {self.paths['github_runner_cluster_native_data_dir']}/fleet-role", service) + self.assertIn(f"ExecStartPre={lib}/write-node-ip {config}/server.yaml.d/00-node-ip.yaml {config}/vpn-auth", service) + self.assertIn(f"ExecStart={self.paths['github_runner_cluster_native_bin_dir']}/k3s server --config {config}/server.yaml", service) + # Everything else is the agent unit's: one runs at a time, under the same limits. + agent = [line for line in self.file("github_runner_cluster_native_unit_dir", "k3s-agent.service").read_text().split("[Service]")[1].split("[Install]")[0].splitlines() if line and not line.startswith("#")] + self.assertEqual([line for line in service if not line.startswith(("ExecCondition=", "ExecStart"))], [line for line in agent if not line.startswith(("ExecCondition=", "ExecStart"))]) + for helper in ("write-node-ip", "fleet-role-is"): + self.assertEqual(self.file("github_runner_cluster_native_lib_dir", helper).read_text(), (ROLE / "files" / "native" / f"{helper}.sh").read_text()) + + def test_a_native_server_joins_a_cluster_of_docker_servers_as_they_do(self) -> None: + (self.state / "voters").write_text("server1,True,true\nserver2,True,true\n") + result = self.run_three() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + config_dir = "github_runner_cluster_native_config_dir" + self.assertEqual(yaml.safe_load(self.file(config_dir, "server.yaml.d/10-join.yaml").read_text()), {"server": f"https://server1.{DOMAIN}:6443"}) + # The test inventory lists laptop first (yaml.safe_dump sorts the hosts), and the SANs are in inventory order. + self.assertEqual(yaml.safe_load(self.file(config_dir, "server.yaml").read_text())["tls-san"], [f"laptop.{DOMAIN}", f"server1.{DOMAIN}", f"server2.{DOMAIN}"]) + unit = self.file("github_runner_cluster_native_unit_dir", "k3s.service").read_text() + self.assertIn(f'Environment="GITHUB_RUNNER_JOIN_HOST=server1.{DOMAIN}"', unit) + # The voters were read through the bootstrap server's container before the server started. + calls = self.calls() + read = next(index for index, call in enumerate(calls) if call.startswith("docker exec github-runner-k3s kubectl get nodes") and "etcd}" in call) + self.assertLess(read, calls.index("systemctl start k3s")) + self.assertEqual(self.starts(), ["start k3s server"]) + + def test_a_native_server_does_not_join_while_another_voter_is_not_ready(self) -> None: + (self.state / "voters").write_text("server1,True,true\nserver2,False,true\n") + result = self.run_three() + self.assertNotEqual(result.returncode, 0) + self.assertIn("voter server2 is not Ready (its Ready condition is False)", re.sub(r"\s+", " ", result.stdout)) + self.assertEqual(self.starts(), []) + self.assertNotIn("systemctl start k3s", self.calls()) + + def test_an_agent_is_promoted_in_place_once_its_node_is_empty(self) -> None: + self.assertEqual(self.run_three(laptop={"github_runner_cluster_node_role": "agent", "github_runner_cluster_voter_eligible": False}).returncode, 0) + # A three-host group with laptop as an agent has one Docker server: the first run is the laptop as an agent of server1, the second makes it the third server. + self.assertEqual(self.role_file(), "agent") + password = self.root / "etc" / "rancher" / "node" / "password" + password.parent.mkdir(parents=True) + password.write_text("node-password") + before = len(self.calls()) + result = self.run_three() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + calls = self.calls()[before:] + order = [ + "kubectl cordon laptop", + next(call for call in calls if call.startswith("kubectl get pods --all-namespaces --selector actions-ephemeral-runner=True --field-selector spec.nodeName=laptop,status.phase!=Succeeded,status.phase!=Failed")), + next(call for call in calls if call.startswith("kubectl drain laptop --ignore-daemonsets --delete-emptydir-data")), + "systemctl disable --now k3s-agent", + "systemctl start k3s", + "kubectl uncordon laptop", + ] + self.assertEqual([call for call in calls if call in order], order) + self.assertEqual(self.role_file(), "server") + self.assertEqual(self.starts()[-1], "start k3s server") + self.assertFalse((self.state / "active-k3s-agent").exists()) + self.assertFalse((self.state / "enabled-k3s-agent").exists()) + self.assertEqual(password.read_text(), "node-password") + # The cordon and drain went through another server, and the uncordon through this one, which now serves the API. + self.assertIn("docker exec github-runner-k3s kubectl cordon laptop", calls) + self.assertNotIn("docker exec github-runner-k3s kubectl uncordon laptop", calls) + + def test_a_promotion_whose_runner_pods_do_not_finish_is_abandoned_with_the_node_uncordoned(self) -> None: + self.assertEqual(self.run_three(laptop={"github_runner_cluster_node_role": "agent"}).returncode, 0) + (self.state / "runner-pods").write_text("pod/runner-abc12\n") + before = len(self.calls()) + result = self.run_three(laptop={"github_runner_cluster_native_runner_wait_timeout": 1}) + self.assertNotEqual(result.returncode, 0) + self.assertIn("its node could not be emptied", re.sub(r"\s+", " ", result.stdout)) + calls = self.calls()[before:] + self.assertIn("kubectl uncordon laptop", calls) + self.assertNotIn("systemctl disable --now k3s-agent", calls) + self.assertEqual(self.role_file(), "agent") + self.assertTrue((self.state / "active-k3s-agent").exists()) + + def test_a_running_server_restarts_only_while_every_other_voter_is_ready_and_a_refused_restart_is_made_later(self) -> None: + self.assertEqual(self.run_three().returncode, 0) + (self.state / "voters").write_text("server1,True,true\nserver2,Unknown,true\nlaptop,True,true\n") + taints = {"github_runner_cluster_node_taints": ["dedicated=ci:NoSchedule"]} + before = len(self.calls()) + refused = self.run_three(laptop=taints) + self.assertNotEqual(refused.returncode, 0) + self.assertIn("Not going ahead with the restart of k3s on laptop: voter server2 is not Ready", re.sub(r"\s+", " ", refused.stdout)) + self.assertEqual(self.unit_calls(before), []) + # The voters were read through this server's own k3s, which is running. + self.assertTrue(any(call.startswith("kubectl get nodes") and "etcd}" in call for call in self.calls()[before:])) + self.assertFalse(any(call.startswith("docker exec") and "etcd}" in call for call in self.calls()[before:])) + (self.state / "voters").write_text("server1,True,true\nserver2,True,true\nlaptop,True,true\n") + before = len(self.calls()) + result = self.run_three(laptop=taints) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.unit_calls(before), ["systemctl restart k3s"]) + before = len(self.calls()) + self.assertEqual(self.run_three(laptop=taints).returncode, 0) + self.assertEqual(self.unit_calls(before), []) + + def test_a_check_run_reports_a_promotion_and_makes_none(self) -> None: + self.assertEqual(self.run_three(laptop={"github_runner_cluster_node_role": "agent"}).returncode, 0) + before = len(self.calls()) + result = self.run_three(check=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("promote k3s on laptop", result.stdout) + self.assertEqual([call for call in self.mutations(before) if not call.startswith(("apt-get", "systemctl enable --now tailscaled"))], []) + self.assertEqual(self.role_file(), "agent") + + def test_a_demotion_that_would_leave_two_voters_is_refused(self) -> None: + self.assert_demotion_refused("server1,True,true\nserver2,True,true\nlaptop,True,true\n", "only 2 other voters (server1, server2) would remain, fewer than the 3") + + def test_a_demotion_while_another_voter_is_not_ready_is_refused(self) -> None: + self.assert_demotion_refused("server1,True,true\nserver2,True,true\nserver3,False,true\nlaptop,True,true\n", "voter server3 is not Ready") + + def test_a_server_is_demoted_once_its_member_is_removed_with_its_datastore_moved_aside_and_its_identity_kept(self) -> None: + before = len(self.calls()) + result = self.demote() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + data = self.paths["github_runner_cluster_native_data_dir"] + self.assertEqual(self.role_file(), "agent") + self.assertFalse((data / "server" / "db").exists()) + moved = data / "server" / "db.demoted" + self.assertEqual((moved / "etcd/member/wal/0000000000000000-0000000000000000.wal").read_text(), "wal-bytes") + self.assertEqual((moved / "snapshots/etcd-snapshot-laptop-1767225600").read_text(), "snapshot") + # The newest copy replaces the one an earlier demotion left. + self.assertEqual((moved / "etcd" / "name").read_text(), "laptop-0a1b2c3d") + for kept in ("server/tls/server-ca.crt", "server/token", "agent/etc/containerd/config.toml"): + self.assertTrue((data / kept).is_file(), kept) + calls = self.calls()[before:] + order = [ + "kubectl cordon laptop", + "kubectl annotate --overwrite node laptop etcd.k3s.cattle.io/remove=true", + "systemctl disable --now k3s", + "kubectl label node laptop node-role.kubernetes.io/control-plane- node-role.kubernetes.io/etcd- node-role.kubernetes.io/master-", + "kubectl annotate node laptop etcd.k3s.cattle.io/remove- etcd.k3s.cattle.io/removed-node-name- etcd.k3s.cattle.io/node-name- etcd.k3s.cattle.io/node-address-", + "systemctl enable --now k3s-agent", + "kubectl uncordon laptop", + ] + self.assertEqual([call for call in calls if call in order], order) + # Everything went through another server's API, since this one stops. + for call in order: + if call.startswith("kubectl"): + self.assertIn("docker exec github-runner-k3s " + call, calls) + self.assertEqual(self.starts()[-1], "enable k3s-agent agent") + self.assertFalse((self.state / "active-k3s").exists()) + + def test_a_server_whose_member_is_not_removed_is_not_stopped(self) -> None: + (self.state / "member-stays").touch() + result = self.demote_quickly() + self.assertNotEqual(result.returncode, 0) + self.assertTrue((self.state / "active-k3s").exists()) + self.assertTrue(self.wal.exists()) + # The role file already names the agent, so the server is not started again by Restart=always or a reboot; a rerun finishes the demotion. + self.assertEqual(self.role_file(), "agent") + (self.state / "member-stays").unlink() + result = self.run_three(laptop={"github_runner_cluster_node_role": "agent"}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertFalse((self.state / "active-k3s").exists()) + self.assertFalse(self.wal.exists()) + + def test_a_former_member_never_starts_a_second_cluster(self) -> None: + moved = self.paths["github_runner_cluster_native_data_dir"] / "server" / "db.demoted" / "etcd" + moved.mkdir(parents=True) + (moved / "name").write_text("laptop-0a1b2c3d") + alone = self.run_role(host_vars={"github_runner_cluster_bootstrap": True}, only_the_node=True) + self.assertNotEqual(alone.returncode, 0) + self.assertIn("with no server to join", re.sub(r"\s+", " ", alone.stdout)) + self.assertEqual(self.starts(), []) + (self.state / "voters").write_text("server1,True,true\nserver2,True,true\n") + result = self.run_three(laptop={"github_runner_cluster_bootstrap": True}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "server.yaml.d/10-join.yaml").read_text()), {"server": f"https://server1.{DOMAIN}:6443"}) + + def test_a_bootstrap_host_with_no_datastore_joins_a_cluster_another_server_runs(self) -> None: + (self.state / "voters").write_text("server1,True,true\nserver2,True,true\n") + result = self.run_three(laptop={"github_runner_cluster_bootstrap": True}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "server.yaml.d/10-join.yaml").read_text()), {"server": f"https://server1.{DOMAIN}:6443"}) + + def test_a_bootstrap_host_with_no_datastore_starts_a_cluster_only_when_every_other_server_is_stopped(self) -> None: + (self.state / "docker-stopped").touch() + result = self.run_three(laptop={"github_runner_cluster_bootstrap": True}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "server.yaml.d/10-join.yaml").read_text()), {"cluster-init": True}) + + def test_a_bootstrap_host_that_cannot_tell_whether_a_cluster_exists_is_refused(self) -> None: + (self.state / "nodes-unreadable").touch() + result = self.run_three(laptop={"github_runner_cluster_bootstrap": True}, github_runner_cluster_allow_k3s_version_jump=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("would start a new cluster (cluster-init) unless another server already runs one, and that cannot be told", re.sub(r"\s+", " ", result.stdout)) + self.assertEqual(self.mutations(), []) + + def test_an_agent_from_before_the_role_file_keeps_running_as_an_agent(self) -> None: + unit = self.file("github_runner_cluster_native_unit_dir", "k3s-agent.service") + unit.write_text("[Service]\nExecStart=/usr/local/bin/k3s agent\n") + old = self.file("github_runner_cluster_native_config_dir", "config.yaml") + old.parent.mkdir(parents=True) + old.write_text("server: https://server1.example.ts.net:6443\n") + for name in ("enabled-k3s-agent", "active-k3s-agent"): + (self.state / name).touch() + self.set_package(TAILSCALE_VERSION) + self.set_tailscale("Running", DOMAIN) + result = self.run_role() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.role_file(), "agent") + self.assertFalse(old.exists()) + self.assertEqual(self.starts(), ["restart k3s-agent agent"]) + + def test_an_eligible_agent_is_staged_as_a_standby_server_and_keeps_its_role(self) -> None: + result = self.run_role(host_vars={"github_runner_cluster_voter_eligible": True}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.role_file(), "agent") + server = yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "server.yaml").read_text()) + self.assertEqual(server["tls-san"], [f"laptop.{DOMAIN}", f"server1.{DOMAIN}"]) + self.assertNotIn("systemctl start k3s", self.calls()) + # The inventory gives an eligible host only the role it starts with. + before = len(self.calls()) + result = self.run_three(laptop={"github_runner_cluster_voter_eligible": True}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.role_file(), "agent") + self.assertEqual(self.unit_calls(before), []) + + def test_a_native_bootstrap_host_is_refused_under_headscale_in_cluster(self) -> None: + result = self.run_role(host_vars={"github_runner_cluster_bootstrap": True, "github_runner_cluster_mesh": "headscale_in_cluster"}) + self.assertNotEqual(result.returncode, 0) + self.assertIn("whose bootstrap server runs the Headscale server as a static pod", re.sub(r"\s+", " ", result.stdout)) + self.assertEqual(self.calls(), []) + + +if __name__ == "__main__": + unittest.main() From ce8e75eecc7a8459019935c9bf718a4f6fe2beeb Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 3 Nov 2026 10:57:49 +0000 Subject: [PATCH 08/10] test(native): seed a running role instead of spending a run to reach it The promotion and demotion tests ran the role once only to leave the host running the role they then switch from. They now write the role file and the unit's state directly, as a run leaves them, since the role writes every file it reads afresh, so each spends one run of the role on its subject. --- tests/unit/native_stand_ins.py | 17 ++++++++++++----- tests/unit/test_native_server.py | 6 ++---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/unit/native_stand_ins.py b/tests/unit/native_stand_ins.py index a7ab2bb1..35c6669e 100644 --- a/tests/unit/native_stand_ins.py +++ b/tests/unit/native_stand_ins.py @@ -284,6 +284,14 @@ def starts(self) -> list[str]: def unit_calls(self, since: int = 0) -> list[str]: return [call for call in self.calls()[since:] if call.startswith("systemctl") and " is-" not in call and "daemon-reload" not in call] + def running_as(self, role: str) -> None: + """Leave the host as a run of the role leaves one that runs ``role``: the role file names it, and its unit is enabled and active. For a test whose subject is the next run, so that it need not spend a run of the role getting there; the role writes every file it reads afresh.""" + self.file("github_runner_cluster_native_data_dir", "fleet-role").parent.mkdir(parents=True, exist_ok=True) + self.file("github_runner_cluster_native_data_dir", "fleet-role").write_text(role) + unit = "k3s" if role == "server" else "k3s-agent" + for state in ("enabled", "active"): + (self.state / f"{state}-{unit}").touch() + def role_file(self) -> str: return self.file("github_runner_cluster_native_data_dir", "fleet-role").read_text() @@ -293,8 +301,7 @@ def run_three(self, *args: str, laptop: dict[str, Any] | None = None, check: boo def demote(self, voters: str = "server1,True,true\nserver2,True,true\nserver3,True,true\nlaptop,True,true\n") -> subprocess.CompletedProcess[str]: """Run laptop as a server, lay out its datastore as k3s leaves it, then run it as an agent with the cluster's voters read as ``voters``.""" - (self.state / "voters").write_text("server1,True,true\nserver2,True,true\n") - self.assertEqual(self.run_three().returncode, 0) + self.running_as("server") self.wal = self.server_tree() older = self.paths["github_runner_cluster_native_data_dir"] / "server" / "db.demoted" / "etcd" / "name" older.parent.mkdir(parents=True) @@ -312,11 +319,11 @@ def assert_demotion_refused(self, voters: str, message: str) -> None: self.assertTrue((self.state / "active-k3s").exists()) self.assertTrue(self.wal.exists()) # The Tailscale package is installed on every run of this test's host, whose dpkg-query stand-in reports none. - self.assertEqual([call for call in self.mutations(before) if not call.startswith(("apt-get", "systemctl enable --now tailscaled"))], []) + # The role writes its units before it decides what to do with them (here, for the first time), which reloads systemd; that touches no running unit. + self.assertEqual([call for call in self.mutations(before) if not call.startswith(("apt-get", "systemctl enable --now tailscaled", "systemctl daemon-reload"))], []) def demote_quickly(self) -> subprocess.CompletedProcess[str]: - (self.state / "voters").write_text("server1,True,true\nserver2,True,true\n") - self.assertEqual(self.run_three().returncode, 0) + self.running_as("server") self.wal = self.server_tree() (self.state / "voters").write_text("server1,True,true\nserver2,True,true\nserver3,True,true\nlaptop,True,true\n") return self.run_three(laptop={"github_runner_cluster_node_role": "agent", "github_runner_cluster_native_ready_timeout": 1}) diff --git a/tests/unit/test_native_server.py b/tests/unit/test_native_server.py index 1daa8cc3..2cd558bd 100644 --- a/tests/unit/test_native_server.py +++ b/tests/unit/test_native_server.py @@ -79,9 +79,7 @@ def test_a_native_server_does_not_join_while_another_voter_is_not_ready(self) -> self.assertNotIn("systemctl start k3s", self.calls()) def test_an_agent_is_promoted_in_place_once_its_node_is_empty(self) -> None: - self.assertEqual(self.run_three(laptop={"github_runner_cluster_node_role": "agent", "github_runner_cluster_voter_eligible": False}).returncode, 0) - # A three-host group with laptop as an agent has one Docker server: the first run is the laptop as an agent of server1, the second makes it the third server. - self.assertEqual(self.role_file(), "agent") + self.running_as("agent") password = self.root / "etc" / "rancher" / "node" / "password" password.parent.mkdir(parents=True) password.write_text("node-password") @@ -108,7 +106,7 @@ def test_an_agent_is_promoted_in_place_once_its_node_is_empty(self) -> None: self.assertNotIn("docker exec github-runner-k3s kubectl uncordon laptop", calls) def test_a_promotion_whose_runner_pods_do_not_finish_is_abandoned_with_the_node_uncordoned(self) -> None: - self.assertEqual(self.run_three(laptop={"github_runner_cluster_node_role": "agent"}).returncode, 0) + self.running_as("agent") (self.state / "runner-pods").write_text("pod/runner-abc12\n") before = len(self.calls()) result = self.run_three(laptop={"github_runner_cluster_native_runner_wait_timeout": 1}) From ce2bcb120ed689d5ef6dd3f9f8d1e89cc6d0090f Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 3 Nov 2026 10:57:52 +0000 Subject: [PATCH 09/10] ci: run the unit tests in five shards and treat the timeout as a hang guard The native server tests made the heaviest shard run past its timeout. The modules are dealt out by size, a rough stand-in for their work, so a fifth shard spreads them further, and the timeout is set above the heaviest shard's run as a guard against a hung test rather than a budget the shards are fitted to. --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43b1c870..2c5d8537 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -149,12 +149,12 @@ jobs: needs: verified if: ${{ !cancelled() && needs.verified.outputs.skip != 'true' }} runs-on: ubuntu-latest - # Each module is run on its own by discovery, dealt out by size so that the shards take about the same amount of work: the tests that run Ansible playbooks cost a process each, and the suite no longer fits one job. - timeout-minutes: 20 + # Each module is run on its own by discovery, dealt out by size, a rough stand-in for its work: the tests that run Ansible playbooks cost a process each, so a module's runs, not its bytes, decide its time, and the suite no longer fits one job. The timeout is a guard against a hung test, set above the heaviest shard's run, never a budget the shards are fitted to. + timeout-minutes: 30 strategy: fail-fast: false matrix: - shard: [0, 1, 2, 3] + shard: [0, 1, 2, 3, 4] steps: - uses: actions/checkout@v7 - name: Install Ansible From d76462129608252d5a98a6ac8fbf7022aaf0ba9d Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 3 Nov 2026 11:07:10 +0000 Subject: [PATCH 10/10] test(mesh): name each write of the native promotion uniquely and keep its error The writes during the promotion reused write_marker's random suffix, which repeats often enough over that many writes that a refusal for a name already taken would count as a lost write, and the error was discarded. Each write is now named by its sequence number, and a failed one records what kubectl said. --- tests/mesh/run.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index adbe99ff..b52fe823 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -969,12 +969,17 @@ native_is_voter() { [ "$(kubectl_on 1 get node "$native_host" -o 'jsonpath={.status.conditions[?(@.type=="EtcdIsVoter")].status}' 2>/dev/null)" = True ] } -# Writes to the cluster through node 1 once a second until the stop file appears, recording each write as ok or failed; a write is a ConfigMap under a name of its own (write_marker). +# Writes to the cluster through node 1 once a second until the stop file appears, recording each write as ok, or as failed with the time and what kubectl said. Each write is a ConfigMap named by its sequence number, so no two writes share a name: write_marker's random suffix repeats often enough over a promotion's worth of writes that a refusal for a name already taken would read as a lost write. write_throughout() { - local log="$1" stop="$2" + local log="$1" stop="$2" sequence=0 error : > "$log" until [ -e "$stop" ]; do - if write_marker 2>/dev/null; then echo ok >> "$log"; else echo "failed $(date -u +%H:%M:%S)" >> "$log"; fi + sequence=$((sequence + 1)) + if error=$(kubectl_on 1 create configmap "grtest-native-write-${sequence}" --from-literal=a=1 2>&1 >/dev/null > "$log" + else + echo "failed $(date -u +%H:%M:%S) ${error}" >> "$log" + fi sleep 1 done }