feat(cluster): run a native host as a k3s server and etcd voter

.github/workflows/ci.yml

@@ -149,12 +149,12 @@ jobs:
     needs: verified
     if: ${{ !cancelled() && needs.verified.outputs.skip != 'true' }}
     runs-on: ubuntu-latest
-    # Each module is run on its own by discovery, dealt out by size so that the shards take about the same amount of work: the tests that run Ansible playbooks cost a process each, and the suite no longer fits one job.
-    timeout-minutes: 20
+    # Each module is run on its own by discovery, dealt out by size, a rough stand-in for its work: the tests that run Ansible playbooks cost a process each, so a module's runs, not its bytes, decide its time, and the suite no longer fits one job. The timeout is a guard against a hung test, set above the heaviest shard's run, never a budget the shards are fitted to.
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
-        shard: [0, 1, 2, 3]
+        shard: [0, 1, 2, 3, 4]
     steps:
       - uses: actions/checkout@v7
       - name: Install Ansible

.github/workflows/mesh-integration.yml

@@ -1,6 +1,6 @@
 name: Mesh integration
 
-# Brings up three k3s nodes in Docker and joins them through Headscale with the cluster role (tests/mesh/run.sh), once per scenario: headscale_hosted, headscale_in_cluster, and the headscale_existing policy check, plus the role's refusal of a Docker Compose release that recreates the containers it has just built. headscale_in_cluster also runs with github_runner_cluster_docker_context pinned while the current Docker context points at nothing, which covers the cluster role's own docker calls and the recovery playbook's, and context_checks checks the role refuses a pin it cannot honour. regrade turns memory-pressure regrading on and steps a server down to etcd-only under (simulated) pressure and back up, checking that it keeps its etcd seat and vote and that no other server steps meanwhile. Only on changes that can affect how nodes join the mesh.
+# Brings up three k3s nodes in Docker and joins them through Headscale with the cluster role (tests/mesh/run.sh), once per scenario: headscale_hosted, headscale_in_cluster, and the headscale_existing policy check, plus the role's refusal of a Docker Compose release that recreates the containers it has just built. headscale_in_cluster also runs with github_runner_cluster_docker_context pinned while the current Docker context points at nothing, which covers the cluster role's own docker calls and the recovery playbook's, and context_checks checks the role refuses a pin it cannot honour. regrade turns memory-pressure regrading on and steps a server down to etcd-only under (simulated) pressure and back up, checking that it keeps its etcd seat and vote and that no other server steps meanwhile. native_server joins a native host (a systemd container) as an agent beside three Docker servers, then promotes it to a server in place: it must join etcd as a learner, be promoted to a voter, and keep the cluster writable throughout. Only on changes that can affect how nodes join the mesh.
 #
 # Runs on GitHub-hosted ubuntu-latest, whose Docker daemon runs the privileged containers the test needs (k3s, and Tailscale inside it). The work directory sits under runner.temp, which the job always has write access to.
 
@@ -38,6 +38,7 @@ jobs:
           - {scenario: hosted, compose: v5.5.1}
           - {scenario: in_cluster, compose: v5.5.1}
           - {scenario: regrade, compose: v5.5.1}
+          - {scenario: native_server, compose: v5.5.1}
           - {scenario: hosted, compose: v2.39.0}
           - {scenario: compose_faulty, compose: v2.38.2}
           - {scenario: in_cluster, compose: v5.5.1, pin: true}

examples/native-server/inventory.yml

@@ -0,0 +1,31 @@
+---
+# A cluster with a native server: node1 and node2 are Docker servers, and linux1 runs k3s directly under systemd as the third server and etcd voter, with its own Tailscale client as the node's device and its etcd peer URL on its Tailscale address. linux2 is a native agent staged as a standby (github_runner_cluster_voter_eligible): its server configuration is written and its name is on every server's certificate, but it stays an agent until its role is switched. See the cluster role's README, Native servers. Run with: ansible-playbook -i examples/native-server/inventory.yml exadev.github_runner.site --limit linux1,linux2 --check   # then without --check
+all:
+  children:
+    github_runner_cluster:
+      vars:
+        github_runner_cluster_tailnet_domain: example-tailnet.ts.net
+        github_runner_secrets_source: onepassword
+        github_runner_secrets_onepassword_vault: Example
+        github_runner_secrets_onepassword_item: github-runner
+      hosts:
+        node1:
+          ansible_host: node1.example.com
+        node2:
+          ansible_host: node2.example.com
+        linux1:
+          # Reached by its LAN name, not over the tailnet, as for a native agent.
+          ansible_host: linux1.local
+          github_runner_cluster_node_mode: native
+          # A native host is a server only when it says so; it never takes an automatic server place.
+          github_runner_cluster_node_role: server
+          github_runner_cluster_native_tailscale_leave_other_tailnet: true
+          github_runner_linux_host_k3s_prerequisites: true
+          github_runner_linux_host_keep_awake: true
+        linux2:
+          ansible_host: linux2.local
+          github_runner_cluster_node_mode: native
+          github_runner_cluster_voter_eligible: true
+          github_runner_cluster_native_tailscale_leave_other_tailnet: true
+          github_runner_linux_host_k3s_prerequisites: true
+          github_runner_linux_host_keep_awake: true

plugins/filter/cluster_native.py

@@ -1,4 +1,4 @@
-"""Filters the github_runner_cluster role's native node uses: which k3s release asset a host's architecture takes, the release and package versions the node's pins name, and which tailnet the host's Tailscale client is logged in to."""
+"""Filters the github_runner_cluster role's native node uses: which k3s release asset a host's architecture takes, the release and package versions the node's pins name, which tailnet the host's Tailscale client is logged in to, which role the host runs and whether it bootstraps a cluster, and whether the cluster's other voters are Ready enough for a server to start, restart or join."""
 
 from __future__ import annotations
 
@@ -82,6 +82,88 @@ def cluster_native_tailnet_login(status: Any, tailnet_domain: Any) -> dict[str,
     return {"state": state, "backend": backend, "domain": domain}
 
 
+NODE_ROLES = ("server", "agent")
+
+
+def _reason(result: dict[str, Any]) -> str:
+    """The last line of what a failed command or task said, or its exit status when it said nothing."""
+    lines = [line.strip() for line in str(result.get("stderr") or result.get("msg") or "").splitlines() if line.strip()]
+    return lines[-1] if lines else f"exit status {result.get('rc')}"
+
+
+def cluster_native_role(recorded: Any, agent_installed: Any, eligible: Any, inventory_role: Any, running: Any = ()) -> dict[str, str]:
+    """Decide the role a native host runs now and the role it is to run after this run.
+
+    :param recorded: what the host's role file holds (empty when it has none). :param agent_installed: whether the host has the k3s-agent unit, which a host set up before the role file existed has. :param eligible: whether the host is marked github_runner_cluster_voter_eligible. :param inventory_role: the role the topology gives the host. :param running: the roles whose unit is active now. :returns: ``current`` (``server``, ``agent``, or empty on a first install) and ``desired``. :raises AnsibleFilterError: if the role file holds anything but a role, or the inventory role or a running role is not one.
+
+    The role file is the host's own record of its role and outlives restarts, reboots and runs of the role. A switch writes the new role to it before it stops the old unit, so while exactly one unit runs, that unit's role is the one the host runs now, and a switch that stopped part-way (a member removal that did not finish) is taken up again by the next run. With neither unit running, the file says; a host from before the file existed ran only the agent, so its unit counts as the agent role. For a host marked eligible the inventory gives only the role it starts with: once the file names a role, that role is kept, since whatever moves votes between hosts changes it at run time and a run of the role must not undo that. For any other host the inventory decides.
+    """
+    text = "" if recorded is None else str(recorded).strip()
+    if text and text not in NODE_ROLES:
+        raise AnsibleFilterError(f"The role file holds {text!r}; it must be 'server' or 'agent'. Write the role the host runs into it, or remove it on a host that runs neither unit.")
+    if inventory_role not in NODE_ROLES:
+        raise AnsibleFilterError(f"{inventory_role!r} is not a node role (server or agent).")
+    active = [str(role) for role in running]
+    if any(role not in NODE_ROLES for role in active):
+        raise AnsibleFilterError(f"{active!r} are not node roles (server or agent).")
+    if len(active) == 1:
+        current = active[0]
+    else:
+        current = text or ("agent" if bool(agent_installed) else "")
+    desired = text if text and bool(eligible) else str(inventory_role)
+    return {"current": current, "desired": desired}
+
+
+def cluster_native_cluster_init(reads: Any, this_host: Any) -> dict[str, Any]:
+    """Say whether a native bootstrap host with no datastore of its own may start a new cluster: only when no other server of the inventory runs one.
+
+    :param reads: the loop results of reading the cluster through each server of the inventory (tasks/cluster_nodes_read.yml's github_runner_cluster_running_reads), each with its host as ``item`` and ``stdout`` "stopped" or "running" followed by node lines. :param this_host: this host's inventory name, whose own read says nothing about another cluster. :returns: ``cluster_init`` (true only when every other server read as stopped, or there is none), ``running`` (the hosts that answered as running a server) and ``unread`` (one line per server that could not be read, which leaves the question open, so the caller refuses).
+    """
+    if not isinstance(reads, list):
+        raise AnsibleFilterError("The reads must be a list of loop results.")
+    running: list[str] = []
+    unread: list[str] = []
+    for result in reads:
+        if not isinstance(result, dict):
+            raise AnsibleFilterError(f"{result!r} is not a loop result.")
+        host = str(result.get("item", "?"))
+        if host == str(this_host):
+            continue
+        lines = str(result.get("stdout", "")).split()
+        if result.get("unreachable") or result.get("rc") != 0:
+            unread.append(f"{host} could not be read ({_reason(result)})")
+        elif lines[:1] == ["running"]:
+            running.append(host)
+        elif lines[:1] != ["stopped"]:
+            unread.append(f"{host} gave an answer this check cannot read")
+    return {"cluster_init": not running and not unread, "running": running, "unread": unread}
+
+
+def cluster_native_voter_blockers(read: Any, node_name: Any, min_others: Any = 0) -> list[str]:
+    """Say why a native server must not start, restart or join now: every other etcd voter the cluster lists must be Ready, so the cluster keeps its quorum while this one is away, and a learner is only ever added beside a healthy membership.
+
+    The voters are read from the cluster, each node's etcd role label, never from the inventory, because a host's role can change at run time. A cluster that lists no other voter (a first bootstrap, or a single server restarting) has no quorum to keep.
+
+    :param read: the result of ``kubectl get nodes`` printing ``<node>,<Ready status>,<etcd role label>`` lines (commas, since either of the last two may be empty), as Ansible registers it. :param node_name: this host's node name, which is left out. :param min_others: the fewest other voters there must be, for a change that takes this server's vote away (a demotion); 0 for one that keeps it. :returns: one sentence per other voter that is not Ready, one more when there are fewer other voters than ``min_others``, or one sentence saying the nodes could not be listed, or an empty list.
+    """
+    if not isinstance(read, dict):
+        raise AnsibleFilterError(f"{read!r} is not a task result.")
+    if read.get("unreachable") or read.get("rc") != 0:
+        return [f"the cluster's nodes could not be listed ({_reason(read)}), so whether the other voters are Ready cannot be told"]
+    problems: list[str] = []
+    others: list[str] = []
+    for line in str(read.get("stdout", "")).splitlines():
+        fields = [field.strip() for field in line.split(",")]
+        if len(fields) != 3 or fields[0] == str(node_name) or fields[2] != "true":
+            continue
+        others.append(fields[0])
+        if fields[1] != "True":
+            problems.append(f"voter {fields[0]} is not Ready (its Ready condition is {fields[1] or 'unknown'})")
+    if len(others) < int(min_others):
+        problems.append(f"only {len(others)} other voters ({', '.join(others) or 'none'}) would remain, fewer than the {int(min_others)} a change that takes this server's vote away leaves")
+    return problems
+
+
 class FilterModule:
     """Registers the native node's filters with Ansible."""
 
@@ -92,4 +174,7 @@ def filters(self) -> dict[str, Any]:
             "cluster_native_k3s_release": cluster_native_k3s_release,
             "cluster_native_tailscale_package_version": cluster_native_tailscale_package_version,
             "cluster_native_tailnet_login": cluster_native_tailnet_login,
+            "cluster_native_role": cluster_native_role,
+            "cluster_native_cluster_init": cluster_native_cluster_init,
+            "cluster_native_voter_blockers": cluster_native_voter_blockers,
         }

plugins/filter/cluster_topology.py

@@ -34,6 +34,7 @@ def _describe_host(name: str, host_vars: Mapping[str, Any], errors: list[str]) -
         "name": name,
         "role_override": role if role in NODE_ROLES else "",
         "native": mode == "native",
+        "eligible": boolean(host_vars.get("github_runner_cluster_voter_eligible", False), strict=False),
         "bootstrap_override": None if bootstrap_override is None else boolean(bootstrap_override, strict=False),
         "node_name": node_name,
         "address": address,
@@ -52,9 +53,9 @@ def _largest_odd_at_most(limit: int) -> int:
 def github_runner_cluster_topology(members: list[str], hostvars: Mapping[str, Mapping[str, Any]], max_servers: Any, api_port: Any) -> dict[str, Any]:
     """Decide every cluster member's part from inventory order and per-host overrides.
 
-    :param members: the cluster group's hosts, in inventory order. :param hostvars: Ansible's hostvars, read for each member's per-host overrides. :param max_servers: the most servers to choose automatically. :param api_port: the k3s API port a joining node connects to. :returns: ``errors`` (empty when the result is valid), ``servers`` and ``bootstrap`` (host names), and ``hosts``, mapping each member to its ``role``, ``bootstrap``, ``node_name``, ``address``, ``server_url`` and ``tls_sans``. :raises AnsibleFilterError: if max_servers or api_port is not an integer.
+    :param members: the cluster group's hosts, in inventory order. :param hostvars: Ansible's hostvars, read for each member's per-host overrides. :param max_servers: the most servers to choose automatically. :param api_port: the k3s API port a joining node connects to. :returns: ``errors`` (empty when the result is valid), ``servers`` and ``bootstrap`` (host names), and ``hosts``, mapping each member to its ``role``, ``bootstrap``, ``node_name``, ``address``, ``server_url``, ``tls_sans``, ``native``, ``eligible`` and ``peer_server_urls`` (the join URL of every other server, in inventory order). :raises AnsibleFilterError: if max_servers or api_port is not an integer.
 
-    The automatic server count is the largest odd number no greater than both the number of hosts that are not forced to be agents (a host with github_runner_cluster_node_mode: native is always an agent) and max_servers, because etcd needs a majority and an even member count tolerates no more failures than the odd count below it. The first hosts in inventory order become servers; the first server not told to join elsewhere bootstraps the cluster, and every other node joins it.
+    The automatic server count is the largest odd number no greater than both the number of hosts that are not forced to be agents and max_servers, because etcd needs a majority and an even member count tolerates no more failures than the odd count below it. A host with github_runner_cluster_node_mode: native is never given a server place automatically and does not count towards that number: it is a server only when it sets github_runner_cluster_node_role: server or github_runner_cluster_bootstrap: true, so an inventory whose native hosts set neither keeps every one of them an agent whatever max_servers allows. The first hosts in inventory order become servers; the first server not told to join elsewhere bootstraps the cluster, and every other node joins it. Every server's TLS SAN list names every server and every host marked github_runner_cluster_voter_eligible, so a standby that is later promoted is already covered by every certificate.
     """
     try:
         server_limit = int(max_servers)
@@ -66,11 +67,9 @@ def github_runner_cluster_topology(members: list[str], hostvars: Mapping[str, Ma
     if server_limit < 1:
         errors.append(f"github_runner_cluster_max_servers must be at least 1 (got {server_limit}).")
     hosts = [_describe_host(name, hostvars[name], errors) for name in members]
-    # A native node runs k3s as a systemd service on the host and only ever as an agent, so it is never offered a server place, and a host that asks for one is refused.
+    # A native node is a server only when its host asks to be one: with automatic placement it would otherwise take a server place silently once native hosts stopped being forced to agents, so a native host that sets neither the role nor the bootstrap is an agent.
     for host in hosts:
-        if host["native"]:
-            if host["role_override"] == "server" or host["bootstrap_override"] is True:
-                errors.append(f"{host['name']} sets github_runner_cluster_node_mode: native, which runs a k3s agent only; it cannot be a server or bootstrap the cluster.")
+        if host["native"] and not host["role_override"] and host["bootstrap_override"] is not True:
             host["role_override"] = "agent"
     # A host told to bootstrap has to be a server, so it takes one of the server places rather than competing for one in inventory order.
     for host in hosts:
@@ -121,13 +120,14 @@ def github_runner_cluster_topology(members: list[str], hostvars: Mapping[str, Ma
         else:
             host["server_url"] = ""
             errors.append(f"{host['name']} has nothing to join: no host in the cluster group bootstraps the cluster, and it sets no github_runner_cluster_server_url.")
-        host["tls_sans"] = host["tls_sans_override"] or [server["address"] for server in servers]
+        host["tls_sans"] = host["tls_sans_override"] or [candidate["address"] for candidate in hosts if candidate["role"] == "server" or candidate["eligible"]]
+        host["peer_server_urls"] = [f"https://{server['address']}:{port}" for server in servers if server is not host]
 
     return {
         "errors": errors,
         "servers": [host["name"] for host in servers],
         "bootstrap": bootstrap_host["name"] if bootstrap_host is not None else "",
-        "hosts": {host["name"]: {key: host[key] for key in ("role", "bootstrap", "node_name", "address", "server_url", "tls_sans")} for host in hosts},
+        "hosts": {host["name"]: {key: host[key] for key in ("role", "bootstrap", "node_name", "address", "server_url", "tls_sans", "native", "eligible", "peer_server_urls")} for host in hosts},
     }