test(cluster): cover the recorded Docker role and its Lease guard

The stand-ins, the env rendering tests and the role tests follow the single recorded role file.

Author: Mearman

tests/cluster_versions/render_env.yml

@@ -1,6 +1,6 @@
 ---
-# Renders the cluster role's env.j2 and env-agent.j2 with the role's defaults and placeholder values for the settings this test does not look at, writing them to env_output and env_agent_output, and, when compose_files_output is set, the Compose file list the node's project would use, for tests/unit/test_cluster_versions.py, test_cluster_etcd_timing.py and test_cluster_instance.py. Role variables are passed as extra variables, which outrank the defaults loaded here.
-- name: Render the node .env templates
+# Renders the cluster role's env.j2, the one .env a Docker node gets whichever role it runs, with the role's defaults and placeholder values for the settings this test does not look at, writing it to env_output, and, when compose_files_output is set, the Compose file list the node's project would use, for tests/unit/test_cluster_versions.py, test_cluster_etcd_timing.py, test_cluster_instance.py, test_cluster_voters.py and test_docker_role.py. Role variables are passed as extra variables, which outrank the defaults loaded here.
+- name: Render the node .env
   hosts: localhost
   connection: local
   gather_facts: false
@@ -22,12 +22,6 @@
         dest: "{{ env_output }}"
         mode: "0600"
 
-    - name: Render env-agent.j2
-      ansible.builtin.copy:
-        content: "{{ lookup('ansible.builtin.template', playbook_dir ~ '/../../roles/github_runner_cluster/templates/env-agent.j2') }}"
-        dest: "{{ env_agent_output }}"
-        mode: "0600"
-
     - name: Write the Compose files the node's project would use
       ansible.builtin.copy:
         content: "{{ github_runner_cluster_compose_file_list | to_json }}"

tests/host_network/play.yml

@@ -7,6 +7,7 @@
   vars:
     github_runner_cluster_docker_cli_environment: {}
     github_runner_cluster_container_name: github-runner-k3s
+    github_runner_cluster_agent_container_name: github-runner-k3s-agent
     github_runner_cluster_compose_project: github-runner
     github_runner_cluster_network_probe_image: "busybox:1.37"
     github_runner_cluster_cluster_read_timeout: 30

tests/regrade/render_env.yml

@@ -9,6 +9,7 @@
   vars:
     github_runner_cluster_effective_node_name: node2
     github_runner_cluster_effective_node_role: server
+    github_runner_cluster_docker_role: server
     github_runner_cluster_topology:
       servers: [node1, node2, node3]
       hosts:

tests/regrade/service.yml

@@ -6,6 +6,7 @@
   gather_facts: false
   vars:
     github_runner_cluster_effective_node_role: server
+    github_runner_cluster_docker_role: server
     github_runner_cluster_dir: /srv/github-runner
     github_runner_cluster_docker_cli_context: ""
     github_runner_cluster_docker_cli_environment: {}

tests/unit/cluster_stand_ins.py

@@ -54,7 +54,7 @@ def flatten(tasks: list[dict[str, Any]]) -> list[dict[str, Any]]:
     return found
 
 
-# Stands in for the docker CLI, with its state in files under $DOCKER_STUB: containers/<name>.state (what docker ps prints), .image, .hash (the Compose configuration hash label), .node/ (the files of the container's /etc/rancher/node), .volume (the volume mounted there) and .cp-fails; volumes/<name>/ and labels/<name>; rendered/<project>.txt (what docker compose config --hash prints); cluster.nodes (what kubectl get nodes prints, "<node> <Ready status>" per line); and containers/<name>.kubectl-fails and never-ready-<node>; leases/<node> counts reads of a node's lease, whose renewal is older than its container's start on the first read and later after that, which is also when the node reports Ready (kubelet confirming itself) unless never-ready-<node> exists. Every call is appended to calls.log with its DOCKER_CONTEXT, and the owner and mode of each file docker cp extracts to owners.log. docker cp moves a real tar stream between processes, so a pipeline that is wired wrongly moves nothing.
+# Stands in for the docker CLI, with its state in files under $DOCKER_STUB: containers/<name>.state (what docker ps prints), .image, .hash (the Compose configuration hash label), .node/ (the files of the container's /etc/rancher/node), .volume (the volume mounted there) and .cp-fails; volumes/<name>/ and labels/<name>; rendered/<project>.txt (what docker compose config --hash prints); cluster.nodes (what kubectl get nodes prints, "<node> <Ready status>" per line); fleet-regrade.lease (what kubectl get lease fleet-regrade prints, nothing when absent); and containers/<name>.kubectl-fails and never-ready-<node>; leases/<node> counts reads of a node's lease, whose renewal is older than its container's start on the first read and later after that, which is also when the node reports Ready (kubelet confirming itself) unless never-ready-<node> exists. Every call is appended to calls.log with its DOCKER_CONTEXT, and the owner and mode of each file docker cp extracts to owners.log. docker cp moves a real tar stream between processes, so a pipeline that is wired wrongly moves nothing.
 DOCKER_STUB = """#!{python}
 import os, re, sys, tarfile
 stub = os.environ["DOCKER_STUB"]
@@ -176,7 +176,9 @@ def set_node(node, status):
     if rest[:2] == ["kubectl", "get"]:
         if read("containers", container + ".kubectl-fails") is not None:
             die("The connection to the server 127.0.0.1:6443 was refused")
-        if rest[2] == "nodes":
+        if rest[2:4] == ["lease", "fleet-regrade"]:
+            print(read("fleet-regrade.lease") or "", end="")
+        elif rest[2] == "nodes":
             print(read("cluster.nodes") or "", end="")
         elif rest[2] == "node":
             print(dict(line.split() for line in (read("cluster.nodes") or "").splitlines() if len(line.split()) == 2).get(rest[3], ""), end="")
@@ -288,9 +290,9 @@ def tearDown(self) -> None:
         # Whatever a test did, no real tool was reached.
         self.assertFalse((self.stub / "forbidden.log").exists(), (self.stub / "forbidden.log").read_text() if (self.stub / "forbidden.log").exists() else "")
 
-    def hosts(self, servers: tuple[str, ...], agents: tuple[str, ...] = (), **extra: Any) -> tuple[dict[str, Any], dict[str, Any]]:
-        """The inventory's hosts and the group variables for a cluster whose servers and agents are named, each host's container named for it."""
-        topology = {"servers": list(servers), "hosts": {name: {"role": "server" if name in servers else "agent", "bootstrap": name == servers[0], "node_name": name} for name in (*servers, *agents)}}
+    def hosts(self, servers: tuple[str, ...], agents: tuple[str, ...] = (), eligible: tuple[str, ...] = (), **extra: Any) -> tuple[dict[str, Any], dict[str, Any]]:
+        """The inventory's hosts and the group variables for a cluster whose servers and agents are named, those in ``eligible`` marked github_runner_cluster_voter_eligible, each host's containers named for it."""
+        topology = {"servers": list(servers), "hosts": {name: {"role": "server" if name in servers else "agent", "bootstrap": name == servers[0], "node_name": name, "address": name, "native": False, "eligible": name in eligible} for name in (*servers, *agents)}}
         hosts: dict[str, Any] = {}
         for name in (*servers, *agents):
             server = name in servers
@@ -299,8 +301,11 @@ def hosts(self, servers: tuple[str, ...], agents: tuple[str, ...] = (), **extra:
             (self.work / "nodes" / name / "docker-compose.yml").write_text("services: {}\n")
             hosts[name] = {
                 "github_runner_cluster_effective_node_role": "server" if server else "agent",
+                # The role tasks/docker_role.yml decides, which a test that runs it sets again.
+                "github_runner_cluster_docker_role": "server" if server else "agent",
                 "github_runner_cluster_effective_node_name": name,
                 "github_runner_cluster_container_name": f"{name}-k3s",
+                "github_runner_cluster_agent_container_name": f"{name}-k3s-agent",
                 "github_runner_cluster_dir": str(self.work / "nodes" / name),
                 "github_runner_cluster_compose_project": name,
                 "github_runner_cluster_compose_file_list": ["docker-compose.yml"],

tests/unit/native_stand_ins.py

@@ -213,6 +213,8 @@ def make_stand_ins(self) -> None:
   "get node "*removed-node-name*) if [ -e "$state/member-removed" ]; then printf 'laptop-0a1b2c3d,'; else printf ',laptop-0a1b2c3d'; fi ;;
   "annotate --overwrite node "*"etcd.k3s.cattle.io/remove=true"*) [ -e "$state/member-stays" ] || touch "$state/member-removed" ;;
   "get node "*) if [ -e "$state/node-ready" ]; then cat "$state/node-ready"; else printf 'True,True'; fi ;;
+  # The regrade Lease, as JSON, and nothing when there is none; any other Lease is a node's, whose renew time is asked for.
+  "get lease fleet-regrade "*) [ ! -e "$state/fleet-regrade-lease" ] || cat "$state/fleet-regrade-lease" ;;
   "get lease "*) if [ -e "$state/lease" ]; then cat "$state/lease"; else echo 2099-01-01T00:00:00Z; fi ;;
   "get pods "*) [ -e "$state/runner-pods" ] && cat "$state/runner-pods"; exit 0 ;;
   cordon*|uncordon*|drain*|label*|annotate*) ;;

tests/unit/test_cluster_compose_files_in_use.py

@@ -142,7 +142,7 @@ def test_the_files_are_placed_then_the_list_is_worked_out_then_the_mesh_and_ever
             self.assertGreater(matching[0], worked_out, later)
 
     def test_no_task_names_the_whole_list_to_a_compose_command_except_where_it_is_defined_or_checked(self) -> None:
-        allowed = {"defaults/main.yml", "tasks/compose_files.yml", "tasks/compose_files_in_use.yml", "templates/env.j2", "templates/env-agent.j2"}
+        allowed = {"defaults/main.yml", "tasks/compose_files.yml", "tasks/compose_files_in_use.yml", "templates/env.j2"}
         offenders = []
         for path in sorted(ROLE.rglob("*")):
             relative = str(path.relative_to(ROLE))

tests/unit/test_cluster_etcd_timing.py

@@ -107,26 +107,21 @@ def test_an_agent_with_other_timing_is_not_compared_with_the_servers(self) -> No
         result = self.check(hosts)
         self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
 
-    def render_env(self, *extra: str) -> tuple[list[str], list[str]]:
-        env, agent = self.work / "env", self.work / "env-agent"
-        args = [ANSIBLE_PLAYBOOK, str(ROOT / "tests" / "cluster_versions" / "render_env.yml"), "-e", f"env_output={env}", "-e", f"env_agent_output={agent}", "-e", "github_runner_cluster_dir=/srv/github-runner", *extra]
+    def render_env(self, *extra: str) -> list[str]:
+        env = self.work / "env"
+        args = [ANSIBLE_PLAYBOOK, str(ROOT / "tests" / "cluster_versions" / "render_env.yml"), "-e", f"env_output={env}", "-e", "github_runner_cluster_dir=/srv/github-runner", *extra]
         result = subprocess.run(args, capture_output=True, text=True, check=False, env=ansible_env(), cwd=self.work)
         self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
-        return env.read_text().splitlines(), agent.read_text().splitlines()
+        return env.read_text().splitlines()
 
-    def test_the_server_env_carries_both_values_when_set(self) -> None:
-        server, _ = self.render_env("-e", f"{HEARTBEAT}=500", "-e", f"{ELECTION}=5000")
-        self.assertIn("K3S_ETCD_HEARTBEAT_INTERVAL=500", server)
-        self.assertIn("K3S_ETCD_ELECTION_TIMEOUT=5000", server)
+    def test_the_env_carries_both_values_when_set(self) -> None:
+        # The agent service takes neither from it: see ComposeFileTest.
+        lines = self.render_env("-e", f"{HEARTBEAT}=500", "-e", f"{ELECTION}=5000")
+        self.assertIn("K3S_ETCD_HEARTBEAT_INTERVAL=500", lines)
+        self.assertIn("K3S_ETCD_ELECTION_TIMEOUT=5000", lines)
 
-    def test_the_server_env_names_neither_by_default(self) -> None:
-        server, agent = self.render_env()
-        for lines in (server, agent):
-            self.assertEqual([line for line in lines if "ETCD" in line], [])
-
-    def test_the_agent_env_never_carries_them(self) -> None:
-        _, agent = self.render_env("-e", f"{HEARTBEAT}=500", "-e", f"{ELECTION}=5000")
-        self.assertEqual([line for line in agent if "ETCD" in line], [])
+    def test_the_env_names_neither_by_default(self) -> None:
+        self.assertEqual([line for line in self.render_env() if "ETCD" in line], [])
 
 
 class ComposeFileTest(unittest.TestCase):

tests/unit/test_cluster_instance.py

@@ -140,34 +140,32 @@ def setUp(self) -> None:
         self.work = Path(tempfile.mkdtemp())
         self.addCleanup(shutil.rmtree, self.work)
 
-    def render(self, *extra: str) -> tuple[list[str], list[str], list[str]]:
-        env, agent, files = self.work / "env", self.work / "env-agent", self.work / "files.json"
-        args = [ANSIBLE_PLAYBOOK, str(ROOT / "tests" / "cluster_versions" / "render_env.yml"), "-e", f"env_output={env}", "-e", f"env_agent_output={agent}", "-e", f"compose_files_output={files}", "-e", "github_runner_cluster_dir=/srv/github-runner", *extra]
+    def render(self, *extra: str) -> tuple[list[str], list[str]]:
+        env, files = self.work / "env", self.work / "files.json"
+        args = [ANSIBLE_PLAYBOOK, str(ROOT / "tests" / "cluster_versions" / "render_env.yml"), "-e", f"env_output={env}", "-e", f"compose_files_output={files}", "-e", "github_runner_cluster_dir=/srv/github-runner", *extra]
         result = subprocess.run(args, capture_output=True, text=True, check=False, env=ansible_env(), cwd=self.work)
         self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
         compose_files: list[str] = json.loads(files.read_text())
-        return env.read_text().splitlines(), agent.read_text().splitlines(), compose_files
+        return env.read_text().splitlines(), compose_files
 
     def test_the_defaults_write_the_existing_names_and_port_and_add_no_compose_file(self) -> None:
-        server, agent, files = self.render()
-        self.assertIn(f"K3S_CONTAINER_NAME={SERVER_CONTAINER}", server)
-        self.assertIn(f"K3S_API_PORT={K3S_PORT}", server)
-        self.assertIn(f"K3S_AGENT_CONTAINER_NAME={AGENT_CONTAINER}", agent)
-        self.assertIn(f"K3S_CONTAINER_NAME={SERVER_CONTAINER}", agent)
-        self.assertIn(f"K3S_API_PORT={K3S_PORT}", agent)
+        lines, files = self.render()
+        self.assertIn(f"K3S_CONTAINER_NAME={SERVER_CONTAINER}", lines)
+        self.assertIn(f"K3S_AGENT_CONTAINER_NAME={AGENT_CONTAINER}", lines)
+        self.assertIn(f"K3S_API_PORT={K3S_PORT}", lines)
         self.assertEqual(files, ["docker-compose.yml"])
-        self.assertIn("COMPOSE_FILE=/srv/github-runner/docker-compose.yml", server)
+        self.assertIn("COMPOSE_FILE=/srv/github-runner/docker-compose.yml", lines)
 
-    def test_a_second_cluster_s_values_reach_both_templates_and_the_port_adds_the_override(self) -> None:
-        server, agent, files = self.render("-e", "github_runner_cluster_container_name=second-k3s", "-e", "github_runner_cluster_agent_container_name=second-k3s-agent", "-e", "github_runner_cluster_api_port=6543")
-        self.assertIn("K3S_CONTAINER_NAME=second-k3s", server)
-        self.assertIn("K3S_API_PORT=6543", server)
-        self.assertIn("K3S_AGENT_CONTAINER_NAME=second-k3s-agent", agent)
+    def test_a_second_cluster_s_values_reach_the_env_and_the_port_adds_the_override(self) -> None:
+        lines, files = self.render("-e", "github_runner_cluster_container_name=second-k3s", "-e", "github_runner_cluster_agent_container_name=second-k3s-agent", "-e", "github_runner_cluster_api_port=6543")
+        self.assertIn("K3S_CONTAINER_NAME=second-k3s", lines)
+        self.assertIn("K3S_API_PORT=6543", lines)
+        self.assertIn("K3S_AGENT_CONTAINER_NAME=second-k3s-agent", lines)
         self.assertEqual(files, ["docker-compose.yml", "docker-compose.api-port.yml"])
-        self.assertIn("COMPOSE_FILE=/srv/github-runner/docker-compose.yml:/srv/github-runner/docker-compose.api-port.yml", server)
+        self.assertIn("COMPOSE_FILE=/srv/github-runner/docker-compose.yml:/srv/github-runner/docker-compose.api-port.yml", lines)
 
     def test_the_default_port_written_as_a_string_adds_no_override(self) -> None:
-        _, _, files = self.render("-e", '{"github_runner_cluster_api_port": "6443"}')
+        _, files = self.render("-e", '{"github_runner_cluster_api_port": "6443"}')
         self.assertEqual(files, ["docker-compose.yml"])
 
 
@@ -319,7 +317,8 @@ def containers(self, *lines: str) -> None:
         (self.stub / "containers").write_text("\n".join(lines) + "\n")
 
     def run_check(self, *extra: str, role: str = "server", group_vars: dict[str, Any] | None = None, hosts: dict[str, dict[str, Any]] | None = None) -> subprocess.CompletedProcess[str]:
-        node = {"github_runner_cluster_effective_node_role": role, "github_runner_cluster_docker_cli_environment": {}, "github_runner_cluster_dir": "/srv/second"}
+        # The role tasks/docker_role.yml would decide is the inventory's, for a host not marked eligible.
+        node = {"github_runner_cluster_effective_node_role": role, "github_runner_cluster_docker_role": role, "github_runner_cluster_docker_cli_environment": {}, "github_runner_cluster_dir": "/srv/second"}
         group = {"vars": {"github_runner_cluster_compose_project": "second", **(group_vars or {})}, "hosts": {name: node | values for name, values in (hosts or {"node1": {}}).items()}}
         inventory = self.work / "inventory.yml"
         inventory.write_text(yaml.safe_dump({"all": {"children": {"github_runner_cluster": group}}}))

tests/unit/test_cluster_regrade.py

@@ -131,6 +131,12 @@ def test_an_agent_is_refused(self) -> None:
         result = self.refused("node4 is an agent", {name: {} for name in ("node1", "node2", "node3", "node4")}, **{ENABLED: True})
         self.assertEqual(re.findall(r"fatal: \[(node\d)\]", result.stdout), ["node4"])
 
+    def test_a_docker_agent_that_may_become_a_server_may_regrade(self) -> None:
+        # The service runs on such a host only while the role it records is server (tasks/regrade_service.yml), so the setting it keeps across a swap is not a mistake.
+        hosts = {name: {ENABLED: True} for name in ("node1", "node2", "node3")} | {"node4": {ENABLED: True, "github_runner_cluster_voter_eligible": True}}
+        result = self.check(hosts)
+        self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+
     def test_regrading_only_the_servers_of_a_group_with_an_agent_passes(self) -> None:
         hosts = {"node1": {ENABLED: True}, "node2": {ENABLED: True}, "node3": {ENABLED: True}, "node4": {}}
         result = self.check(hosts)
@@ -466,14 +472,16 @@ def test_the_settings_are_checked_before_the_role_changes_anything(self) -> None
         self.assertLess(order.index("topology.yml"), order.index("regrade.yml"))
         self.assertLess(order.index("regrade.yml"), order.index("Create this host's Compose project directory"))
 
-    def test_the_service_runs_on_servers_after_the_node_is_started(self) -> None:
+    def test_the_service_runs_on_servers_after_the_node_is_started_and_is_removed_from_agents(self) -> None:
         tasks = yaml.safe_load((ROLE / "tasks" / "main.yml").read_text())
-        [block] = [task for task in tasks if "block" in task and task.get("when") == 'github_runner_cluster_effective_node_role == "server"']
+        [block] = [task for task in tasks if "block" in task and task.get("when") == 'github_runner_cluster_docker_role == "server"']
         steps = [step.get("ansible.builtin.include_tasks") for step in block["block"]]
         self.assertLess(steps.index("start_servers.yml"), steps.index("regrade_service.yml"))
         self.assertLess(steps.index("regrade_service.yml"), steps.index("end_checks.yml"))
-        [agent] = [task for task in tasks if "block" in task and task.get("when") == 'github_runner_cluster_effective_node_role == "agent"']
-        self.assertNotIn("regrade_service.yml", [step.get("ansible.builtin.include_tasks") for step in agent["block"]])
+        # On an agent it only removes the service, which a host that ran as a server before its recorded role changed may still run.
+        [agent] = [task for task in tasks if "block" in task and task.get("when") == 'github_runner_cluster_docker_role == "agent"']
+        agent_steps = [step.get("ansible.builtin.include_tasks") for step in agent["block"]]
+        self.assertLess(agent_steps.index("start_agent.yml"), agent_steps.index("regrade_service.yml"))
 
     def test_the_wait_for_ready_leaves_out_a_server_regrading_has_stepped_down(self) -> None:
         text = (ROLE / "tasks" / "wait_for_ready.yml").read_text()