From ca69f5b69bbf486923ef79dc8b28b363fc5f74c3 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 3 Nov 2026 17:52:55 +0000 Subject: [PATCH 1/2] feat(cluster): publish the medium of each host's link as a Node label A node in a container or a VM sees only its own virtual interface, so the role controller's probe cannot tell a Mac's Wi-Fi from its Ethernet. The role now reads the medium on the host itself, in every run that publishes the voter labels, and passes it to k3s as github-runner.exadev.com/link (wired, wifi or unknown) beside them. On a Mac with a bridged Colima profile the medium is that of the interface network.interface names, mapped through the hardware port networksetup lists it under; on any other Mac runtime, and on Linux, it is that of the interface the default route leaves by, on Linux from its /sys/class/net entry. files/node-link.sh does the reading so that the probe can share it. --- .github/workflows/ci.yml | 1 + roles/github_runner_cluster/README.md | 2 + roles/github_runner_cluster/defaults/main.yml | 12 +- .../github_runner_cluster/files/node-link.sh | 83 +++++++ roles/github_runner_cluster/tasks/link.yml | 40 ++++ roles/github_runner_cluster/tasks/main.yml | 4 + .../tasks/native/main.yml | 4 + tests/cluster_link/link.yml | 26 +++ tests/cluster_versions/render_env.yml | 2 + tests/unit/native_stand_ins.py | 12 + tests/unit/test_cluster_voters.py | 17 +- tests/unit/test_native_node.py | 8 +- tests/unit/test_node_link.py | 210 ++++++++++++++++++ 13 files changed, 410 insertions(+), 11 deletions(-) create mode 100755 roles/github_runner_cluster/files/node-link.sh create mode 100644 roles/github_runner_cluster/tasks/link.yml create mode 100644 tests/cluster_link/link.yml create mode 100644 tests/unit/test_node_link.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44b00057..998f669f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -250,6 +250,7 @@ jobs: tarball=$(ls exadev-github_runner-*.tar.gz) tar -tzf "$tarball" | grep -Fx roles/github_runner_cluster/files/docker-compose.yml tar -tzf "$tarball" | grep -Fx roles/github_runner_cluster/files/k3s/node-entrypoint.sh + tar -tzf "$tarball" | grep -Fx roles/github_runner_cluster/files/node-link.sh for file in Dockerfile fleet-regrade.sh fleet-regrade-core.sh fleet-regrade-k3s.sh fleet-regrade-docker.sh fleet-regrade-systemd.sh; do tar -tzf "$tarball" | grep -Fx "roles/github_runner_cluster/files/regrade/${file}" done diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index 755aeb5d..203b74ba 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -319,6 +319,8 @@ While the controller is on, the role refuses a target that is even, under 3, or The role publishes the two host settings as Node labels, `github-runner.exadev.com/voter-eligible` and `github-runner.exadev.com/voter-pinned` (`github_runner_cluster_voter_eligible_label` and `github_runner_cluster_voter_pinned_label`), by passing them to k3s as `--node-label` on every runtime: a Docker server or agent, and a native one. The kubelet writes its labels to its Node each time it starts and never removes one, so while the controller is on, or a host is marked eligible or pinned, a host carries both labels with the value `true` or `false`, and turning a setting off writes `false` over the `true`. A host that uses none of this gets no label and no extra Compose file, so a fleet that does not use the controller keeps every node's configuration as it is and no container is recreated by an upgrade. A host's labels change with its next run of the role and the recreation or restart that follows it. +Beside them the role publishes the medium of the host's link, `wired`, `wifi` or `unknown`, as `github-runner.exadev.com/link` (`github_runner_cluster_link_label`), read on the host in each run that publishes the other two (`tasks/link.yml`, with `files/node-link.sh`). A node in a container or a VM cannot read it itself: it sees a veth, or a VM's virtio NIC, whatever the host beneath it uses. On a Mac whose node's Colima profile is bridged (`network.mode: bridged` with `network.address: true` in the profile's `colima.yaml`), the medium is that of the interface the VM is bridged to (`network.interface`), which is not the interface the Mac's own traffic takes when macOS prefers another, such as an Ethernet adapter over the Wi-Fi the VM is bridged to; for any other Mac runtime it is that of the interface the Mac's default route leaves by. The medium of a Mac interface is the name of the hardware port `networksetup -listallhardwareports` lists it under: Wi-Fi is `wifi`, a port whose name says Ethernet or LAN (built-in Ethernet, Thunderbolt Ethernet, a USB LAN adapter) is `wired`, and any other is `unknown`. On Linux it is that of the interface the default route leaves by, from its `/sys/class/net` entry: a wireless phy is `wifi`, a device is `wired`, and a virtual interface (a bridge, a bond, a tunnel) is `unknown`, since the link beneath it is not read. The label is the medium as the last run of the role found it, so a host whose link changes (a cable plugged in, a Wi-Fi fallback) carries the new medium after its next run of the role and the recreation or restart that follows it, as for the other labels; a host that publishes them already is recreated or restarted by the first run that adds this one. + ## Networking Every node's k3s uses its mesh address for everything: its node IP, etcd's peer URLs, the API server and pod traffic. Which physical path the mesh takes between two nodes is Tailscale's choice, among the endpoints each node's Tailscale client can offer its peers. On Compose's default bridge network (`github_runner_cluster_network_mode: bridge`, the default) the client sits behind the Docker host's NAT and, inside a Docker VM, behind the VM's too, so it has no address its peers can reach and every pair of nodes is relayed, even on one LAN. diff --git a/roles/github_runner_cluster/defaults/main.yml b/roles/github_runner_cluster/defaults/main.yml index ae804122..506dcf23 100644 --- a/roles/github_runner_cluster/defaults/main.yml +++ b/roles/github_runner_cluster/defaults/main.yml @@ -62,11 +62,17 @@ github_runner_cluster_voters: 3 # Derived, never set directly: the Node labels the role publishes the two host settings under, with the value true or false on every node, so that turning a setting off writes false over the true a node registered with (a kubelet adds and overwrites the labels it is given and never removes one). The ARC role's role controller reads them (github_runner_arc_role_controller_eligible_label and github_runner_arc_role_controller_pinned_label default to these), as a label with the value true. github_runner_cluster_voter_eligible_label: "github-runner.exadev.com/voter-eligible" github_runner_cluster_voter_pinned_label: "github-runner.exadev.com/voter-pinned" -# Derived, never set directly: this host's labels, as the key=value entries every runtime passes to k3s as --node-label, empty unless the role controller is on or this host is marked eligible or pinned, so that a fleet that uses none of them keeps its nodes' configuration as it is (a Docker node is not recreated by an upgrade). +# The Node label the role publishes the medium of this host's link under, wired, wifi or unknown, which tasks/link.yml reads on the host: the medium of the interface a Mac's Colima VM is bridged to (or of the interface the Mac's default route leaves by, for a VM that is not bridged or a runtime that is not Colima), or of the interface a Linux host's default route leaves by. The role controller's probe reports it (github_runner_arc_role_controller_link_label defaults to this), since a node in a container or a VM cannot see the link beneath its own virtual interface. +github_runner_cluster_link_label: "github-runner.exadev.com/link" +# Derived, never set directly: whether this host publishes the labels above, which it does while the role controller is on or the host is marked eligible or pinned, so that a fleet that uses none of them keeps its nodes' configuration as it is (a Docker node is not recreated by an upgrade). +github_runner_cluster_publishes_node_labels: >- + {{ github_runner_cluster_role_controller | bool or github_runner_cluster_voter_eligible | bool or github_runner_cluster_voter_pinned | bool }} +# Derived, never set directly: this host's labels, as the key=value entries every runtime passes to k3s as --node-label, empty unless github_runner_cluster_publishes_node_labels. github_runner_cluster_link is the medium tasks/link.yml read on the host in this run. github_runner_cluster_node_labels: >- {{ [github_runner_cluster_voter_eligible_label ~ '=' ~ ('true' if github_runner_cluster_voter_eligible | bool else 'false'), - github_runner_cluster_voter_pinned_label ~ '=' ~ ('true' if github_runner_cluster_voter_pinned | bool else 'false')] - if github_runner_cluster_role_controller | bool or github_runner_cluster_voter_eligible | bool or github_runner_cluster_voter_pinned | bool else [] }} + github_runner_cluster_voter_pinned_label ~ '=' ~ ('true' if github_runner_cluster_voter_pinned | bool else 'false'), + github_runner_cluster_link_label ~ '=' ~ github_runner_cluster_link] + if github_runner_cluster_publishes_node_labels | bool else [] }} # The images every node's k3s container is built from: the rancher/k3s tag (k3s's version with "+" written as "-") and the tailscale/tailscale tag whose tailscale and tailscaled binaries are copied in. Pinned, never latest: Compose builds a node's image only when it is missing, so a floating tag would put whatever version is newest on the day a node is first built, rebuilt or moved into a running cluster, unannounced. Set both once for the whole cluster group, in inventory or group_vars (the check below reads every host's through hostvars). Before it changes anything on a host the role fails if either is not a pinned tag, if the group's hosts name different k3s minor versions, or if the k3s version would break k3s's upgrade path or Kubernetes' version skew policy from what this node and the cluster run now (see the role README's Versions section). Changing them changes no running node: its image is kept until it is rebuilt (see the role README's Versions section for upgrading). github_runner_cluster_k3s_version: "{{ github_runner_cluster_k3s_tested_version }}" diff --git a/roles/github_runner_cluster/files/node-link.sh b/roles/github_runner_cluster/files/node-link.sh new file mode 100755 index 00000000..8860e2c6 --- /dev/null +++ b/roles/github_runner_cluster/files/node-link.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# The medium of a host's network link, wired, wifi or unknown, read the same way by the cluster role on the host (which publishes it as the github_runner_cluster_link_label Node label, see tasks/link.yml) and by the role controller's probe inside the node (scripts/fleet-role-probe.sh, which sources this file). Bash 3.2 compatible, since the role runs it on macOS, whose own bash is that release. +# +# Run, it prints {"interface": "", "link": ""} for the interface given as its argument, or for the interface the default route leaves by when none is given (empty, with the medium unknown, when there is no default route). Sourced, it defines the functions below and does nothing else. +# +# On Linux the medium is read from the interface's own /sys/class/net entry: wifi when the kernel gives it a wireless phy, wired when it is backed by a device, unknown when it is virtual (a veth in a container, a bridge, a tunnel) or absent, since then the link beneath it cannot be seen from there. On macOS it is read from the hardware port networksetup lists the interface under: wifi for Wi-Fi, wired for a port whose name says Ethernet or LAN (built-in Ethernet, Thunderbolt Ethernet, a USB LAN adapter), unknown for any other (a Thunderbolt port or bridge, an iPhone's USB link) and for an interface it does not list. +# +# Env: +# - NODE_LINK_SYSTEM: the kernel name to read as (default the output of uname -s), so the macOS reading can be tested on Linux. +# - SYS_CLASS_NET, PROC_NET_ROUTE: where Linux's /sys/class/net and /proc/net/route are (the defaults). + +NODE_LINK_SYSTEM="${NODE_LINK_SYSTEM:-$(uname -s)}" +SYS_CLASS_NET="${SYS_CLASS_NET:-/sys/class/net}" +PROC_NET_ROUTE="${PROC_NET_ROUTE:-/proc/net/route}" + +# Linux: the interface the default route leaves by, the one with the lowest metric among the routes that are up and have a gateway; empty when there is none. /proc/net/route is hexadecimal, tab separated, with a header line. +linux_default_route_interface() { + local iface destination _gateway flags _refcnt _use metric mask best='' best_metric='' + while read -r iface destination _gateway flags _refcnt _use metric mask _; do + [ "$destination" = 00000000 ] && [ "$mask" = 00000000 ] || continue + (( (0x$flags & 3) == 3 )) || continue + if [ -z "$best" ] || [ "$metric" -lt "$best_metric" ]; then + best=$iface + best_metric=$metric + fi + done < "$PROC_NET_ROUTE" + echo "$best" +} + +# Linux: the medium of an interface. Reads the interface's own directory and resolves it with cd -P, never a find over /sys, whose symlinks (device, subsystem, phy80211) loop back on themselves. +linux_link_type() { + local iface=$1 dir="${SYS_CLASS_NET}/$1" real + if [ -z "$iface" ] || [ ! -d "$dir" ]; then + echo unknown + return + fi + if [ -e "${dir}/phy80211" ] || [ -d "${dir}/wireless" ] || grep -qx 'DEVTYPE=wlan' "${dir}/uevent" 2> /dev/null; then + echo wifi + return + fi + real="$(cd -P "$dir" && pwd -P)" + case "$real" in + */devices/virtual/*) echo unknown ;; + *) echo wired ;; + esac +} + +# macOS: the interface the default route leaves by, from route's own answer; empty when there is none, for which route fails ("not in table"). +macos_default_route_interface() { + { route -n get default 2> /dev/null || true; } | awk '$1 == "interface:" { print $2; exit }' +} + +# macOS: the medium of an interface, from the name of the hardware port networksetup lists it under. Each port is a "Hardware Port: " line followed by its "Device: " line. +macos_link_type() { + local iface=$1 port + if [ -z "$iface" ]; then + echo unknown + return + fi + port="$(networksetup -listallhardwareports | awk -v iface="$iface" ' + /^Hardware Port: / { port = substr($0, length("Hardware Port: ") + 1) } + /^Device: / && substr($0, length("Device: ") + 1) == iface { print port; exit }')" + case "$port" in + *Wi-Fi* | *AirPort*) echo wifi ;; + *Ethernet* | *LAN*) echo wired ;; + *) echo unknown ;; + esac +} + +default_route_interface() { + if [ "$NODE_LINK_SYSTEM" = Darwin ]; then macos_default_route_interface; else linux_default_route_interface; fi +} + +link_type() { + if [ "$NODE_LINK_SYSTEM" = Darwin ]; then macos_link_type "$1"; else linux_link_type "$1"; fi +} + +# Run rather than sourced: the reading for the interface given, or the default route's. +if [ "${BASH_SOURCE[0]}" = "$0" ]; then + set -euo pipefail + node_link_interface="${1:-$(default_route_interface)}" + printf '{"interface": "%s", "link": "%s"}\n' "$node_link_interface" "$(link_type "$node_link_interface")" +fi diff --git a/roles/github_runner_cluster/tasks/link.yml b/roles/github_runner_cluster/tasks/link.yml new file mode 100644 index 00000000..b39f6b45 --- /dev/null +++ b/roles/github_runner_cluster/tasks/link.yml @@ -0,0 +1,40 @@ +--- +# Reads the medium of this host's link, wired, wifi or unknown, on the host itself, for the github_runner_cluster_link_label Node label (defaults/main.yml's github_runner_cluster_node_labels), with files/node-link.sh, the reader the role controller's probe runs inside the node too. Only while the host publishes node labels (github_runner_cluster_publishes_node_labels): a host that publishes none reads nothing. Read-only, so it runs under --check too. +# +# A Mac's node runs in a Colima VM bridged to one interface of the Mac (network.mode bridged with network.address in the profile's colima.yaml), so its link is that interface's, whichever interface the Mac's own default route leaves by. A VM that is not bridged reaches the network through the Mac's routes, as Docker Desktop's and any other runtime's do, so its link is the one the Mac's default route leaves by, and so is a Linux host's. Publishes github_runner_cluster_link and github_runner_cluster_link_interface. + +- name: Read the Colima profile's network settings + # //colima.yaml, the file colima start reads; the Colima home tasks/runtime.yml worked out is empty for Colima's own default, ~/.colima. + ansible.builtin.slurp: + src: >- + {{ github_runner_cluster_runtime_colima_home or (ansible_facts['env']['HOME'] ~ '/.colima') }}/{{ github_runner_cluster_effective_runtime.colima_profile }}/colima.yaml + register: github_runner_cluster_link_colima_config + check_mode: false + when: + - ansible_facts['system'] == 'Darwin' + - github_runner_cluster_effective_runtime.kind == 'colima' + +- name: Work out which interface the node's traffic leaves the host by + # The interface a bridged Colima VM is bridged to, or empty, which has the reader take the interface the host's default route leaves by. Colima writes every network key into the file when it starts a profile. + vars: + github_runner_cluster_link_colima_network: "{{ (github_runner_cluster_link_colima_config.content | b64decode | from_yaml).network }}" + ansible.builtin.set_fact: + github_runner_cluster_link_bridged_interface: >- + {{ github_runner_cluster_link_colima_network.interface + if (github_runner_cluster_link_colima_config is not skipped + and github_runner_cluster_link_colima_network.mode == 'bridged' and github_runner_cluster_link_colima_network.address | bool) + else '' }} + +- name: Read the medium of that interface + ansible.builtin.script: + cmd: "node-link.sh {{ github_runner_cluster_link_bridged_interface | quote if github_runner_cluster_link_bridged_interface | length > 0 else '' }}" + register: github_runner_cluster_link_reading + changed_when: false + check_mode: false + +- name: Note the medium and the interface it was read from + vars: + github_runner_cluster_link_read: "{{ github_runner_cluster_link_reading.stdout | from_json }}" + ansible.builtin.set_fact: + github_runner_cluster_link: "{{ github_runner_cluster_link_read.link }}" + github_runner_cluster_link_interface: "{{ github_runner_cluster_link_read.interface }}" diff --git a/roles/github_runner_cluster/tasks/main.yml b/roles/github_runner_cluster/tasks/main.yml index d942017c..54e91fd2 100644 --- a/roles/github_runner_cluster/tasks/main.yml +++ b/roles/github_runner_cluster/tasks/main.yml @@ -64,6 +64,10 @@ - name: Check the role controller's settings ansible.builtin.include_tasks: voters.yml +- name: Read the medium of this host's link, for its Node label + ansible.builtin.include_tasks: link.yml + when: github_runner_cluster_publishes_node_labels | bool + - name: Check the memory-pressure regrading settings ansible.builtin.include_tasks: regrade.yml diff --git a/roles/github_runner_cluster/tasks/native/main.yml b/roles/github_runner_cluster/tasks/native/main.yml index 222b56e5..7d18132a 100644 --- a/roles/github_runner_cluster/tasks/native/main.yml +++ b/roles/github_runner_cluster/tasks/native/main.yml @@ -67,6 +67,10 @@ - name: Check the role controller's settings ansible.builtin.include_tasks: voters.yml +- name: Read the medium of this host's link, for its Node label + ansible.builtin.include_tasks: link.yml + when: github_runner_cluster_publishes_node_labels | bool + - name: Check the versions the node is pinned to ansible.builtin.include_tasks: versions.yml diff --git a/tests/cluster_link/link.yml b/tests/cluster_link/link.yml new file mode 100644 index 00000000..7f3230db --- /dev/null +++ b/tests/cluster_link/link.yml @@ -0,0 +1,26 @@ +--- +# Runs the cluster role's tasks/link.yml through the role itself, with the facts and variables the earlier tasks of a real run would have set passed as extra variables (link_system, link_home and, for a Mac, github_runner_cluster_effective_runtime and github_runner_cluster_runtime_colima_home), and writes the medium and the interface it read to link_output, for tests/unit/test_node_link.py. The reader runs on this machine, so the test points it at stand-ins through the environment the run inherits (NODE_LINK_SYSTEM, SYS_CLASS_NET, PROC_NET_ROUTE, and networksetup and route on the PATH). +- name: Read a host's link as the cluster role does + hosts: localhost + connection: local + gather_facts: false + tasks: + - name: Give the host the facts the role's earlier tasks gather + # Cacheable, so that they are host facts and the role reads them through ansible_facts as it reads gathered ones. + ansible.builtin.set_fact: + system: "{{ link_system }}" + env: {HOME: "{{ link_home }}"} + cacheable: true + + - name: Include the role's link reading + ansible.builtin.include_role: + name: exadev.github_runner.github_runner_cluster + tasks_from: link + # So that the role's defaults, github_runner_cluster_node_labels among them, are visible to the task below. + public: true + + - name: Write what it read + ansible.builtin.copy: + content: "{{ {'link': github_runner_cluster_link, 'interface': github_runner_cluster_link_interface, 'labels': github_runner_cluster_node_labels} | to_json }}" + dest: "{{ link_output }}" + mode: "0600" diff --git a/tests/cluster_versions/render_env.yml b/tests/cluster_versions/render_env.yml index b397b114..823af481 100644 --- a/tests/cluster_versions/render_env.yml +++ b/tests/cluster_versions/render_env.yml @@ -13,6 +13,8 @@ github_runner_cluster_mesh_control_server_url: "" github_runner_cluster_effective_server_url: "" github_runner_cluster_effective_node_name: node1 + # What tasks/link.yml would have read on the host. + github_runner_cluster_link: wired tasks: - name: Render env.j2 ansible.builtin.copy: diff --git a/tests/unit/native_stand_ins.py b/tests/unit/native_stand_ins.py index dd3e1144..7441e8a9 100644 --- a/tests/unit/native_stand_ins.py +++ b/tests/unit/native_stand_ins.py @@ -106,6 +106,7 @@ def setUp(self) -> None: # A kernel with memory pressure, as every kernel the role supports has unless it was built or booted without it. self.paths["github_runner_cluster_native_psi_file"].write_text("some avg10=0.00 avg60=0.00 avg300=0.00 total=0\nfull avg10=0.00 avg60=0.00 avg300=0.00 total=0\n") self.make_stand_ins() + self.make_network() self.set_tailscale(None, None) self.facts = dict(FACTS) # What the host holds before the role runs, which written() leaves out. @@ -118,6 +119,16 @@ def written(self) -> list[Path]: """The files on the stand-in host that were not there before the role ran, in path order.""" return sorted(self.files_on_host() - self.host_files) + def make_network(self) -> None: + """A /sys/class/net and /proc/net/route for the role's link reader (files/node-link.sh) to read instead of the machine's own: a default route through a PCI Ethernet interface, which reads wired.""" + net = self.stubs / "net" + device = net / "devices" / "pci0000:00" / "0000:00:1f.6" / "net" / "eno1" + device.mkdir(parents=True) + (net / "class" / "net").mkdir(parents=True) + (net / "class" / "net" / "eno1").symlink_to(device) + (net / "route").write_text("Iface\tDestination\tGateway \tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU\tWindow\tIRTT\neno1\t00000000\t0101A8C0\t0003\t0\t0\t100\t00000000\t0\t0\t0\n") + self.network = {"NODE_LINK_SYSTEM": "Linux", "SYS_CLASS_NET": str(net / "class" / "net"), "PROC_NET_ROUTE": str(net / "route")} + def make_stand_ins(self) -> None: bin_dir = self.stubs / "bin" bin_dir.mkdir() @@ -262,6 +273,7 @@ def run_role(self, *args: str, host_vars: dict[str, Any] | None = None, check: b "ANSIBLE_LOCALHOST_WARNING": "false", "ANSIBLE_INVENTORY_UNPARSED_WARNING": "false", "ANSIBLE_RETRY_FILES_ENABLED": "false", + **self.network, } command = [ANSIBLE_PLAYBOOK, "-i", str(self.inventory(only_the_node, second_server, **(host_vars or {}))), str(RUN), "-e", json.dumps({"host_facts": self.facts, **extra}), *args] if check: diff --git a/tests/unit/test_cluster_voters.py b/tests/unit/test_cluster_voters.py index c4f6319d..d4c2b647 100644 --- a/tests/unit/test_cluster_voters.py +++ b/tests/unit/test_cluster_voters.py @@ -26,6 +26,7 @@ PINNED = "github_runner_cluster_voter_pinned" ELIGIBLE_LABEL = "github-runner.exadev.com/voter-eligible" PINNED_LABEL = "github-runner.exadev.com/voter-pinned" +LINK_LABEL = "github-runner.exadev.com/link" MAX_SERVERS = "github_runner_cluster_max_servers" @@ -48,6 +49,7 @@ def test_nothing_is_on_eligible_or_pinned_by_default_and_the_target_is_three(sel def test_the_label_names_are_the_ones_the_controller_reads(self) -> None: self.assertEqual(DEFAULTS["github_runner_cluster_voter_eligible_label"], ELIGIBLE_LABEL) self.assertEqual(DEFAULTS["github_runner_cluster_voter_pinned_label"], PINNED_LABEL) + self.assertEqual(DEFAULTS["github_runner_cluster_link_label"], LINK_LABEL) def test_the_default_target_is_a_valid_one_for_the_default_maximum(self) -> None: self.assertEqual(DEFAULTS[VOTERS] % 2, 1) @@ -142,24 +144,31 @@ def test_a_host_that_uses_none_of_it_keeps_its_configuration_exactly(self) -> No def test_an_eligible_host_publishes_eligible_true_and_pinned_false_in_its_env_and_its_compose_files(self) -> None: server, agent, files = self.render_env("-e", f"{ELIGIBLE}=true") - expected = f"K3S_NODE_LABELS={ELIGIBLE_LABEL}=true {PINNED_LABEL}=false" + expected = f"K3S_NODE_LABELS={ELIGIBLE_LABEL}=true {PINNED_LABEL}=false {LINK_LABEL}=wired" self.assertIn(expected, server) self.assertIn(expected, agent) self.assertEqual(files, ["docker-compose.yml", "docker-compose.node-labels.yml"]) def test_a_pinned_host_publishes_pinned_true(self) -> None: server, _, _ = self.render_env("-e", f"{PINNED}=true") - self.assertIn(f"K3S_NODE_LABELS={ELIGIBLE_LABEL}=false {PINNED_LABEL}=true", server) + self.assertIn(f"K3S_NODE_LABELS={ELIGIBLE_LABEL}=false {PINNED_LABEL}=true {LINK_LABEL}=wired", server) def test_with_the_controller_on_every_host_publishes_both_as_false_so_that_a_setting_turned_off_is_written_over(self) -> None: server, agent, files = self.render_env("-e", f"{CONTROLLER}=true") for lines in (server, agent): - self.assertIn(f"K3S_NODE_LABELS={ELIGIBLE_LABEL}=false {PINNED_LABEL}=false", lines) + self.assertIn(f"K3S_NODE_LABELS={ELIGIBLE_LABEL}=false {PINNED_LABEL}=false {LINK_LABEL}=wired", lines) self.assertIn("docker-compose.node-labels.yml", files) def test_the_label_names_can_be_changed(self) -> None: server, _, _ = self.render_env("-e", f"{ELIGIBLE}=true", "-e", "github_runner_cluster_voter_eligible_label=example.com/candidate") - self.assertIn(f"K3S_NODE_LABELS=example.com/candidate=true {PINNED_LABEL}=false", server) + self.assertIn(f"K3S_NODE_LABELS=example.com/candidate=true {PINNED_LABEL}=false {LINK_LABEL}=wired", server) + + def test_the_link_label_carries_the_medium_read_on_the_host(self) -> None: + for medium in ("wifi", "unknown"): + with self.subTest(medium): + server, agent, _ = self.render_env("-e", f"{CONTROLLER}=true", "-e", f"github_runner_cluster_link={medium}", "-e", "github_runner_cluster_link_label=example.com/link") + for lines in (server, agent): + self.assertIn(f"K3S_NODE_LABELS={ELIGIBLE_LABEL}=false {PINNED_LABEL}=false example.com/link={medium}", lines) class ComposeOverrideTest(unittest.TestCase): diff --git a/tests/unit/test_native_node.py b/tests/unit/test_native_node.py index 9020a381..fc5d1953 100644 --- a/tests/unit/test_native_node.py +++ b/tests/unit/test_native_node.py @@ -472,15 +472,15 @@ def test_a_host_that_uses_no_voter_setting_gets_no_node_labels(self) -> None: config = yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "agent.yaml").read_text()) self.assertNotIn("node-label", config) - def test_an_eligible_host_gets_its_voter_labels_as_k3s_node_labels(self) -> None: + def test_an_eligible_host_gets_its_voter_labels_and_the_medium_of_its_link_as_k3s_node_labels(self) -> None: self.assertEqual(self.run_role(host_vars={"github_runner_cluster_voter_eligible": True}).returncode, 0) config = yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "agent.yaml").read_text()) - self.assertEqual(config["node-label"], ["github-runner.exadev.com/voter-eligible=true", "github-runner.exadev.com/voter-pinned=false"]) + self.assertEqual(config["node-label"], ["github-runner.exadev.com/voter-eligible=true", "github-runner.exadev.com/voter-pinned=false", "github-runner.exadev.com/link=wired"]) - def test_with_the_role_controller_on_every_host_gets_both_labels_false(self) -> None: + def test_with_the_role_controller_on_every_host_gets_both_voter_labels_false_and_its_link(self) -> None: self.assertEqual(self.run_role(host_vars={"github_runner_cluster_role_controller": True}).returncode, 0) config = yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "agent.yaml").read_text()) - self.assertEqual(config["node-label"], ["github-runner.exadev.com/voter-eligible=false", "github-runner.exadev.com/voter-pinned=false"]) + self.assertEqual(config["node-label"], ["github-runner.exadev.com/voter-eligible=false", "github-runner.exadev.com/voter-pinned=false", "github-runner.exadev.com/link=wired"]) def test_a_host_that_sets_its_node_name_joins_under_it(self) -> None: self.assertEqual(self.run_role(host_vars={"github_runner_cluster_node_name": "laptop-renamed"}).returncode, 0) diff --git a/tests/unit/test_node_link.py b/tests/unit/test_node_link.py new file mode 100644 index 00000000..08739fe1 --- /dev/null +++ b/tests/unit/test_node_link.py @@ -0,0 +1,210 @@ +"""Tests for the cluster role's link reader (roles/github_runner_cluster/files/node-link.sh) run as the role runs it, and for tasks/link.yml, which chooses the interface it reads (tests/cluster_link/link.yml runs it through the role). The Linux reading of sysfs is held to the kernel's tree in test_fleet_role_probe.py's LinkTest, through the probe that sources the same file; here it is the macOS reading, against stand-ins for networksetup and route, and which interface the role asks about.""" + +from __future__ import annotations + +import json +import os +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +READER = ROOT / "roles" / "github_runner_cluster" / "files" / "node-link.sh" +PLAYBOOK = ROOT / "tests" / "cluster_link" / "link.yml" +ANSIBLE_PLAYBOOK = os.environ.get("ANSIBLE_PLAYBOOK", "ansible-playbook") +LINK_LABEL = "github-runner.exadev.com/link" + +# What `networksetup -listallhardwareports` prints on a Mac with built-in Wi-Fi, a USB Ethernet adapter and Thunderbolt ports, in its own layout. +HARDWARE_PORTS = textwrap.dedent( + """\ + + Hardware Port: Ethernet Adapter (en4) + Device: en4 + Ethernet Address: 02:00:00:00:00:04 + + Hardware Port: USB 10/100/1000 LAN + Device: en7 + Ethernet Address: 02:00:00:00:00:07 + + Hardware Port: Thunderbolt Ethernet Slot 0 + Device: en8 + Ethernet Address: 02:00:00:00:00:08 + + Hardware Port: Thunderbolt Bridge + Device: bridge0 + Ethernet Address: 02:00:00:00:00:00 + + Hardware Port: Wi-Fi + Device: en0 + Ethernet Address: 02:00:00:00:00:01 + + Hardware Port: Thunderbolt 1 + Device: en1 + Ethernet Address: 02:00:00:00:00:02 + + VLAN Configurations + =================== + """ +) + + +class Mac: + """A scratch directory with stand-ins for a Mac's networksetup and route on a PATH, route answering that the default route leaves by `default_interface`.""" + + def __init__(self, default_interface: str | None) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.dir = Path(self.tmp.name) + self.bin = self.dir / "bin" + self.bin.mkdir() + (self.dir / "ports").write_text(HARDWARE_PORTS) + self.write("networksetup", f'[ "$*" = -listallhardwareports ] || exit 2\ncat {self.dir / "ports"}\n') + route = "echo 'route: writing to routing socket: not in table' >&2\nexit 1\n" if default_interface is None else ( + f'[ "$*" = "-n get default" ] || exit 2\nprintf " route to: default\\ndestination: default\\n gateway: 192.0.2.1\\n interface: {default_interface}\\n flags: \\n"\n' + ) + self.write("route", route) + + def write(self, name: str, body: str) -> None: + path = self.bin / name + path.write_text("#!/bin/sh\n" + body) + path.chmod(0o755) + + def env(self) -> dict[str, str]: + return {**os.environ, "PATH": f"{self.bin}{os.pathsep}{os.environ['PATH']}", "NODE_LINK_SYSTEM": "Darwin"} + + def cleanup(self) -> None: + self.tmp.cleanup() + + +class MacReaderTest(unittest.TestCase): + def read(self, *args: str, default_interface: str | None = "en0") -> dict[str, str]: + mac = Mac(default_interface) + self.addCleanup(mac.cleanup) + completed = subprocess.run([str(READER), *args], env=mac.env(), capture_output=True, text=True, check=False) + self.assertEqual(completed.returncode, 0, completed.stderr) + return json.loads(completed.stdout) + + def test_the_wi_fi_port_is_wifi(self) -> None: + self.assertEqual(self.read("en0"), {"interface": "en0", "link": "wifi"}) + + def test_a_port_named_for_ethernet_or_a_lan_is_wired(self) -> None: + for interface in ("en4", "en7", "en8"): + with self.subTest(interface): + self.assertEqual(self.read(interface)["link"], "wired") + + def test_a_thunderbolt_port_or_bridge_is_unknown(self) -> None: + for interface in ("en1", "bridge0"): + with self.subTest(interface): + self.assertEqual(self.read(interface)["link"], "unknown") + + def test_an_interface_networksetup_does_not_list_is_unknown(self) -> None: + self.assertEqual(self.read("utun3"), {"interface": "utun3", "link": "unknown"}) + + def test_a_device_name_that_only_begins_another_is_not_it(self) -> None: + # en1 is listed; en10 is not, and must not be read as en1's port. + self.assertEqual(self.read("en10")["link"], "unknown") + + def test_with_no_interface_given_the_default_routes_is_read(self) -> None: + self.assertEqual(self.read(default_interface="en7"), {"interface": "en7", "link": "wired"}) + + def test_with_no_default_route_the_link_is_unknown_and_there_is_no_interface(self) -> None: + self.assertEqual(self.read(default_interface=None), {"interface": "", "link": "unknown"}) + + def test_sourced_it_defines_the_readers_and_prints_nothing(self) -> None: + mac = Mac("en0") + self.addCleanup(mac.cleanup) + completed = subprocess.run(["bash", "-c", f'. {READER} && link_type "$(default_route_interface)"'], env=mac.env(), capture_output=True, text=True, check=False) + self.assertEqual((completed.returncode, completed.stdout), (0, "wifi\n"), completed.stderr) + + +class RoleTest(unittest.TestCase): + """tasks/link.yml through the role: which interface it reads, and the label it gives the node.""" + + def setUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.dir = Path(self.tmp.name) + self.home = self.dir / "home" + self.home.mkdir() + + def colima_profile(self, colima_home: Path, profile: str, *, mode: str, interface: str, address: bool) -> None: + """A Colima profile's colima.yaml as Colima writes it on start, every network key present.""" + directory = colima_home / profile + directory.mkdir(parents=True) + network = {"address": address, "mode": mode, "interface": interface, "preferredRoute": False, "dns": None, "dnsHosts": {}, "hostAddresses": False, "gatewayAddress": "192.168.5.2"} + (directory / "colima.yaml").write_text(yaml.safe_dump({"cpu": 4, "memory": 8, "vmType": "vz", "network": network})) + + def run_role(self, *, system: str, env: dict[str, str], **extra: object) -> dict[str, object]: + output = self.dir / "link.json" + variables = {"link_system": system, "link_home": str(self.home), "link_output": str(output), "github_runner_cluster_role_controller": True, **extra} + completed = subprocess.run( + [ANSIBLE_PLAYBOOK, str(PLAYBOOK), "-e", json.dumps(variables)], + env={**env, "ANSIBLE_COLLECTIONS_PATH": str(ROOT / "playbooks" / "collections"), "ANSIBLE_LOCALHOST_WARNING": "false", "ANSIBLE_INVENTORY_UNPARSED_WARNING": "false", "ANSIBLE_NOCOLOR": "1"}, + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + result: dict[str, object] = json.loads(output.read_text()) + return result + + def mac(self, default_interface: str | None = "en0") -> dict[str, str]: + mac = Mac(default_interface) + self.addCleanup(mac.cleanup) + return mac.env() + + def colima(self, profile: str = "node", colima_home: str = "") -> dict[str, object]: + return {"github_runner_cluster_effective_runtime": {"kind": "colima", "colima_profile": profile}, "github_runner_cluster_runtime_colima_home": colima_home} + + def test_a_bridged_colima_vm_reads_the_interface_it_is_bridged_to_not_the_macs_default_route(self) -> None: + # The Mac's own traffic leaves by its USB adapter, as macOS prefers Ethernet, while the VM stays bridged to the Wi-Fi. + home = self.dir / "colima-node" + self.colima_profile(home, "node", mode="bridged", interface="en0", address=True) + result = self.run_role(system="Darwin", env=self.mac(default_interface="en7"), **self.colima(colima_home=str(home))) + self.assertEqual((result["link"], result["interface"]), ("wifi", "en0")) + self.assertIn(f"{LINK_LABEL}=wifi", result["labels"]) + + def test_a_vm_bridged_to_the_usb_adapter_is_wired(self) -> None: + home = self.dir / "colima-node" + self.colima_profile(home, "node", mode="bridged", interface="en7", address=True) + result = self.run_role(system="Darwin", env=self.mac(default_interface="en0"), **self.colima(colima_home=str(home))) + self.assertEqual((result["link"], result["interface"]), ("wired", "en7")) + + def test_a_profile_in_colimas_default_home_is_read_from_it(self) -> None: + self.colima_profile(self.home / ".colima", "default", mode="bridged", interface="en7", address=True) + result = self.run_role(system="Darwin", env=self.mac(default_interface="en0"), **self.colima(profile="default")) + self.assertEqual(result["interface"], "en7") + + def test_a_vm_that_is_not_bridged_uses_the_macs_routes(self) -> None: + home = self.dir / "colima-node" + for mode, address in (("shared", False), ("shared", True), ("bridged", False)): + with self.subTest(mode=mode, address=address): + if (home / "node").exists(): + (home / "node" / "colima.yaml").unlink() + (home / "node").rmdir() + self.colima_profile(home, "node", mode=mode, interface="en0", address=address) + result = self.run_role(system="Darwin", env=self.mac(default_interface="en7"), **self.colima(colima_home=str(home))) + self.assertEqual((result["link"], result["interface"]), ("wired", "en7")) + + def test_a_mac_runtime_that_is_not_colima_uses_the_macs_routes(self) -> None: + result = self.run_role(system="Darwin", env=self.mac(default_interface="en0"), github_runner_cluster_effective_runtime={"kind": "docker_desktop", "colima_profile": ""}) + self.assertEqual((result["link"], result["interface"]), ("wifi", "en0")) + + def test_a_linux_host_reads_the_interface_its_default_route_leaves_by(self) -> None: + net = self.dir / "net" + device = net / "devices" / "pci0000:00" / "0000:00:14.3" / "net" / "wlp2s0" + (device / "wireless").mkdir(parents=True) + (net / "class" / "net").mkdir(parents=True) + (net / "class" / "net" / "wlp2s0").symlink_to(device) + (net / "route").write_text("Iface\tDestination\tGateway \tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU\tWindow\tIRTT\nwlp2s0\t00000000\t0100000A\t0003\t0\t0\t600\t00000000\t0\t0\t0\n") + env = {**os.environ, "NODE_LINK_SYSTEM": "Linux", "SYS_CLASS_NET": str(net / "class" / "net"), "PROC_NET_ROUTE": str(net / "route")} + result = self.run_role(system="Linux", env=env) + self.assertEqual((result["link"], result["interface"]), ("wifi", "wlp2s0")) + self.assertIn(f"{LINK_LABEL}=wifi", result["labels"]) + + +if __name__ == "__main__": + unittest.main() From bc996230a9a949131666e95d27a0e6567a528869 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 3 Nov 2026 17:53:05 +0000 Subject: [PATCH 2/2] feat(role): report the link medium the cluster role labelled each node with The probe reads its node's github-runner.exadev.com/link label from the nodes it already lists each sample and reports that medium, falling back to its own interface's only when the node has no such label: a k3s container's veth reads unknown, and a VM's virtio NIC reads wired over a Mac's Wi-Fi. Each sample says which it used (link_source), and the controller carries both into fleet-role-status and names the link in its log. The probe now sources the cluster role's link reader from the image rather than keeping its own copy, and the ARC role passes the label's name (github_runner_arc_role_controller_link_label, from the cluster role's) to it and checks it is a label key distinct from the voter labels. --- .github/workflows/mesh-integration.yml | 2 +- README.md | 4 +- fleet-role/Dockerfile | 3 +- roles/github_runner_arc/README.md | 2 +- roles/github_runner_arc/defaults/main.yml | 2 + .../github_runner_arc/meta/argument_specs.yml | 3 + roles/github_runner_arc/tasks/validate.yml | 6 +- .../templates/fleet-role.yaml.j2 | 2 + scripts/fleet-role-probe.sh | 63 +++++++------------ scripts/fleet-role.jq | 2 + scripts/fleet-role.sh | 2 +- tests/fleet_role/image-smoke.sh | 7 ++- tests/mesh/run.sh | 12 ++-- tests/unit/fleet_role_support.py | 9 ++- tests/unit/test_fleet_role.py | 9 ++- tests/unit/test_fleet_role_arc.py | 20 +++++- tests/unit/test_fleet_role_probe.py | 29 ++++++++- 17 files changed, 116 insertions(+), 61 deletions(-) diff --git a/.github/workflows/mesh-integration.yml b/.github/workflows/mesh-integration.yml index 4821fbee..d13962bc 100644 --- a/.github/workflows/mesh-integration.yml +++ b/.github/workflows/mesh-integration.yml @@ -1,6 +1,6 @@ name: Mesh integration -# Brings up three k3s nodes in Docker and joins them through Headscale with the cluster role (tests/mesh/run.sh), once per scenario: headscale_hosted, headscale_in_cluster, and the headscale_existing policy check, plus the role's refusal of a Docker Compose release that recreates the containers it has just built. headscale_in_cluster also runs with github_runner_cluster_docker_context pinned while the current Docker context points at nothing, which covers the cluster role's own docker calls and the recovery playbook's, and context_checks checks the role refuses a pin it cannot honour. regrade turns memory-pressure regrading on and steps a server down to etcd-only under (simulated) pressure and back up, checking that it keeps its etcd seat and vote and that no other server steps meanwhile. native_server joins a native host (a systemd container) as an agent beside three Docker servers, then promotes it to a server in place: it must join etcd as a learner, be promoted to a voter, and keep the cluster writable throughout. role_controller adds a standby agent, runs the role controller's probe and controller from the image built from this checkout, makes a voter slow with tc netem and pauses another, and checks the controller reports the voter unfit with the swap named, reports that it would not act with a voter paused, and changes nothing. Only on changes that can affect how nodes join the mesh. +# Brings up three k3s nodes in Docker and joins them through Headscale with the cluster role (tests/mesh/run.sh), once per scenario: headscale_hosted, headscale_in_cluster, and the headscale_existing policy check, plus the role's refusal of a Docker Compose release that recreates the containers it has just built. headscale_in_cluster also runs with github_runner_cluster_docker_context pinned while the current Docker context points at nothing, which covers the cluster role's own docker calls and the recovery playbook's, and context_checks checks the role refuses a pin it cannot honour. regrade turns memory-pressure regrading on and steps a server down to etcd-only under (simulated) pressure and back up, checking that it keeps its etcd seat and vote and that no other server steps meanwhile. native_server joins a native host (a systemd container) as an agent beside three Docker servers, then promotes it to a server in place: it must join etcd as a learner, be promoted to a voter, and keep the cluster writable throughout. role_controller adds a standby agent, runs the role controller's probe and controller from the image built from this checkout, makes a voter slow with tc netem and pauses another, and checks every probe reports the link medium the cluster role labelled its node with, the controller reports the voter unfit with the swap named, reports that it would not act with a voter paused, and changes nothing. Only on changes that can affect how nodes join the mesh. # # Runs on GitHub-hosted ubuntu-latest, whose Docker daemon runs the privileged containers the test needs (k3s, and Tailscale inside it). The work directory sits under runner.temp, which the job always has write access to. diff --git a/README.md b/README.md index d84f6970..b8671bfa 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,7 @@ Its image, `github_runner_arc_node_recovery_image`, is pulled without a pull Sec `fleet-role-probe` and `fleet-role` (an optional DaemonSet and Deployment in the platform namespace, see `roles/github_runner_arc/templates/fleet-role.yaml.j2`, `scripts/fleet-role-probe.sh` and `scripts/fleet-role.sh`; `github_runner_cluster_role_controller`, off by default, which `github_runner_arc_role_controller_enabled` takes as its default) measure which hosts are fit to hold an etcd vote and report the swap of an unfit voter for a fit standby that the controller would make. They only report: the pods' ClusterRoles can `get` and `list` nodes and nothing else, and the only objects they write are their own two ConfigMaps. The controller refuses to start with any setting other than `FLEET_ROLE_DRY_RUN=true`, and runs unchanged when no host is marked eligible, when it reports the voters' fitness alone. -The probe runs on every Linux node that is an etcd voter (k3s's `node-role.kubernetes.io/etcd` label) or is marked eligible and has a kubelet (a server regraded to etcd-only has none, and its side is read from its peers), on the node's own network, through a node affinity on those two labels, so a node that is neither (a Unraid host, an image-flashed agent) has no pod and nothing is asked of its kernel. A pod stopped on a node that has lost both labels, or is gone, removes the node's entry as it ends, so the ConfigMap, which holds at most a mebibyte, grows with the hosts that decide the control plane and not with the fleet. Every `github_runner_arc_role_controller_sample_seconds` (15) it appends one sample to its node's key of the `fleet-role-probe` ConfigMap, which keeps the last window of them so that a replaced pod loses nothing. A sample holds the TCP connect time to every voter's etcd peer port (a connect that does not finish within the election timeout is recorded as the election timeout, so loss shows as tail latency); on a voter, what its etcd metrics (the `127.0.0.1:2381` listener) gained since the last sample: the WAL fsync and backend commit histograms, the round trip to each peer, leader changes, and whether it has a leader, is a learner and knows as many members as there are voters (a learner or a member that has not started shows as an extra known member); on a node that is not a voter, an fdatasync reading from fio with the write size etcd's own disk check uses, once per window; the memory pressure (`/proc/pressure/memory` of the pod's own procfs, which serves the whole node's, so nothing is mounted) and `MemAvailable` the regrade core reads, with the core's own functions, where a node on a kernel without pressure stall information gives `MemAvailable` and `MemTotal` and no pressure figure, which the controller counts as not over the memory limit; and whether the interface the default route leaves by is `wired`, `wifi` or `unknown` (an interface that is virtual, such as the k3s container's veth, hides the link beneath it). The link is read from the interface's own `/sys/class/net` entry and never by walking `/sys`, whose symlinks loop. `scripts/fleet-role-probe.sh check link|connect
|memory|etcd|fsync` takes one of these readings alone, with no cluster access, to compare with a reading taken by hand on the host. +The probe runs on every Linux node that is an etcd voter (k3s's `node-role.kubernetes.io/etcd` label) or is marked eligible and has a kubelet (a server regraded to etcd-only has none, and its side is read from its peers), on the node's own network, through a node affinity on those two labels, so a node that is neither (a Unraid host, an image-flashed agent) has no pod and nothing is asked of its kernel. A pod stopped on a node that has lost both labels, or is gone, removes the node's entry as it ends, so the ConfigMap, which holds at most a mebibyte, grows with the hosts that decide the control plane and not with the fleet. Every `github_runner_arc_role_controller_sample_seconds` (15) it appends one sample to its node's key of the `fleet-role-probe` ConfigMap, which keeps the last window of them so that a replaced pod loses nothing. A sample holds the TCP connect time to every voter's etcd peer port (a connect that does not finish within the election timeout is recorded as the election timeout, so loss shows as tail latency); on a voter, what its etcd metrics (the `127.0.0.1:2381` listener) gained since the last sample: the WAL fsync and backend commit histograms, the round trip to each peer, leader changes, and whether it has a leader, is a learner and knows as many members as there are voters (a learner or a member that has not started shows as an extra known member); on a node that is not a voter, an fdatasync reading from fio with the write size etcd's own disk check uses, once per window; the memory pressure (`/proc/pressure/memory` of the pod's own procfs, which serves the whole node's, so nothing is mounted) and `MemAvailable` the regrade core reads, with the core's own functions, where a node on a kernel without pressure stall information gives `MemAvailable` and `MemTotal` and no pressure figure, which the controller counts as not over the memory limit; and the medium of the node's link, `wired`, `wifi` or `unknown`: the value of the node's `github-runner.exadev.com/link` label (`github_runner_arc_role_controller_link_label`), which the cluster role reads on the host itself (see the cluster role's README) since a node in a container or a VM sees only its own virtual interface, and otherwise the medium of the interface the pod's default route leaves by (an interface that is virtual, such as the k3s container's veth, hides the link beneath it), each sample saying which (`link_source`, `label` or `interface`). The pod's own reading is taken from the interface's `/sys/class/net` entry and never by walking `/sys`, whose symlinks loop. `scripts/fleet-role-probe.sh check link|connect
|memory|etcd|fsync` takes one of these readings alone (the link as the pod's own interface gives it), with no cluster access, to compare with a reading taken by hand on the host. The controller reads the Nodes and the probes' samples every sample period and publishes `fleet-role-status`, which the heartbeat republishes in its gist beside the autoscaler's status. For every voter and every eligible host it gives a verdict, `fit`, `unfit` or `unknown`, with each reason and the numbers behind it. Every limit derives from etcd's guidance and the cluster's own timing, not chosen numbers: @@ -206,7 +206,7 @@ The controller reads the Nodes and the probes' samples every sample period and p - Memory is never over the regrade limit (`github_runner_cluster_regrade_pressure_threshold` and `github_runner_cluster_regrade_available_floor_percent`) in any sample of the window, and `MemAvailable` is at least etcd's storage quota (what the voters report, 2 GiB by default). - Stability: no `NotReady` spell in the window and no k3s restart, seen on a voter as etcd's counters going backwards. On a node that is not a voter the only sign of a restart is its probe starting again, so a new probe pod counts as one. - A window is 100 samples, the fewest that give the 99th percentile a sample of its own, so one outlier never decides; a host with fewer is `unknown`, as is one whose probe has not reported for three sample periods. Readings from before the last change of the voter set are not counted. -- Hosts that pass are ranked wired before Wi-Fi before unknown, then by round trip, then by disk. The link ranks hosts and is never a gate. +- Hosts that pass are ranked wired before Wi-Fi before unknown, then by round trip, then by disk. The link ranks hosts and is never a gate; the status gives each host's link and where it was read (`link` and `link_source`), and the log names it beside the verdict. A slow link makes both of its ends unfit, since a voter is unfit when its round trip to any other voter is, so the voter to replace is the one at fault: the unfit voter on the most slow links (a link is slow when either end measures it above the limit), the worst-ranked among those on as many. A swap is reported only when that voter is eligible and not pinned (when it is not, replacing a voter that shares a slow link with it would fix nothing, so nothing is swapped) and a fit eligible standby ranks above it. A candidate is judged against the voters that would remain, since a slow link to the very voter it would replace says nothing against it. It is held back, with the reason in the status and the log, while the voter count is not `github_runner_arc_role_controller_voters`, any voter is below `full`, is not Ready, has no leader, is a learner or knows a member that is not a voter, is not connected to every other voter, fewer hosts are eligible than the target, or the cooldown after a change of the voter set has not passed (k3s's five minutes to remove a stalled learner plus one window). The target must be odd, at least 3, and at most `github_runner_cluster_max_servers` minus one, which leaves a swap one place of headroom; the role refuses anything else. A host is eligible, and a voter pinned, by a Node label with the value `true` (`github_runner_arc_role_controller_eligible_label` and `_pinned_label`), which the cluster role publishes from `github_runner_cluster_voter_eligible` and `github_runner_cluster_voter_pinned` per host (and the target from `github_runner_cluster_voters`, refused unless odd, at least 3 and below `github_runner_cluster_max_servers`; see the cluster role's README); a voter not marked eligible is never swapped out, since nothing has prepared it to come back. diff --git a/fleet-role/Dockerfile b/fleet-role/Dockerfile index 02b0e8cc..c4624945 100644 --- a/fleet-role/Dockerfile +++ b/fleet-role/Dockerfile @@ -1,4 +1,4 @@ -# Image for the role controller (see roles/github_runner_arc/templates/fleet-role.yaml.j2): the `fleet-role` Deployment runs scripts/fleet-role.sh through loop.sh, and the `fleet-role-probe` DaemonSet runs scripts/fleet-role-probe.sh from the same image. bash, curl, jq, kubectl and fio, all from Alpine's apk (fio is the disk check etcd's own documentation uses), so it is small and builds for either architecture. Build context is the repository root so it can COPY the scripts and the regrade core, whose own functions the probe reads memory with. The Deployment and the DaemonSet pull it without a pull Secret, so the published package must be public. +# Image for the role controller (see roles/github_runner_arc/templates/fleet-role.yaml.j2): the `fleet-role` Deployment runs scripts/fleet-role.sh through loop.sh, and the `fleet-role-probe` DaemonSet runs scripts/fleet-role-probe.sh from the same image. bash, curl, jq, kubectl and fio, all from Alpine's apk (fio is the disk check etcd's own documentation uses), so it is small and builds for either architecture. Build context is the repository root so it can COPY the scripts, the regrade core, whose own functions the probe reads memory with, and the cluster role's link reader, whose own functions it reads the link with. The Deployment and the DaemonSet pull it without a pull Secret, so the published package must be public. FROM alpine:3.20 RUN apk add --no-cache bash curl jq kubectl fio @@ -8,6 +8,7 @@ COPY scripts/fleet-role.sh scripts/fleet-role-probe.sh /app/scripts/ # The jq programs the scripts run from their own directory, and the module they share. COPY scripts/fleet-role.jq scripts/fleet-role-stats.jq /app/scripts/ COPY roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh /app/fleet-regrade-core.sh +COPY roles/github_runner_cluster/files/node-link.sh /app/node-link.sh RUN chmod +x /app/loop.sh /app/scripts/fleet-role.sh /app/scripts/fleet-role-probe.sh # A non-root user by number, so the Deployment's runAsNonRoot check can verify it without reading /etc/passwd. The DaemonSet runs the same image as root, for the directory it measures the disk in. diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 35abcafc..379ee194 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -56,7 +56,7 @@ Helm 4 and server-side apply. The role reads `helm version --short` and, on Helm - `github_runner_arc_heartbeat_gist_id`: install the fleet-health platform from this host, refreshing this gist. `github_runner_arc_heartbeat_bootstrap_gist`, `github_runner_arc_heartbeat_gist_description` and `github_runner_arc_heartbeat_gist_consumer` control the bootstrap described above. - `github_runner_arc_autoscaler_usable_budget_gi`, `github_runner_arc_autoscaler_floor`: required when a profile sets `autoscale: true` and the platform is installed. The floor must equal the autoscaled profile's `maxRunners`, which every Helm upgrade reverts to. The autoscaler works its ceiling out from the nodes on every poll (see the project README's Autoscaler section); `github_runner_arc_autoscaler_max_ceiling`, optional, is an upper bound on it and must be no lower than the floor. The other `github_runner_arc_autoscaler_*` and `github_runner_arc_heartbeat_*` settings have defaults; see `defaults/main.yml`. - `github_runner_arc_node_recovery_enabled` (default `true`), `github_runner_arc_node_recovery_after_seconds` (default 120), `github_runner_arc_node_recovery_poll_seconds` (default 15), `github_runner_arc_node_recovery_prune_after_seconds` (default empty, which leaves pruning off), `github_runner_arc_node_recovery_prune_label` (default `github-runner.exadev.io/removable`), `github_runner_arc_node_recovery_image`: the node recovery watcher (see [Node recovery](#node-recovery)). -- `github_runner_arc_role_controller_enabled` (default the cluster role's `github_runner_cluster_role_controller`, itself `false`), `github_runner_arc_role_controller_image`, `github_runner_arc_role_controller_sample_seconds` (default 15), `github_runner_arc_role_controller_voters` (default 3), `github_runner_arc_role_controller_max_servers`, `github_runner_arc_role_controller_election_timeout_ms`, `github_runner_arc_role_controller_pressure_threshold`, `github_runner_arc_role_controller_available_floor_percent`, `github_runner_arc_role_controller_eligible_label`, `github_runner_arc_role_controller_pinned_label`, `github_runner_arc_role_controller_level_label`, `github_runner_arc_role_controller_probe_configmap`, `github_runner_arc_role_controller_status_configmap`, `github_runner_arc_role_controller_data_dir`: the role controller, which reports which hosts are fit to hold an etcd vote (see [Role controller](#role-controller)). The five that mirror the cluster role (`max_servers`, `election_timeout_ms`, `pressure_threshold`, `available_floor_percent`, `level_label`) take the cluster role's own variable when it is defined. +- `github_runner_arc_role_controller_enabled` (default the cluster role's `github_runner_cluster_role_controller`, itself `false`), `github_runner_arc_role_controller_image`, `github_runner_arc_role_controller_sample_seconds` (default 15), `github_runner_arc_role_controller_voters` (default 3), `github_runner_arc_role_controller_max_servers`, `github_runner_arc_role_controller_election_timeout_ms`, `github_runner_arc_role_controller_pressure_threshold`, `github_runner_arc_role_controller_available_floor_percent`, `github_runner_arc_role_controller_eligible_label`, `github_runner_arc_role_controller_pinned_label`, `github_runner_arc_role_controller_level_label`, `github_runner_arc_role_controller_link_label`, `github_runner_arc_role_controller_probe_configmap`, `github_runner_arc_role_controller_status_configmap`, `github_runner_arc_role_controller_data_dir`: the role controller, which reports which hosts are fit to hold an etcd vote (see [Role controller](#role-controller)). Those that mirror the cluster role (`voters`, `max_servers`, `election_timeout_ms`, `pressure_threshold`, `available_floor_percent`, `eligible_label`, `pinned_label`, `level_label`, `link_label`) take the cluster role's own variable when it is defined. - `github_runner_arc_runner_reaper_enabled` (default `false`), `github_runner_arc_runner_reaper_max_job_seconds` (default 432000, five days), `github_runner_arc_runner_reaper_completed_grace_seconds` (default 300), `github_runner_arc_runner_reaper_schedule` (default every 5 minutes), `github_runner_arc_runner_reaper_dry_run` (default `false`), `github_runner_arc_runner_reaper_image` (default the renewal's image): the runner reaper (see [Runner reaper](#runner-reaper)). - `github_runner_arc_runner_active_deadline_seconds` (default unset): `activeDeadlineSeconds` on every runner pod (see [Runner reaper](#runner-reaper)). - `github_runner_arc_app_setup_*`, `github_runner_arc_app_manifest_code`, `github_runner_arc_app_private_key_path`: inputs to `playbooks/github_app_setup.yml` (see below), including `github_runner_arc_app_setup_write_secret`, `github_runner_arc_app_setup_secret_namespaces`, `github_runner_arc_app_setup_secret_name`, `github_runner_arc_app_setup_keep_private_key` and `github_runner_arc_app_setup_command`. diff --git a/roles/github_runner_arc/defaults/main.yml b/roles/github_runner_arc/defaults/main.yml index e5ac7c3d..ce0c9fea 100644 --- a/roles/github_runner_arc/defaults/main.yml +++ b/roles/github_runner_arc/defaults/main.yml @@ -209,6 +209,8 @@ github_runner_arc_role_controller_available_floor_percent: "{{ github_runner_clu github_runner_arc_role_controller_eligible_label: "{{ github_runner_cluster_voter_eligible_label | default('github-runner.exadev.com/voter-eligible') }}" github_runner_arc_role_controller_pinned_label: "{{ github_runner_cluster_voter_pinned_label | default('github-runner.exadev.com/voter-pinned') }}" github_runner_arc_role_controller_level_label: "{{ github_runner_cluster_regrade_level_label | default('github-runner.exadev.com/regrade-level') }}" +# The Node label the cluster role publishes the medium of each host's link under (wired, wifi or unknown), which the probe reports for its node in place of what it can see itself: the cluster role's github_runner_cluster_link_label when that is defined. +github_runner_arc_role_controller_link_label: "{{ github_runner_cluster_link_label | default('github-runner.exadev.com/link') }}" # The ConfigMaps in the platform namespace that the probes publish to (a key per node) and that the controller publishes its status to, which the heartbeat republishes beside the autoscaler's. github_runner_arc_role_controller_probe_configmap: fleet-role-probe github_runner_arc_role_controller_status_configmap: fleet-role-status diff --git a/roles/github_runner_arc/meta/argument_specs.yml b/roles/github_runner_arc/meta/argument_specs.yml index 2edb9fb7..78ca8566 100644 --- a/roles/github_runner_arc/meta/argument_specs.yml +++ b/roles/github_runner_arc/meta/argument_specs.yml @@ -411,6 +411,9 @@ argument_specs: github_runner_arc_role_controller_level_label: type: str description: The label regrading puts on a server below full, from github_runner_cluster_regrade_level_label when defined, else github-runner.exadev.com/regrade-level. + github_runner_arc_role_controller_link_label: + type: str + description: The Node label the cluster role publishes the medium of each host's link under, wired, wifi or unknown, which the probe reports for its node, from github_runner_cluster_link_label when defined, else github-runner.exadev.com/link. github_runner_arc_role_controller_probe_configmap: type: str default: fleet-role-probe diff --git a/roles/github_runner_arc/tasks/validate.yml b/roles/github_runner_arc/tasks/validate.yml index 4d346fb4..303e2a94 100644 --- a/roles/github_runner_arc/tasks/validate.yml +++ b/roles/github_runner_arc/tasks/validate.yml @@ -187,13 +187,13 @@ or github_runner_arc_role_controller_voters | int < 3 or github_runner_arc_role_controller_voters | int > (github_runner_arc_role_controller_max_servers | int) - 1) }} - - msg: github_runner_arc_role_controller_eligible_label and github_runner_arc_role_controller_pinned_label must be Kubernetes label keys, such as example.com/voter-eligible, and differ. + - msg: github_runner_arc_role_controller_eligible_label must be a Kubernetes label key, such as example.com/voter-eligible, as must github_runner_arc_role_controller_pinned_label and github_runner_arc_role_controller_link_label, and the three must differ. failed: >- {{ github_runner_arc_role_controller_enabled | bool - and ([github_runner_arc_role_controller_eligible_label, github_runner_arc_role_controller_pinned_label] + and ([github_runner_arc_role_controller_eligible_label, github_runner_arc_role_controller_pinned_label, github_runner_arc_role_controller_link_label] | map('string') | reject('match', '^([a-z0-9]([-a-z0-9.]{0,251}[a-z0-9])?/)?[A-Za-z0-9]([-A-Za-z0-9_.]{0,61}[A-Za-z0-9])?$') | list | length > 0 - or github_runner_arc_role_controller_eligible_label == github_runner_arc_role_controller_pinned_label) + or [github_runner_arc_role_controller_eligible_label, github_runner_arc_role_controller_pinned_label, github_runner_arc_role_controller_link_label] | unique | length < 3) }} - msg: github_runner_arc_runner_reaper_max_job_seconds must be a whole number of seconds, at least 1, and github_runner_arc_runner_reaper_completed_grace_seconds a whole number of seconds. failed: >- diff --git a/roles/github_runner_arc/templates/fleet-role.yaml.j2 b/roles/github_runner_arc/templates/fleet-role.yaml.j2 index 1fed9085..53823d3a 100644 --- a/roles/github_runner_arc/templates/fleet-role.yaml.j2 +++ b/roles/github_runner_arc/templates/fleet-role.yaml.j2 @@ -206,6 +206,8 @@ spec: value: {{ github_runner_arc_role_controller_election_timeout_ms | string | to_json }} - name: ELIGIBLE_LABEL value: {{ github_runner_arc_role_controller_eligible_label | to_json }} + - name: LINK_LABEL + value: {{ github_runner_arc_role_controller_link_label | to_json }} - name: VOTER_LABEL value: {{ (voter_label ~ '=true') | to_json }} - name: PROBE_CONFIGMAP diff --git a/scripts/fleet-role-probe.sh b/scripts/fleet-role-probe.sh index f8b3f38b..f1ffc96f 100755 --- a/scripts/fleet-role-probe.sh +++ b/scripts/fleet-role-probe.sh @@ -3,7 +3,7 @@ # # Only a node the controller judges is measured: a voter, or a host the owner marks eligible (ELIGIBLE_LABEL, with the value true). The DaemonSet schedules a pod on no other node, and a pod that finds its node is neither (a label removed while it runs) records nothing and removes the entry it has from when it was one, as does a pod that is stopped on a node that is neither any more or is gone, so the ConfigMap, which holds at most a mebibyte, grows with the hosts that decide the cluster's control plane and not with the fleet. # -# A sample holds, per node: the TCP connect time to every voter's etcd peer port (a connect that does not complete within the election timeout is recorded as the election timeout, so loss shows as tail latency); on a voter, what etcd's own metrics gained since the last sample (the WAL fsync and backend commit histograms, the round trip to each peer, leader changes) and whether it has a leader, is a learner and is connected to its peers; on a node that is not a voter, an fdatasync probe in the k3s data directory once per window, with the write size etcd's own fio check uses; the memory pressure and MemAvailable the regrade core reads; and the link the default route leaves by. +# A sample holds, per node: the TCP connect time to every voter's etcd peer port (a connect that does not complete within the election timeout is recorded as the election timeout, so loss shows as tail latency); on a voter, what etcd's own metrics gained since the last sample (the WAL fsync and backend commit histograms, the round trip to each peer, leader changes) and whether it has a leader, is a learner and is connected to its peers; on a node that is not a voter, an fdatasync probe in the k3s data directory once per window, with the write size etcd's own fio check uses; the memory pressure and MemAvailable the regrade core reads; and the medium of the node's link: the value of its LINK_LABEL Node label when it has one, which the cluster role reads on the host itself (the medium of the interface a Mac's Colima VM is bridged to, or of the interface a Linux host's default route leaves by), and otherwise the medium of the interface this pod's default route leaves by. The label decides when the node has it because a node in a container or a VM sees only its own virtual interface: a k3s container's veth, which reads unknown, or a VM's virtio NIC, which reads wired whatever the Mac beneath it uses. # # `fleet-role-probe.sh check link|connect
|memory|etcd|fsync` takes one of those readings alone, prints it and exits, with no cluster access (connect needs ELECTION_TIMEOUT_MS; the others need nothing but the settings below), so that each can be compared with a reading taken by hand on the host it runs on. # @@ -12,6 +12,7 @@ # - NAMESPACE: the namespace of the ConfigMap. Required. # - ELECTION_TIMEOUT_MS: the cluster's etcd election timeout, which is how long a peer connect may take before it counts as lost. Required. # - ELIGIBLE_LABEL: the Node label, with the value true, that marks a host the controller may promote or demote. Required. +# - LINK_LABEL: the Node label the cluster role publishes the host's link medium in, wired, wifi or unknown. Required. # - PROBE_CONFIGMAP: the shared ConfigMap, created by the role, that every probe patches its own key of (default fleet-role-probe). # - REPORT_SECONDS: seconds between samples (default 15, the node recovery watcher's poll). # - REPORT_ONCE: when "true", take one sample and exit, for tests. @@ -19,7 +20,7 @@ # - ETCD_METRICS_URL: where a voter's etcd serves its metrics (default http://127.0.0.1:2381/metrics, the listener k3s keeps). # - ETCD_PEER_PORT: etcd's peer port (default 2380). # - FSYNC_DIR: a directory on the filesystem k3s keeps its data on, which the fdatasync probe writes in (default /var/lib/rancher/k3s/fleet-role-probe). -# - SYS_CLASS_NET, PROC_NET_ROUTE: where the host's /sys/class/net and /proc/net/route are (the defaults). +# - NODE_LINK: the link reader the cluster role runs on the host too (roles/github_runner_cluster/files/node-link.sh), whose default_route_interface and link_type read this pod's own (default /app/node-link.sh). SYS_CLASS_NET and PROC_NET_ROUTE, which it reads, are where the host's /sys/class/net and /proc/net/route are (the defaults). # - FLEET_REGRADE_CORE: the regrade core (roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh), whose own psi_avg300 and meminfo_kb read the memory signals (default /app/fleet-regrade-core.sh). PSI_FILE and MEMINFO_FILE, which it reads, default here to /proc/pressure/memory and /proc/meminfo, which a pod's own procfs serves for the whole node as it does /proc/meminfo, so nothing is mounted for them. A kernel without pressure stall information has no /proc/pressure, and the sample then has no pressure reading and the memory figures from /proc/meminfo alone, which the controller counts as not over the limit. set -euo pipefail @@ -30,8 +31,7 @@ VOTER_LABEL="${VOTER_LABEL:-node-role.kubernetes.io/etcd=true}" ETCD_METRICS_URL="${ETCD_METRICS_URL:-http://127.0.0.1:2381/metrics}" ETCD_PEER_PORT="${ETCD_PEER_PORT:-2380}" FSYNC_DIR="${FSYNC_DIR:-/var/lib/rancher/k3s/fleet-role-probe}" -SYS_CLASS_NET="${SYS_CLASS_NET:-/sys/class/net}" -PROC_NET_ROUTE="${PROC_NET_ROUTE:-/proc/net/route}" +NODE_LINK="${NODE_LINK:-/app/node-link.sh}" FLEET_REGRADE_CORE="${FLEET_REGRADE_CORE:-/app/fleet-regrade-core.sh}" PSI_FILE="${PSI_FILE:-/proc/pressure/memory}" MEMINFO_FILE="${MEMINFO_FILE:-/proc/meminfo}" @@ -45,6 +45,11 @@ readonly etcd_fio_write_bytes=2300 # shellcheck source=/dev/null . "$FLEET_REGRADE_CORE" +# A pod is a Linux process whatever the host beneath its node runs, so the reader reads Linux's files. +# shellcheck disable=SC2034 # read by the sourced reader +NODE_LINK_SYSTEM=Linux +# shellcheck source=/dev/null +. "$NODE_LINK" log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" >&2; } @@ -52,38 +57,6 @@ log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" >&2; } read -r tail_percentile window_samples < <(jq -nr -L "$MODULES" 'include "fleet-role-stats"; "\(tail_percentile) \(window_samples)"') window_seconds=$((REPORT_SECONDS * window_samples)) -# The interface the default route leaves by, the one with the lowest metric among the routes that are up and have a gateway; empty when there is none. /proc/net/route is hexadecimal, tab separated, with a header line. -default_route_interface() { - local iface destination _gateway flags _refcnt _use metric mask best='' best_metric='' - while read -r iface destination _gateway flags _refcnt _use metric mask _; do - [ "$destination" = 00000000 ] && [ "$mask" = 00000000 ] || continue - (( (0x$flags & 3) == 3 )) || continue - if [ -z "$best" ] || [ "$metric" -lt "$best_metric" ]; then - best=$iface - best_metric=$metric - fi - done < "$PROC_NET_ROUTE" - echo "$best" -} - -# wired, wifi or unknown for an interface: wifi when the kernel gives it a wireless phy, wired when it is backed by a device, unknown when it is virtual (a veth in a container, a bridge, a tunnel) or absent, since then the link beneath it cannot be seen from here. Reads the interface's own directory and resolves it with cd -P, never a find over /sys, whose symlinks (device, subsystem, phy80211) loop back on themselves. -link_type() { - local iface=$1 dir="${SYS_CLASS_NET}/$1" real - if [ -z "$iface" ] || [ ! -d "$dir" ]; then - echo unknown - return - fi - if [ -e "${dir}/phy80211" ] || [ -d "${dir}/wireless" ] || grep -qx 'DEVTYPE=wlan' "${dir}/uevent" 2> /dev/null; then - echo wifi - return - fi - real="$(cd -P "$dir" && pwd -P)" - case "$real" in - */devices/virtual/*) echo unknown ;; - *) echo wired ;; - esac -} - # The time curl took to connect to a peer's etcd peer port, in milliseconds. The request itself is not the point (the port speaks TLS, and curl's HTTP is refused), only the connection: curl reports a connect time of zero when it never completed, and then the reading is the election timeout. connect_ms() { local host=$1 seconds @@ -116,6 +89,11 @@ is_eligible_in() { jq --arg node "$NODE_NAME" --arg label "$ELIGIBLE_LABEL" 'any(.items[]; .metadata.name == $node and (.metadata.labels // {})[$label] == "true")' <<< "$1" } +# The value of this node's LINK_LABEL in the nodes JSON, empty when the node has no such label. +labelled_link_in() { + jq -r --arg node "$NODE_NAME" --arg label "$LINK_LABEL" '[.items[] | select(.metadata.name == $node) | (.metadata.labels // {})[$label] | strings] | first // ""' <<< "$1" +} + # Removes this node's entry from the ConfigMap. A key set to null in a merge patch is removed. remove_entry() { local patch @@ -138,7 +116,7 @@ retire() { # Takes one sample and publishes it. $1 is "true" on the first pass of this process, which for a node that is not a voter is the only sign this probe has of k3s having restarted (a voter sees it in etcd's counters going backwards). report() { local first=$1 now entry voters is_voter eligible tcp='{}' peer_name peer_ip ms etcd='null' restarted=false cumulative='null' etcd_id='null' - local iface fsync='null' fsync_at patch metrics reading change psi='' + local iface link link_source fsync='null' fsync_at patch metrics reading change psi='' now="$(date +%s)" # This node's entry is read from the ConfigMap once, when the process starts, and kept from then on: the probe is the only writer of its key, and two API calls a sample (the nodes, then the patch) are all a probe on a poor path can be asked to make in a period. if [ "$entry_loaded" != true ]; then @@ -182,6 +160,12 @@ report() { fi iface="$(default_route_interface)" + link="$(labelled_link_in "$nodes_json")" + link_source=label + if [ -z "$link" ]; then + link="$(link_type "$iface")" + link_source=interface + fi fsync_at="$(jq -c '.fsync_at' <<< "$entry")" if [ "$is_voter" != true ] && { [ "$fsync_at" = null ] || [ $((now - fsync_at)) -ge "$window_seconds" ]; }; then fsync="$(fsync_ms)" @@ -196,12 +180,12 @@ report() { # The entry goes in on standard input, never as an argument: it holds a window of samples, and one argument is limited to a fraction of the argument space (128 KiB on Linux). new_entry="$(jq -c -L "$MODULES" --argjson now "$now" --argjson tcp "$tcp" --argjson etcd "$etcd" \ --argjson restarted "$restarted" --argjson cumulative "$cumulative" --argjson etcd_id "$etcd_id" --argjson fsync "$fsync" --argjson fsync_at "$fsync_at" \ - --arg psi "$psi" --arg avail "$(meminfo_kb MemAvailable)" --arg total "$(meminfo_kb MemTotal)" --arg iface "$iface" --arg link "$(link_type "$iface")" ' + --arg psi "$psi" --arg avail "$(meminfo_kb MemAvailable)" --arg total "$(meminfo_kb MemTotal)" --arg iface "$iface" --arg link "$link" --arg link_source "$link_source" ' include "fleet-role-stats"; def number: if . == "" then null else tonumber end; . as $entry | {t: $now, tcp: $tcp, etcd: $etcd, restarted: $restarted, psi: ($psi | number), avail_kb: ($avail | number), total_kb: ($total | number), - link: $link, iface: (if $iface == "" then null else $iface end), fsync_ms: $fsync} as $sample + link: $link, link_source: $link_source, iface: (if $iface == "" then null else $iface end), fsync_ms: $fsync} as $sample | {etcd_id: $etcd_id, last: $cumulative, fsync_at: $fsync_at, samples: ((($entry.samples // []) + [$sample]) | .[-window_samples:])}' <<< "$entry")" || return 1 patch="$(mktemp)" jq --arg node "$NODE_NAME" '{data: {($node): tojson}}' <<< "$new_entry" > "$patch" @@ -236,6 +220,7 @@ fi : "${NAMESPACE:?NAMESPACE is required}" : "${ELECTION_TIMEOUT_MS:?ELECTION_TIMEOUT_MS is required}" : "${ELIGIBLE_LABEL:?ELIGIBLE_LABEL is required}" +: "${LINK_LABEL:?LINK_LABEL is required}" trap retire TERM INT diff --git a/scripts/fleet-role.jq b/scripts/fleet-role.jq index d10c1e07..bce988d8 100644 --- a/scripts/fleet-role.jq +++ b/scripts/fleet-role.jq @@ -105,6 +105,7 @@ $config as $c fresh: ($last_at != null and $last_at >= ($now - $stale_seconds)), samples: ($w | length), link: ($w[-1].link // "unknown"), + link_source: ($w[-1].link_source // null), interface: ($w[-1].iface // null), pairs: $pairs, rtt_ms: $rtt_ms, @@ -176,6 +177,7 @@ $config as $c verdict: (if ($gaps | length) > 0 then "unknown" elif ($faults | length) > 0 then "unfit" else "fit" end), reasons: (if ($gaps | length) > 0 then $gaps else $faults end), link: $m.link, + link_source: $m.link_source, samples: $m.samples, figures: {rtt_p99_ms: ($m.rtt_ms | round2), rtt_peer: $m.rtt_peer, wal_fsync_p99_ms: ($m.wal_p99_ms | round2), backend_commit_p99_ms: ($m.commit_p99_ms | round2), fdatasync_p99_ms: ($m.fsync_p99_ms | round2), diff --git a/scripts/fleet-role.sh b/scripts/fleet-role.sh index 8a7a78e0..f89477d4 100755 --- a/scripts/fleet-role.sh +++ b/scripts/fleet-role.sh @@ -80,7 +80,7 @@ kubectl patch configmap "$status_configmap" -n "$namespace" --type merge --patch # What each host's verdict rests on, then the decision, in the log. while IFS= read -r line; do log "$line"; done < <(jq -r ' .hosts | to_entries[] - | "\(.key) (\(.value.role)\(if .value.pinned then ", pinned" else "" end)\(if .value.eligible then "" else ", not eligible" end)): \(.value.verdict), \(.value.reasons | if length == 0 then "all readings within their limits" else join("; ") end)"' <<< "$result") + | "\(.key) (\(.value.role)\(if .value.pinned then ", pinned" else "" end)\(if .value.eligible then "" else ", not eligible" end), link \(.value.link)): \(.value.verdict), \(.value.reasons | if length == 0 then "all readings within their limits" else join("; ") end)"' <<< "$result") while IFS= read -r line; do log "$line"; done < <(jq -r ' if .decision.action == "swap" then "DRY RUN: would swap out \(.decision.swap.out) for \(.decision.swap.in): \(.decision.because | join("; "))" else "DRY RUN: would not act: \(.decision.because | join("; "))\(if .decision.wanted == null and (.blocks | length) > 0 then " (also: \(.blocks | join("; ")))" else "" end)" end' <<< "$result") diff --git a/tests/fleet_role/image-smoke.sh b/tests/fleet_role/image-smoke.sh index bafa97d8..bfc425f5 100755 --- a/tests/fleet_role/image-smoke.sh +++ b/tests/fleet_role/image-smoke.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Runs what the role controller's pods run, from the image as built (fleet-role/Dockerfile), with nothing that reaches a cluster: the tools and files the pods need are present and executable, the jq modules compile under the image's own jq (not the host's), the probe's check mode reads the memory signals through the regrade core the image carries, and one pass of the controller runs end to end against a kubectl that answers an empty cluster and records the status it is asked to publish. Needs only Docker. +# Runs what the role controller's pods run, from the image as built (fleet-role/Dockerfile), with nothing that reaches a cluster: the tools and files the pods need are present and executable, the jq modules compile under the image's own jq (not the host's), the probe's check mode reads the memory signals through the regrade core the image carries and the link through the cluster role's link reader, and one pass of the controller runs end to end against a kubectl that answers an empty cluster and records the status it is asked to publish. Needs only Docker. # # Usage: tests/fleet_role/image-smoke.sh (the image must already be built) set -euo pipefail @@ -14,7 +14,7 @@ docker run --rm --entrypoint bash "$image" -c ' set -euo pipefail for tool in jq kubectl curl fio; do command -v "$tool" > /dev/null; done test -x /app/loop.sh -a -x /app/scripts/fleet-role.sh -a -x /app/scripts/fleet-role-probe.sh - test -f /app/fleet-regrade-core.sh -a -f /app/scripts/fleet-role.jq -a -f /app/scripts/fleet-role-stats.jq' + test -f /app/fleet-regrade-core.sh -a -f /app/node-link.sh -a -f /app/scripts/fleet-role.jq -a -f /app/scripts/fleet-role-stats.jq' docker run --rm --entrypoint jq "$image" -en -L /app/scripts 'include "fleet-role-stats"; {window: window_samples, tail: tail_percentile} | select(.window == 100 and .tail == 99)' > /dev/null @@ -30,6 +30,9 @@ host_has_psi=false docker run --rm --entrypoint /app/scripts/fleet-role-probe.sh "$image" check memory \ | jq -e --argjson psi "$host_has_psi" '(.psi_avg300 != null) == $psi and .mem_total_kb > 0' > /dev/null +# The link reader the image carries reads the container's own interface: a veth, whose link it cannot see. +docker run --rm --entrypoint /app/scripts/fleet-role-probe.sh "$image" check link | jq -e '.link == "unknown" and (.interface | length > 0)' > /dev/null + # A kubectl that answers an empty cluster and keeps the patch it is given, in place of the image's own on the PATH. mkdir "$work/out" chmod 777 "$work/out" diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index 7081a9c7..94fcafff 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Mesh integration test for the github_runner_cluster role: stands up three k3s nodes in Docker on one machine, each in its own copy of the Compose project, and has the role join them through Headscale. Scenarios: hosted headscale_hosted: the role runs Headscale under Compose beside the first node. in_cluster headscale_in_cluster: the role runs Headscale inside the cluster, on the bootstrap server. existing headscale_existing: the policy check fails, with the entry to add, against a server whose policy lacks autoApprovers, and passes once it has one. Every scenario uses automatic server selection (three hosts, no overrides, so three servers). native_server three Docker servers and a native host (a privileged systemd container, tests/mesh/native/Dockerfile, managed over Ansible's docker connection) that joins as an agent, is then made a server by the inventory and promoted in place: it must join etcd as a learner, be promoted to a voter with its mesh address as its peer URL, and keep the cluster writable throughout, and a rerun must not restart it. role_controller four hosts (three servers and a standby agent, with etcd's data on memory so the machine's disk does not decide): the role controller's probe and controller run in the cluster from the image built from this checkout, with the manifests the ARC role renders; every host is reported fit from a full window of real samples, a voter made slow with tc netem on its mesh interface is reported unfit with the swap named, a paused voter makes the controller report that it would not act, and the controller changes nothing (not in the default set; needs jq, and takes a window of 100 samples before it can judge). compose_faulty the role refuses a Docker Compose release that recreates the containers it has just built, before writing anything (run it under such a release; it is not in the default set). host_network each node runs in a Docker daemon of its own (a docker:dind container standing in for a VM with an address on the LAN, on the shared test network) with github_runner_cluster_network_mode host, joined through a Headscale server the test runs (headscale_existing); the nodes' Tailscale clients must then reach each other directly at those daemons' addresses, and the role must not publish any port; then the role must refuse to move two running nodes to the bridge network in one run, and move one there and back through its guard, the node keeping its mesh address and etcd membership. Linux only, since the control node reaches each daemon at its address on the test network. The cluster scenarios assert that etcd has three members, that every node is Ready, and that pods on different nodes reach each other over the mesh. +# Mesh integration test for the github_runner_cluster role: stands up three k3s nodes in Docker on one machine, each in its own copy of the Compose project, and has the role join them through Headscale. Scenarios: hosted headscale_hosted: the role runs Headscale under Compose beside the first node. in_cluster headscale_in_cluster: the role runs Headscale inside the cluster, on the bootstrap server. existing headscale_existing: the policy check fails, with the entry to add, against a server whose policy lacks autoApprovers, and passes once it has one. Every scenario uses automatic server selection (three hosts, no overrides, so three servers). native_server three Docker servers and a native host (a privileged systemd container, tests/mesh/native/Dockerfile, managed over Ansible's docker connection) that joins as an agent, is then made a server by the inventory and promoted in place: it must join etcd as a learner, be promoted to a voter with its mesh address as its peer URL, and keep the cluster writable throughout, and a rerun must not restart it. role_controller four hosts (three servers and a standby agent, with etcd's data on memory so the machine's disk does not decide): the role controller's probe and controller run in the cluster from the image built from this checkout, with the manifests the ARC role renders; every host is reported fit from a full window of real samples, with the link the cluster role labelled its node with, a voter made slow with tc netem on its mesh interface is reported unfit with the swap named, a paused voter makes the controller report that it would not act, and the controller changes nothing (not in the default set; needs jq, and takes a window of 100 samples before it can judge). compose_faulty the role refuses a Docker Compose release that recreates the containers it has just built, before writing anything (run it under such a release; it is not in the default set). host_network each node runs in a Docker daemon of its own (a docker:dind container standing in for a VM with an address on the LAN, on the shared test network) with github_runner_cluster_network_mode host, joined through a Headscale server the test runs (headscale_existing); the nodes' Tailscale clients must then reach each other directly at those daemons' addresses, and the role must not publish any port; then the role must refuse to move two running nodes to the bridge network in one run, and move one there and back through its guard, the node keeping its mesh address and etcd membership. Linux only, since the control node reaches each daemon at its address on the test network. The cluster scenarios assert that etcd has three members, that every node is Ready, and that pods on different nodes reach each other over the mesh. # # With GRTEST_PIN_CONTEXT=1, the cluster scenarios run the role, and the in_cluster recovery playbook, with github_runner_cluster_docker_context pinned while the current Docker context they see points at nothing (tests/lib/docker_context.sh), so any docker call that ignored the pin fails the run; afterwards the current context must be unchanged. context_checks the role refuses a pinned context that does not exist, and a pinned context alongside DOCKER_HOST, before writing anything (not in the default set). # @@ -1148,7 +1148,7 @@ mesh_delay() { # Brings up three servers and a standby agent, runs the role controller's probe and controller in the cluster (the pods from the image built from this checkout, the manifests the ARC role renders), and checks that the controller reports what it sees on real k3s and real etcd: all hosts fit, then one voter made slow with tc netem on its mesh interface reported unfit with the swap named, then a voter paused and the controller reporting that it would not act, and throughout that it has changed nothing. scenario_role_controller() { - local index ids image_file manifests slow paused paused_name controller_node deadline account verb expected + local index ids image_file manifests slow paused paused_name controller_node deadline account verb expected link node_count=4 agent_from=4 etcd_on_memory=1 @@ -1217,8 +1217,11 @@ metadata: {name: fleet-role-status, namespace: ${platform_namespace}} data: {"status.json": ""} MANIFESTS kubectl_on 1 apply -f - < "$manifests" >/dev/null - log "The role published the labels from the inventory: eligible on every host, pinned on the first alone" + # Every host of this scenario is this machine, which the role reads the link of with its own reader. + link="$("${repo_root}/roles/github_runner_cluster/files/node-link.sh" | jq -r .link)" + log "The role published the labels from the inventory, eligible on every host and pinned on the first alone, and the medium of this machine's link (${link}) on every host" for index in $(seq 1 "$node_count"); do + [ "$(kubectl_on 1 get node "$(node_name "$index")" -o 'jsonpath={.metadata.labels.github-runner\.exadev\.com/link}')" = "$link" ] || fail "$(node_name "$index") does not carry link=${link}" [ "$(kubectl_on 1 get node "$(node_name "$index")" -o 'jsonpath={.metadata.labels.github-runner\.exadev\.com/voter-eligible}')" = true ] || fail "$(node_name "$index") is not labelled eligible" expected=false if [ "$index" = 1 ]; then expected=true; fi @@ -1242,7 +1245,8 @@ MANIFESTS wait_for "every host to be reported fit and no swap wanted" 1800 role_status_is '(.hosts | length == 4) and ([.hosts[].verdict] | all(. == "fit")) and .decision.action == "none" and (.blocks | length == 0)' role_status | jq . role_status | jq -e "(.hosts | to_entries | all(.value.eligible)) and ([.hosts[] | select(.pinned)] | length == 1) and .hosts[\"$(node_name 1)\"].pinned" >/dev/null || fail "the controller did not read the labels the role published: $(role_status | jq -c '[.hosts[] | {eligible, pinned}]')" - role_status | jq -e '[.hosts[].link] | all(. == "unknown")' >/dev/null || fail "a node in a container should have a link it cannot see: $(role_status | jq -c '[.hosts[].link]')" + # A node in a container sees only its veth, so what each probe reports is the label the role published. + role_status | jq -e --arg link "$link" '[.hosts[] | .link == $link and .link_source == "label"] | all' >/dev/null || fail "the probes did not report the link the role labelled the nodes with (${link}): $(role_status | jq -c '[.hosts[] | {link, link_source}]')" log "A voter is made slow with tc netem on its mesh interface (${role_controller_delay_ms} ms each way): the controller reports it unfit and names the swap" slow=3 diff --git a/tests/unit/fleet_role_support.py b/tests/unit/fleet_role_support.py index 364d9857..34e4ffd1 100644 --- a/tests/unit/fleet_role_support.py +++ b/tests/unit/fleet_role_support.py @@ -15,12 +15,14 @@ ROOT = Path(__file__).resolve().parents[2] SCRIPTS = ROOT / "scripts" REGRADE_CORE = ROOT / "roles" / "github_runner_cluster" / "files" / "regrade" / "fleet-regrade-core.sh" +NODE_LINK = ROOT / "roles" / "github_runner_cluster" / "files" / "node-link.sh" NAMESPACE = "github-runner-platform" PROBE_CONFIGMAP = "fleet-role-probe" STATUS_CONFIGMAP = "fleet-role-status" ELIGIBLE_LABEL = "github-runner.exadev.com/voter-eligible" PINNED_LABEL = "github-runner.exadev.com/voter-pinned" +LINK_LABEL = "github-runner.exadev.com/link" LEVEL_LABEL = "github-runner.exadev.com/regrade-level" VOTER_LABEL = "node-role.kubernetes.io/etcd" @@ -208,8 +210,9 @@ def node( ready_for: float = 24 * 3600, level: str | None = None, address: str | None = None, + link: str | None = None, ) -> dict[str, Any]: - """A Node as `kubectl get nodes -o json` lists it; ready_for is how long ago its Ready condition last changed.""" + """A Node as `kubectl get nodes -o json` lists it; ready_for is how long ago its Ready condition last changed, and link the medium the cluster role labelled it with, none when it has no such label.""" labels: dict[str, str] = {} if voter: labels[VOTER_LABEL] = "true" @@ -219,6 +222,8 @@ def node( labels[PINNED_LABEL] = "true" if level is not None: labels[LEVEL_LABEL] = level + if link is not None: + labels[LINK_LABEL] = link return { "metadata": {"name": name, "labels": labels}, "status": { @@ -272,6 +277,7 @@ def sample( avail_kb: int | None = AVAILABLE_KB, total_kb: int | None = TOTAL_KB, link: str = "wired", + link_source: str = "interface", fsync_ms: float | None = None, ) -> dict[str, Any]: return { @@ -283,6 +289,7 @@ def sample( "avail_kb": avail_kb, "total_kb": total_kb, "link": link, + "link_source": link_source, "iface": "eth0", "fsync_ms": fsync_ms, } diff --git a/tests/unit/test_fleet_role.py b/tests/unit/test_fleet_role.py index b39c643b..9c46a1d2 100644 --- a/tests/unit/test_fleet_role.py +++ b/tests/unit/test_fleet_role.py @@ -56,6 +56,7 @@ class Host: ready_for: float = 24 * 3600 level: str | None = None link: str = "wired" + link_source: str = "interface" # Milliseconds to every voter, or per peer; a list is one reading per sample. tcp: Series | dict[str, Series] = 5.0 # What etcd's own round trip histogram holds per peer, in milliseconds, for a voter; none leaves the histograms out. @@ -116,6 +117,7 @@ def populate(cluster: Cluster, hosts: list[Host]) -> None: avail_kb=h.avail_kb, total_kb=h.total_kb, link=h.link, + link_source=h.link_source, fsync_ms=None if h.voter else h.fsync_ms if i == 0 else None, ) ) @@ -556,12 +558,17 @@ def test_each_host_says_why_with_the_numbers(self) -> None: self.assertIn(f"over {UNFIT_RTT_MS:g} ms", reasons) self.assertIn(f"{ELECTION_TIMEOUT_MS} ms election timeout", reasons) self.assertIn("memory was over the regrade limit in 100 of 100 samples", reasons) - self.assertIn("a (voter): unfit", self.log) + self.assertIn("a (voter, link wired): unfit", self.log) def test_the_link_type_is_reported_for_every_host(self) -> None: status = self.pass_(healthy(a={"link": "wifi"}, d={"link": "unknown"})) self.assertEqual({name: host["link"] for name, host in status["hosts"].items()}, {"a": "wifi", "b": "wired", "c": "wired", "d": "unknown"}) + def test_where_each_hosts_link_was_read_is_reported(self) -> None: + status = self.pass_(healthy(a={"link": "wifi", "link_source": "label"})) + self.assertEqual({name: host["link_source"] for name, host in status["hosts"].items()}, {"a": "label", "b": "interface", "c": "interface", "d": "interface"}) + self.assertIn("a (voter, link wifi): fit", self.log) + def test_a_pass_writes_only_its_own_status_and_changes_nothing_in_the_cluster(self) -> None: self.pass_(healthy(a={"tcp": UNFIT_RTT_MS + 50})) calls = [json.loads(line) for line in self.cluster.calls.read_text().splitlines()] diff --git a/tests/unit/test_fleet_role_arc.py b/tests/unit/test_fleet_role_arc.py index 8d7b4dd1..b30718b0 100644 --- a/tests/unit/test_fleet_role_arc.py +++ b/tests/unit/test_fleet_role_arc.py @@ -123,7 +123,7 @@ def test_the_controller_gets_every_setting_its_script_requires(self) -> None: def test_the_probe_gets_every_setting_its_script_requires(self) -> None: daemonset = of_kind(self.manifests, "DaemonSet")[0] required = script_requirements("fleet-role-probe.sh") - self.assertEqual(required, {"NODE_NAME", "NAMESPACE", "ELECTION_TIMEOUT_MS", "ELIGIBLE_LABEL"}) + self.assertEqual(required, {"NODE_NAME", "NAMESPACE", "ELECTION_TIMEOUT_MS", "ELIGIBLE_LABEL", "LINK_LABEL"}) self.assertLessEqual(required, set(env_of(daemonset))) def test_the_default_settings_reach_the_controller_as_strings(self) -> None: @@ -156,6 +156,7 @@ def test_the_cluster_roles_own_settings_reach_the_controller_when_they_are_defin github_runner_cluster_voters=5, github_runner_cluster_voter_eligible_label="example.com/candidate", github_runner_cluster_voter_pinned_label="example.com/pinned", + github_runner_cluster_link_label="example.com/link", ) env = env_of(of_kind(manifests, "Deployment")[0]) self.assertEqual( @@ -167,6 +168,7 @@ def test_the_cluster_roles_own_settings_reach_the_controller_when_they_are_defin (env["FLEET_ROLE_VOTERS"], env["FLEET_ROLE_ELIGIBLE_LABEL"], env["FLEET_ROLE_PINNED_LABEL"], env_of(of_kind(manifests, "DaemonSet")[0])["ELIGIBLE_LABEL"]), ("5", "example.com/candidate", "example.com/pinned", "example.com/candidate"), ) + self.assertEqual(env_of(of_kind(manifests, "DaemonSet")[0])["LINK_LABEL"], "example.com/link") def test_the_controller_and_the_probe_agree_on_the_election_timeout_and_the_period(self) -> None: manifests = render(**{ARC + "sample_seconds": 30}) @@ -278,6 +280,13 @@ def test_every_jq_module_the_scripts_include_is_copied_beside_them(self) -> None for program in ("fleet-role.jq", "fleet-role-stats.jq"): self.assertRegex(dockerfile, rf"COPY [^\n]*scripts/{re.escape(program)}[^\n]* /app/scripts/") + def test_the_probe_reads_the_link_reader_the_image_carries(self) -> None: + dockerfile = (ROOT / "fleet-role" / "Dockerfile").read_text() + script = (ROOT / "scripts" / "fleet-role-probe.sh").read_text() + default = re.search(r'NODE_LINK="\$\{NODE_LINK:-([^}]+)\}"', script) + assert default is not None + self.assertIn(f"COPY roles/github_runner_cluster/files/node-link.sh {default.group(1)}", dockerfile) + def test_the_probe_reads_the_regrade_core_the_image_carries(self) -> None: dockerfile = (ROOT / "fleet-role" / "Dockerfile").read_text() script = (ROOT / "scripts" / "fleet-role-probe.sh").read_text() @@ -349,6 +358,9 @@ def test_the_voter_target_and_the_two_voter_labels(self) -> None: self.assertEqual(self.fallback("eligible_label"), self.cluster["github_runner_cluster_voter_eligible_label"]) self.assertEqual(self.fallback("pinned_label"), self.cluster["github_runner_cluster_voter_pinned_label"]) + def test_the_link_label(self) -> None: + self.assertEqual(self.fallback("link_label"), self.cluster["github_runner_cluster_link_label"]) + def test_the_level_label(self) -> None: self.assertEqual(self.fallback("level_label"), self.cluster["github_runner_cluster_regrade_level_label"]) @@ -384,6 +396,8 @@ class ValidationTest(unittest.TestCase): "max servers text": {**ON, MAX: "many"}, "same label": {**ON, ARC + "pinned_label": "example.com/x", ARC + "eligible_label": "example.com/x"}, "bad label": {**ON, ARC + "eligible_label": "has space"}, + "link label same as eligible": {**ON, ARC + "link_label": "github-runner.exadev.com/voter-eligible"}, + "bad link label": {**ON, ARC + "link_label": "has space"}, "good labels": {**ON, ARC + "eligible_label": "example.com/eligible", ARC + "pinned_label": "pinned"}, "bad settings while off": {VOTERS: 4, ARC + "sample_seconds": 0, ARC + "eligible_label": "has space"}, } @@ -435,8 +449,8 @@ def test_the_period_the_timeout_and_the_maximum_must_be_whole_numbers_of_at_leas with self.subTest(case): self.assertTrue(self.refused(case)) - def test_the_two_labels_must_be_label_keys_and_differ(self) -> None: - for case in ("same label", "bad label"): + def test_the_three_labels_must_be_label_keys_and_differ(self) -> None: + for case in ("same label", "bad label", "link label same as eligible", "bad link label"): with self.subTest(case): self.assertTrue(self.refused(case)) self.assertFalse(self.refused("good labels")) diff --git a/tests/unit/test_fleet_role_probe.py b/tests/unit/test_fleet_role_probe.py index 3c2dabce..44369d23 100644 --- a/tests/unit/test_fleet_role_probe.py +++ b/tests/unit/test_fleet_role_probe.py @@ -31,6 +31,8 @@ ELECTION_TIMEOUT_MS, ELIGIBLE_LABEL, NAMESPACE, + LINK_LABEL, + NODE_LINK, REGRADE_CORE, SAMPLE_SECONDS, TARGET, @@ -171,6 +173,8 @@ def probe(self, name: str = "me", *, voters: list[dict[str, Any]] | None = None, "SYS_CLASS_NET": str(self.sys / "class" / "net"), "PROC_NET_ROUTE": str(self.route), "FLEET_REGRADE_CORE": str(REGRADE_CORE), + "NODE_LINK": str(NODE_LINK), + "LINK_LABEL": LINK_LABEL, "PSI_FILE": str(self.psi), "MEMINFO_FILE": str(self.meminfo), } @@ -308,6 +312,26 @@ def test_no_default_route_is_unknown(self) -> None: def test_an_interface_the_tree_does_not_have_is_unknown(self) -> None: self.assertEqual(self.link_via(route_line("enp9s0")), "unknown") + def test_a_reading_of_the_pods_own_interface_says_so(self) -> None: + self.tree.pci_ethernet() + self.link_via(route_line("enp3s0")) + self.assertEqual(self.newest()["link_source"], "interface") + + def test_the_label_the_cluster_role_published_decides_over_the_pods_own_interface(self) -> None: + # What a node in a Colima VM bridged to a Mac's Wi-Fi sees: its own virtio NIC, which is backed by a device and so reads wired, while the Mac beneath it is on Wi-Fi. + self.tree.pci_ethernet("eth0") + for medium in ("wifi", "wired", "unknown"): + with self.subTest(medium): + self.nodes = [node("me", eligible=True, link=medium)] + self.assertEqual(self.link_via(route_line("eth0")), medium) + self.assertEqual((self.newest()["link_source"], self.newest()["iface"]), ("label", "eth0")) + + def test_another_nodes_label_is_not_this_ones(self) -> None: + self.tree.pci_ethernet() + self.nodes = [self.settled_node, node("other", eligible=True, link="wifi")] + self.assertEqual(self.link_via(route_line("enp3s0")), "wired") + self.assertEqual(self.newest()["link_source"], "interface") + def test_the_kernels_looping_links_do_not_stop_the_reading(self) -> None: # The tree really loops: from the class directory, an interface's subsystem is the class directory again, and its device leads back to the interface. A find that follows links fails on a tree like this, which is how a real host failed. self.tree.pci_ethernet() @@ -680,7 +704,7 @@ def loop(self, wanted: int, **env: str) -> list[dict[str, Any]]: [str(Path(__file__).resolve().parents[2] / "scripts" / "fleet-role-probe.sh")], env=self.cluster.env( NODE_NAME="me", NAMESPACE=NAMESPACE, ELECTION_TIMEOUT_MS=str(ELECTION_TIMEOUT_MS), ELIGIBLE_LABEL=ELIGIBLE_LABEL, REPORT_SECONDS="1", ETCD_PEER_PORT="1", FSYNC_DIR=str(self.fsync_dir), - SYS_CLASS_NET=str(self.sys / "class" / "net"), PROC_NET_ROUTE=str(self.route), FLEET_REGRADE_CORE=str(REGRADE_CORE), PSI_FILE=str(self.psi), MEMINFO_FILE=str(self.meminfo), **env, + SYS_CLASS_NET=str(self.sys / "class" / "net"), PROC_NET_ROUTE=str(self.route), FLEET_REGRADE_CORE=str(REGRADE_CORE), NODE_LINK=str(NODE_LINK), LINK_LABEL=LINK_LABEL, PSI_FILE=str(self.psi), MEMINFO_FILE=str(self.meminfo), **env, ), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, @@ -720,7 +744,7 @@ def stop_loop(self, **env: str) -> subprocess.Popen[bytes]: [str(SCRIPT)], env=self.cluster.env( NODE_NAME="me", NAMESPACE=NAMESPACE, ELECTION_TIMEOUT_MS=str(ELECTION_TIMEOUT_MS), ELIGIBLE_LABEL=ELIGIBLE_LABEL, REPORT_SECONDS="600", ETCD_PEER_PORT="1", FSYNC_DIR=str(self.fsync_dir), - SYS_CLASS_NET=str(self.sys / "class" / "net"), PROC_NET_ROUTE=str(self.route), FLEET_REGRADE_CORE=str(REGRADE_CORE), PSI_FILE=str(self.psi), MEMINFO_FILE=str(self.meminfo), **env, + SYS_CLASS_NET=str(self.sys / "class" / "net"), PROC_NET_ROUTE=str(self.route), FLEET_REGRADE_CORE=str(REGRADE_CORE), NODE_LINK=str(NODE_LINK), LINK_LABEL=LINK_LABEL, PSI_FILE=str(self.psi), MEMINFO_FILE=str(self.meminfo), **env, ), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, @@ -856,6 +880,7 @@ def check(self, *args: str, **env: str) -> subprocess.CompletedProcess[str]: "SYS_CLASS_NET": str(self.sys / "class" / "net"), "PROC_NET_ROUTE": str(self.route), "FLEET_REGRADE_CORE": str(REGRADE_CORE), + "NODE_LINK": str(NODE_LINK), "PSI_FILE": str(self.psi), "MEMINFO_FILE": str(self.meminfo), }