feat(role): measure etcd vote fitness and report the role controller's decisions in dry-run

.github/workflows/ci.yml

@@ -157,12 +157,12 @@ jobs:
         shard: [0, 1, 2, 3, 4]
     steps:
       - uses: actions/checkout@v7
-      - name: Install Ansible and zsh
-        # zsh is one of the two shells the host tools role writes startup lines for, and its tests read those lines back with a real zsh.
+      - name: Install Ansible, zsh and fio
+        # zsh is one of the two shells the host tools role writes startup lines for, and its tests read those lines back with a real zsh. fio is the disk check the role controller's probe takes its fdatasync reading with, which its tests run for real.
         run: |
           set -euo pipefail
           sudo apt-get update -qq
-          sudo apt-get install -y -qq ansible pkg-config zsh
+          sudo apt-get install -y -qq ansible pkg-config zsh fio
       - name: Install mise
         # The host tools tests read the mise configuration the role renders back with a real mise, to check that every value reaches the variable as given. A release more than a week old, pinned to its published checksum so the download is the file that was tested; the version is the one the tests were written against, raised by hand with its checksum from the release's SHASUMS256.txt.
         env:
@@ -176,6 +176,20 @@ jobs:
           echo "${MISE_SHA256}  ${directory}/mise.tar.gz" | sha256sum --check --strict
           tar -xzf "${directory}/mise.tar.gz" -C "${directory}"
           echo "${directory}/mise/bin" >> "$GITHUB_PATH"
+      - name: Install etcd for the probe's live test
+        # The probe's tests start a real three-member etcd to read its metrics from. Ubuntu's own etcd-server is 3.4, which has none of the 3.6 metrics the probe reads, so the official release is used: the exact version, and the SHA-256 of its linux-amd64 tarball as the release publishes it in its SHA256SUMS asset (https://github.com/etcd-io/etcd/releases/download/v3.6.15/SHA256SUMS), checked before anything is extracted. A mismatch fails the job.
+        env:
+          ETCD_VERSION: v3.6.15
+          ETCD_LINUX_AMD64_SHA256: b51d86e6bff2168de5ba8cd631c0fe6c785c7a1b32b083079f5677d6b7e2772c
+        run: |
+          set -euo pipefail
+          [ "$(uname -m)" = x86_64 ] || { echo "the pinned checksum is for linux-amd64, not $(uname -m)" >&2; exit 1; }
+          dir="${RUNNER_TEMP}/etcd"
+          mkdir -p "$dir"
+          curl -fsSL -o "${dir}/etcd.tar.gz" "https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz"
+          echo "${ETCD_LINUX_AMD64_SHA256}  ${dir}/etcd.tar.gz" | sha256sum --check --strict
+          tar -xzf "${dir}/etcd.tar.gz" -C "$dir" --strip-components=1 "etcd-${ETCD_VERSION}-linux-amd64/etcd" "etcd-${ETCD_VERSION}-linux-amd64/etcdctl"
+          echo "ETCD_BINARY=${dir}/etcd" >> "$GITHUB_ENV"
       - name: Unit-test the collection
         # Ubuntu's ansible package puts ansible-core on the system Python the filter plugins import from. The runner image can carry another Ansible earlier on PATH, so the tests run with /usr/bin first and ANSIBLE_PLAYBOOK pointing at Ubuntu's, which makes every playbook a test runs use the same ansible-core as the filters it imports; both versions are printed for the record. A module is a shard's when its position in the list of modules, largest first, leaves this shard's index modulo the number of shards.
         env:
@@ -271,9 +285,30 @@ jobs:
           PR_TITLE: ${{ github.event.pull_request.title }}
         run: printf '%s\n' "$PR_TITLE" | npx --no-install commitlint --verbose
 
+  fleet-role-image:
+    name: Role controller image build
+    needs: verified
+    if: ${{ !cancelled() && needs.verified.outputs.skip != 'true' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v7
+      - uses: docker/setup-buildx-action@v3
+      # Built and not pushed, from the same Dockerfile and context the release workflow builds it from, so a missing COPY source or a package apk cannot install fails the pull request and not the release.
+      - uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: fleet-role/Dockerfile
+          push: false
+          load: true
+          tags: fleet-role:ci
+      - name: Run what the pods run
+        # The tools and files the pods need, the jq modules under the image's own jq, the probe's check mode through the regrade core the image carries, and one pass of the controller against a kubectl that answers an empty cluster. Nothing reaches a cluster.
+        run: tests/fleet_role/image-smoke.sh fleet-role:ci
+
   required-checks:
     name: Required Checks
-    needs: [shellcheck, actionlint, yamllint, hadolint, ansible-syntax-check, unit-tests, ansible-lint, collection-build, commitlint]
+    needs: [shellcheck, actionlint, yamllint, hadolint, ansible-syntax-check, unit-tests, ansible-lint, collection-build, commitlint, fleet-role-image]
     # Always runs, even if a dependency failed or was skipped, so it can report that failure itself rather than being skipped - a required check that's silently absent blocks a PR indefinitely instead of failing it visibly.
     if: always()
     runs-on: ubuntu-latest

.github/workflows/mesh-integration.yml

@@ -1,6 +1,6 @@
 name: Mesh integration
 
-# Brings up three k3s nodes in Docker and joins them through Headscale with the cluster role (tests/mesh/run.sh), once per scenario: headscale_hosted, headscale_in_cluster, and the headscale_existing policy check, plus the role's refusal of a Docker Compose release that recreates the containers it has just built. headscale_in_cluster also runs with github_runner_cluster_docker_context pinned while the current Docker context points at nothing, which covers the cluster role's own docker calls and the recovery playbook's, and context_checks checks the role refuses a pin it cannot honour. regrade turns memory-pressure regrading on and steps a server down to etcd-only under (simulated) pressure and back up, checking that it keeps its etcd seat and vote and that no other server steps meanwhile. native_server joins a native host (a systemd container) as an agent beside three Docker servers, then promotes it to a server in place: it must join etcd as a learner, be promoted to a voter, and keep the cluster writable throughout. Only on changes that can affect how nodes join the mesh.
+# Brings up three k3s nodes in Docker and joins them through Headscale with the cluster role (tests/mesh/run.sh), once per scenario: headscale_hosted, headscale_in_cluster, and the headscale_existing policy check, plus the role's refusal of a Docker Compose release that recreates the containers it has just built. headscale_in_cluster also runs with github_runner_cluster_docker_context pinned while the current Docker context points at nothing, which covers the cluster role's own docker calls and the recovery playbook's, and context_checks checks the role refuses a pin it cannot honour. regrade turns memory-pressure regrading on and steps a server down to etcd-only under (simulated) pressure and back up, checking that it keeps its etcd seat and vote and that no other server steps meanwhile. native_server joins a native host (a systemd container) as an agent beside three Docker servers, then promotes it to a server in place: it must join etcd as a learner, be promoted to a voter, and keep the cluster writable throughout. role_controller adds a standby agent, runs the role controller's probe and controller from the image built from this checkout, makes a voter slow with tc netem and pauses another, and checks the controller reports the voter unfit with the swap named, reports that it would not act with a voter paused, and changes nothing. Only on changes that can affect how nodes join the mesh.
 #
 # Runs on GitHub-hosted ubuntu-latest, whose Docker daemon runs the privileged containers the test needs (k3s, and Tailscale inside it). The work directory sits under runner.temp, which the job always has write access to.
 
@@ -14,6 +14,10 @@ on:
       - playbooks/recover_in_cluster_mesh.yml
       - tests/mesh/**
       - tests/lib/**
+      - tests/fleet_role/**
+      - fleet-role/**
+      - scripts/fleet-role*
+      - roles/github_runner_arc/templates/fleet-role.yaml.j2
       - .github/workflows/mesh-integration.yml
   workflow_dispatch:
 
@@ -39,6 +43,7 @@ jobs:
           - {scenario: in_cluster, compose: v5.5.1}
           - {scenario: regrade, compose: v5.5.1}
           - {scenario: native_server, compose: v5.5.1}
+          - {scenario: role_controller, compose: v5.5.1}
           - {scenario: hosted, compose: v2.39.0}
           - {scenario: compose_faulty, compose: v2.38.2}
           - {scenario: in_cluster, compose: v5.5.1, pin: true}

.github/workflows/release.yml

@@ -1,6 +1,6 @@
 name: Release
 
-# Runs semantic-release against every push to main. Almost every conventional-commit type triggers at least a patch release (see .releaserc.json's commit-analyzer releaseRules), so in practice this fires on nearly every push, stamping the collection version into galaxy.yml and the arc role's heartbeat, autoscaler, pull-secret-renewer and node-recovery image defaults, building the collection tarball, and, if a release was actually published, building and pushing all five container images (runner, heartbeat, autoscaler, pull-secret-renewer, node-recovery) tagged with the exact new version alongside floating major and latest tags on the same manifest.
+# Runs semantic-release against every push to main. Almost every conventional-commit type triggers at least a patch release (see .releaserc.json's commit-analyzer releaseRules), so in practice this fires on nearly every push, stamping the collection version into galaxy.yml and the arc role's heartbeat, autoscaler, pull-secret-renewer, node-recovery and role controller image defaults, building the collection tarball, and, if a release was actually published, building and pushing every container image (runner, heartbeat, autoscaler, pull-secret-renewer, node-recovery, fleet-role) tagged with the exact new version alongside floating major and latest tags on the same manifest.
 #
 # The release commit and tag are pushed over SSH with a write-enabled deploy key (secret RELEASE_DEPLOY_KEY): actions/checkout loads it for every later git command, and .releaserc.json's repositoryUrl is the SSH remote, so @semantic-release/git pushes with it. A deploy key is scoped to this one repository and can be listed as a ruleset bypass actor. It can't call the GitHub API, so the semantic-release process uses the ordinary secrets.GITHUB_TOKEN for creating the Release and uploading the collection tarball, which need no bypass.
 #
@@ -79,6 +79,9 @@ jobs:
           - name: node-recovery
             dockerfile: node-recovery/Dockerfile
             image: ghcr.io/exadev/github-runner-node-recovery
+          - name: fleet-role
+            dockerfile: fleet-role/Dockerfile
+            image: ghcr.io/exadev/github-runner-fleet-role
     steps:
       - uses: actions/checkout@v7
         with: