From d68c82c1a4caf600a2e12638b21f01b3cd8c558c Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 3 Nov 2026 13:24:55 +0000 Subject: [PATCH 1/2] test(mesh): wait for the Docker servers to serve before the native host's run The role's first start of a Docker server returns once Compose has started the container, before its API server serves. native_server ran the native host's play straight after, and its version check, which reads the cluster through every server and refuses when none answers, found all three still starting and refused. The scenario now waits until every server's /readyz answers and every node is Ready, and logs whether the race was live. --- tests/mesh/run.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index b65a7b28..d34abec7 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -980,6 +980,15 @@ start_native_host() { native_exec() { docker exec -i "$native_host" "$@"; } native_systemd_up() { case "$(native_exec systemctl is-system-running 2>/dev/null)" in running | degraded) return 0 ;; *) return 1 ;; esac; } +# Whether every Docker server's own API server reports ready (/readyz through its own container) and every Docker node is Ready. The role's first start of a server returns once Compose has started its container, which is before its API server serves, so a play that reads the cluster straight after (the native host's version check reads through every server, and refuses when none answers) waits for this first. +docker_servers_serving() { + local index + for index in $(seq 1 "$node_count"); do + [ "$(kubectl_on "$index" get --raw /readyz 2>/dev/null /dev/null /dev/null | awk '{print $2}')" = Ready ]; } native_unit_is() { [ "$(native_exec systemctl is-active "$1" /dev/null)" = "$2" ]; } @@ -1031,6 +1040,9 @@ scenario_native_server() { # A play runs a native host's tasks before any Docker host's (the role's first task includes them, and Ansible's linear strategy runs an include for its hosts while the others wait), so the servers a native node joins are brought up first, as the role README says. log "Bringing up three Docker servers, then the native host as an agent" run_role --limit "$(node_name 1),$(node_name 2),$(node_name 3)" + # Said either way, so the log shows whether this run's role returned before its servers served, which is the race the wait closes. + if docker_servers_serving; then log "The Docker servers already serve as the role returns"; else log "The Docker servers do not serve yet as the role returns; waiting"; fi + wait_for "every Docker server's API to serve and every Docker node to be Ready" 600 docker_servers_serving run_role --limit "$native_host" wait_for "${native_host} to be Ready as an agent" 300 native_node_ready native_unit_is k3s-agent active || fail "${native_host} does not run k3s-agent" From 0de58fd79f9ad2dd6e258358381371a5490ade3c Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 3 Nov 2026 13:24:56 +0000 Subject: [PATCH 2/2] ci: install Galaxy collections without the response cache ansible-core's _call_galaxy writes an empty cache entry before it sends a request and fills it only on success. A request it retries (429, 502, 520) therefore finds that unexpired entry on the retry and fails the install with "Missing expected 'results' in ansible-galaxy cache", which a single transient Galaxy error turned into a failed regrade job. Every install that reaches a Galaxy server, in the workflows and bootstrap.sh, now passes --no-cache, and a unit test checks each one and reproduces the cache behaviour through ansible-core's own call. --- .github/workflows/arc-upgrade.yml | 3 +- .github/workflows/helm-apply.yml | 5 +- .github/workflows/litestream.yml | 3 +- .github/workflows/mesh-integration.yml | 3 +- bootstrap.sh | 3 +- tests/unit/test_galaxy_installs.py | 74 ++++++++++++++++++++++++++ 6 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 tests/unit/test_galaxy_installs.py diff --git a/.github/workflows/arc-upgrade.yml b/.github/workflows/arc-upgrade.yml index ae5281c2..b545e42f 100644 --- a/.github/workflows/arc-upgrade.yml +++ b/.github/workflows/arc-upgrade.yml @@ -38,7 +38,8 @@ jobs: fi "${RUNNER_TEMP}/venv/bin/pip" install --quiet --prefer-binary ansible-core kubernetes echo "${RUNNER_TEMP}/venv/bin" >> "$GITHUB_PATH" - "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install kubernetes.core + # --no-cache: ansible-core's Galaxy response cache keeps an empty entry when a request it retries (a 429, 502 or 520) fails, and the retry then reads that entry and fails the install with "Missing expected 'results'" (_call_galaxy in ansible/galaxy/api.py); without the cache the retry asks the server again. + "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install --no-cache kubernetes.core - name: Run the ARC upgrade test run: tests/arc_upgrade/run.sh diff --git a/.github/workflows/helm-apply.yml b/.github/workflows/helm-apply.yml index c9879f96..1c9a30cd 100644 --- a/.github/workflows/helm-apply.yml +++ b/.github/workflows/helm-apply.yml @@ -48,9 +48,10 @@ jobs: echo "${RUNNER_TEMP}/venv/bin" >> "$GITHUB_PATH" # The newest kubernetes.core in the default path, and exactly the lowest one galaxy.yml accepts in a path of its own. floor="$(PYTHON="${RUNNER_TEMP}/venv/bin/python" tests/helm_apply/kubernetes-core-floor.sh)" - "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install "kubernetes.core>=${floor}" + # --no-cache: ansible-core's Galaxy response cache keeps an empty entry when a request it retries (a 429, 502 or 520) fails, and the retry then reads that entry and fails the install with "Missing expected 'results'" (_call_galaxy in ansible/galaxy/api.py); without the cache the retry asks the server again. + "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install --no-cache "kubernetes.core>=${floor}" # --force: otherwise ansible-galaxy counts the newest already in the default path as satisfying ==floor when the two are the same version, and installs nothing into the floor path. - "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install --force "kubernetes.core==${floor}" -p "${RUNNER_TEMP}/kubernetes-core-floor" + "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install --no-cache --force "kubernetes.core==${floor}" -p "${RUNNER_TEMP}/kubernetes-core-floor" - name: Download Helm 3 and Helm 4 run: | diff --git a/.github/workflows/litestream.yml b/.github/workflows/litestream.yml index 8fa9ad68..01f8a84e 100644 --- a/.github/workflows/litestream.yml +++ b/.github/workflows/litestream.yml @@ -68,7 +68,8 @@ jobs: python3 -m venv "${RUNNER_TEMP}/venv" fi "${RUNNER_TEMP}/venv/bin/pip" install --quiet --prefer-binary ansible-core - "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install -r ansible/requirements.yml -p "${RUNNER_TEMP}/collections" + # --no-cache: ansible-core's Galaxy response cache keeps an empty entry when a request it retries (a 429, 502 or 520) fails, and the retry then reads that entry and fails the install with "Missing expected 'results'" (_call_galaxy in ansible/galaxy/api.py); without the cache the retry asks the server again. + "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install --no-cache -r ansible/requirements.yml -p "${RUNNER_TEMP}/collections" echo "${RUNNER_TEMP}/venv/bin" >> "$GITHUB_PATH" - name: Run the test diff --git a/.github/workflows/mesh-integration.yml b/.github/workflows/mesh-integration.yml index 9da80d1f..4821fbee 100644 --- a/.github/workflows/mesh-integration.yml +++ b/.github/workflows/mesh-integration.yml @@ -80,7 +80,8 @@ jobs: python3 -m venv "${RUNNER_TEMP}/venv" fi "${RUNNER_TEMP}/venv/bin/pip" install --quiet --prefer-binary ansible-core - "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install -r ansible/requirements.yml -p "${RUNNER_TEMP}/collections" + # --no-cache: ansible-core's Galaxy response cache keeps an empty entry when a request it retries (a 429, 502 or 520) fails, and the retry then reads that entry and fails the install with "Missing expected 'results'" (_call_galaxy in ansible/galaxy/api.py); without the cache the retry asks the server again. + "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install --no-cache -r ansible/requirements.yml -p "${RUNNER_TEMP}/collections" echo "${RUNNER_TEMP}/venv/bin" >> "$GITHUB_PATH" - name: Run the ${{ matrix.scenario }} scenario diff --git a/bootstrap.sh b/bootstrap.sh index 1e746129..9205a483 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -19,7 +19,8 @@ if [ ! -x "${venv}/bin/ansible-playbook" ]; then # --prefer-binary: cryptography (an ansible-core dependency) stopped publishing Intel macOS wheels after 48.0.1, so without it pip picks the newest version on an Intel Mac and compiles it from source, which needs a working Rust and C toolchain. "${venv}/bin/pip" install --quiet --prefer-binary ansible-core fi -"${venv}/bin/ansible-galaxy" collection install -r ansible/requirements.yml -p "${venv}/collections" >/dev/null +# --no-cache: ansible-core's Galaxy response cache keeps an empty entry when a request it retries (a 429, 502 or 520) fails, and the retry then reads that entry and fails the install with "Missing expected 'results'" (_call_galaxy in ansible/galaxy/api.py); without the cache the retry asks the server again. +"${venv}/bin/ansible-galaxy" collection install --no-cache -r ansible/requirements.yml -p "${venv}/collections" >/dev/null if [ -n "${K3S_AGENT_SERVER_URL:-}" ]; then node_role=agent diff --git a/tests/unit/test_galaxy_installs.py b/tests/unit/test_galaxy_installs.py new file mode 100644 index 00000000..69a7f475 --- /dev/null +++ b/tests/unit/test_galaxy_installs.py @@ -0,0 +1,74 @@ +"""Checks that every ansible-galaxy install that reaches a Galaxy server runs without the server response cache. + +ansible-core's _call_galaxy (ansible/galaxy/api.py) writes an empty cache entry before it sends a request, and fills it only when the request succeeds. The call is retried on a 429, 502 or 520, and the retry finds the empty entry, which is unexpired, and fails with "Missing expected 'results' in ansible-galaxy cache". --no-cache leaves the cache out, so the retry asks the server again. +""" + +from __future__ import annotations + +import io +import re +import tempfile +import unittest +from pathlib import Path +from unittest import mock +from urllib.error import HTTPError + +from ansible.errors import AnsibleError +from ansible.galaxy import api + +ROOT = Path(__file__).resolve().parents[2] +INSTALL = re.compile(r"ansible-galaxy\"? collection install(?P[^\n]*)") + + +def install_lines() -> list[tuple[str, str]]: + """Every ansible-galaxy collection install in the workflows and shell scripts that resolves from a server: (file, arguments). An install of a local tarball reaches no server.""" + files = sorted((ROOT / ".github" / "workflows").glob("*.yml")) + sorted(ROOT.glob("*.sh")) + found = [] + for path in files: + for match in INSTALL.finditer(path.read_text()): + if '"$tarball"' not in match.group("args"): + found.append((str(path.relative_to(ROOT)), match.group("args"))) + return found + + +class GalaxyInstallTest(unittest.TestCase): + def test_every_install_from_a_server_runs_without_the_response_cache(self) -> None: + lines = install_lines() + self.assertGreater(len(lines), 0) + for path, args in lines: + with self.subTest(path=path, args=args): + self.assertIn("--no-cache", args.split()) + + def call_after_one_bad_gateway(self, no_cache: bool) -> tuple[list[str], BaseException | None]: + """Call the Galaxy API through ansible-core's own _call_galaxy, whose first request answers 502: the requests it sent, and what it raised (None when it succeeded).""" + calls: list[str] = [] + + def open_url(url: str, **_: object) -> io.BytesIO: + calls.append(url) + if len(calls) == 1: + raise HTTPError(url, 502, "Bad Gateway", {}, io.BytesIO(b"{}")) # type: ignore[arg-type] + return io.BytesIO(b'{"name": "example"}') + + with tempfile.TemporaryDirectory() as cache, mock.patch.object(api, "open_url", open_url), mock.patch.object(api.C, "GALAXY_CACHE_DIR", cache), mock.patch("ansible.module_utils.api.time.sleep"): + server = api.GalaxyAPI(mock.MagicMock(), "galaxy", "https://galaxy.example/api/", no_cache=no_cache) + try: + server._call_galaxy("https://galaxy.example/api/v3/collections/example/example/", cache=True) + # What it raised is the result under test: a KeyError on older ansible-core, an AnsibleError wrapping it on newer. + except (KeyError, AnsibleError) as error: + return calls, error + return calls, None + + def test_without_the_cache_a_retried_request_is_sent_again(self) -> None: + calls, error = self.call_after_one_bad_gateway(no_cache=True) + self.assertIsNone(error) + self.assertEqual(len(calls), 2) + + def test_with_the_cache_a_retried_request_reads_the_empty_entry(self) -> None: + # The behaviour --no-cache exists to avoid: the retry sends nothing and fails on the entry the failed request left. When an ansible-core release stops caching a failed request, this fails and the flag and this test can go. + calls, error = self.call_after_one_bad_gateway(no_cache=False) + self.assertEqual(len(calls), 1) + self.assertIsNotNone(error) + self.assertIn("results", str(error)) + +if __name__ == "__main__": + unittest.main()