diff --git a/.github/workflows/arc-upgrade.yml b/.github/workflows/arc-upgrade.yml index ae5281c2..b545e42f 100644 --- a/.github/workflows/arc-upgrade.yml +++ b/.github/workflows/arc-upgrade.yml @@ -38,7 +38,8 @@ jobs: fi "${RUNNER_TEMP}/venv/bin/pip" install --quiet --prefer-binary ansible-core kubernetes echo "${RUNNER_TEMP}/venv/bin" >> "$GITHUB_PATH" - "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install kubernetes.core + # --no-cache: ansible-core's Galaxy response cache keeps an empty entry when a request it retries (a 429, 502 or 520) fails, and the retry then reads that entry and fails the install with "Missing expected 'results'" (_call_galaxy in ansible/galaxy/api.py); without the cache the retry asks the server again. + "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install --no-cache kubernetes.core - name: Run the ARC upgrade test run: tests/arc_upgrade/run.sh diff --git a/.github/workflows/helm-apply.yml b/.github/workflows/helm-apply.yml index c9879f96..1c9a30cd 100644 --- a/.github/workflows/helm-apply.yml +++ b/.github/workflows/helm-apply.yml @@ -48,9 +48,10 @@ jobs: echo "${RUNNER_TEMP}/venv/bin" >> "$GITHUB_PATH" # The newest kubernetes.core in the default path, and exactly the lowest one galaxy.yml accepts in a path of its own. floor="$(PYTHON="${RUNNER_TEMP}/venv/bin/python" tests/helm_apply/kubernetes-core-floor.sh)" - "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install "kubernetes.core>=${floor}" + # --no-cache: ansible-core's Galaxy response cache keeps an empty entry when a request it retries (a 429, 502 or 520) fails, and the retry then reads that entry and fails the install with "Missing expected 'results'" (_call_galaxy in ansible/galaxy/api.py); without the cache the retry asks the server again. + "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install --no-cache "kubernetes.core>=${floor}" # --force: otherwise ansible-galaxy counts the newest already in the default path as satisfying ==floor when the two are the same version, and installs nothing into the floor path. - "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install --force "kubernetes.core==${floor}" -p "${RUNNER_TEMP}/kubernetes-core-floor" + "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install --no-cache --force "kubernetes.core==${floor}" -p "${RUNNER_TEMP}/kubernetes-core-floor" - name: Download Helm 3 and Helm 4 run: | diff --git a/.github/workflows/litestream.yml b/.github/workflows/litestream.yml index 8fa9ad68..01f8a84e 100644 --- a/.github/workflows/litestream.yml +++ b/.github/workflows/litestream.yml @@ -68,7 +68,8 @@ jobs: python3 -m venv "${RUNNER_TEMP}/venv" fi "${RUNNER_TEMP}/venv/bin/pip" install --quiet --prefer-binary ansible-core - "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install -r ansible/requirements.yml -p "${RUNNER_TEMP}/collections" + # --no-cache: ansible-core's Galaxy response cache keeps an empty entry when a request it retries (a 429, 502 or 520) fails, and the retry then reads that entry and fails the install with "Missing expected 'results'" (_call_galaxy in ansible/galaxy/api.py); without the cache the retry asks the server again. + "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install --no-cache -r ansible/requirements.yml -p "${RUNNER_TEMP}/collections" echo "${RUNNER_TEMP}/venv/bin" >> "$GITHUB_PATH" - name: Run the test diff --git a/.github/workflows/mesh-integration.yml b/.github/workflows/mesh-integration.yml index 9da80d1f..4821fbee 100644 --- a/.github/workflows/mesh-integration.yml +++ b/.github/workflows/mesh-integration.yml @@ -80,7 +80,8 @@ jobs: python3 -m venv "${RUNNER_TEMP}/venv" fi "${RUNNER_TEMP}/venv/bin/pip" install --quiet --prefer-binary ansible-core - "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install -r ansible/requirements.yml -p "${RUNNER_TEMP}/collections" + # --no-cache: ansible-core's Galaxy response cache keeps an empty entry when a request it retries (a 429, 502 or 520) fails, and the retry then reads that entry and fails the install with "Missing expected 'results'" (_call_galaxy in ansible/galaxy/api.py); without the cache the retry asks the server again. + "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install --no-cache -r ansible/requirements.yml -p "${RUNNER_TEMP}/collections" echo "${RUNNER_TEMP}/venv/bin" >> "$GITHUB_PATH" - name: Run the ${{ matrix.scenario }} scenario diff --git a/bootstrap.sh b/bootstrap.sh index 1e746129..9205a483 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -19,7 +19,8 @@ if [ ! -x "${venv}/bin/ansible-playbook" ]; then # --prefer-binary: cryptography (an ansible-core dependency) stopped publishing Intel macOS wheels after 48.0.1, so without it pip picks the newest version on an Intel Mac and compiles it from source, which needs a working Rust and C toolchain. "${venv}/bin/pip" install --quiet --prefer-binary ansible-core fi -"${venv}/bin/ansible-galaxy" collection install -r ansible/requirements.yml -p "${venv}/collections" >/dev/null +# --no-cache: ansible-core's Galaxy response cache keeps an empty entry when a request it retries (a 429, 502 or 520) fails, and the retry then reads that entry and fails the install with "Missing expected 'results'" (_call_galaxy in ansible/galaxy/api.py); without the cache the retry asks the server again. +"${venv}/bin/ansible-galaxy" collection install --no-cache -r ansible/requirements.yml -p "${venv}/collections" >/dev/null if [ -n "${K3S_AGENT_SERVER_URL:-}" ]; then node_role=agent diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index b65a7b28..d34abec7 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -980,6 +980,15 @@ start_native_host() { native_exec() { docker exec -i "$native_host" "$@"; } native_systemd_up() { case "$(native_exec systemctl is-system-running 2>/dev/null)" in running | degraded) return 0 ;; *) return 1 ;; esac; } +# Whether every Docker server's own API server reports ready (/readyz through its own container) and every Docker node is Ready. The role's first start of a server returns once Compose has started its container, which is before its API server serves, so a play that reads the cluster straight after (the native host's version check reads through every server, and refuses when none answers) waits for this first. +docker_servers_serving() { + local index + for index in $(seq 1 "$node_count"); do + [ "$(kubectl_on "$index" get --raw /readyz 2>/dev/null /dev/null /dev/null | awk '{print $2}')" = Ready ]; } native_unit_is() { [ "$(native_exec systemctl is-active "$1" /dev/null)" = "$2" ]; } @@ -1031,6 +1040,9 @@ scenario_native_server() { # A play runs a native host's tasks before any Docker host's (the role's first task includes them, and Ansible's linear strategy runs an include for its hosts while the others wait), so the servers a native node joins are brought up first, as the role README says. log "Bringing up three Docker servers, then the native host as an agent" run_role --limit "$(node_name 1),$(node_name 2),$(node_name 3)" + # Said either way, so the log shows whether this run's role returned before its servers served, which is the race the wait closes. + if docker_servers_serving; then log "The Docker servers already serve as the role returns"; else log "The Docker servers do not serve yet as the role returns; waiting"; fi + wait_for "every Docker server's API to serve and every Docker node to be Ready" 600 docker_servers_serving run_role --limit "$native_host" wait_for "${native_host} to be Ready as an agent" 300 native_node_ready native_unit_is k3s-agent active || fail "${native_host} does not run k3s-agent" diff --git a/tests/unit/test_galaxy_installs.py b/tests/unit/test_galaxy_installs.py new file mode 100644 index 00000000..69a7f475 --- /dev/null +++ b/tests/unit/test_galaxy_installs.py @@ -0,0 +1,74 @@ +"""Checks that every ansible-galaxy install that reaches a Galaxy server runs without the server response cache. + +ansible-core's _call_galaxy (ansible/galaxy/api.py) writes an empty cache entry before it sends a request, and fills it only when the request succeeds. The call is retried on a 429, 502 or 520, and the retry finds the empty entry, which is unexpired, and fails with "Missing expected 'results' in ansible-galaxy cache". --no-cache leaves the cache out, so the retry asks the server again. +""" + +from __future__ import annotations + +import io +import re +import tempfile +import unittest +from pathlib import Path +from unittest import mock +from urllib.error import HTTPError + +from ansible.errors import AnsibleError +from ansible.galaxy import api + +ROOT = Path(__file__).resolve().parents[2] +INSTALL = re.compile(r"ansible-galaxy\"? collection install(?P[^\n]*)") + + +def install_lines() -> list[tuple[str, str]]: + """Every ansible-galaxy collection install in the workflows and shell scripts that resolves from a server: (file, arguments). An install of a local tarball reaches no server.""" + files = sorted((ROOT / ".github" / "workflows").glob("*.yml")) + sorted(ROOT.glob("*.sh")) + found = [] + for path in files: + for match in INSTALL.finditer(path.read_text()): + if '"$tarball"' not in match.group("args"): + found.append((str(path.relative_to(ROOT)), match.group("args"))) + return found + + +class GalaxyInstallTest(unittest.TestCase): + def test_every_install_from_a_server_runs_without_the_response_cache(self) -> None: + lines = install_lines() + self.assertGreater(len(lines), 0) + for path, args in lines: + with self.subTest(path=path, args=args): + self.assertIn("--no-cache", args.split()) + + def call_after_one_bad_gateway(self, no_cache: bool) -> tuple[list[str], BaseException | None]: + """Call the Galaxy API through ansible-core's own _call_galaxy, whose first request answers 502: the requests it sent, and what it raised (None when it succeeded).""" + calls: list[str] = [] + + def open_url(url: str, **_: object) -> io.BytesIO: + calls.append(url) + if len(calls) == 1: + raise HTTPError(url, 502, "Bad Gateway", {}, io.BytesIO(b"{}")) # type: ignore[arg-type] + return io.BytesIO(b'{"name": "example"}') + + with tempfile.TemporaryDirectory() as cache, mock.patch.object(api, "open_url", open_url), mock.patch.object(api.C, "GALAXY_CACHE_DIR", cache), mock.patch("ansible.module_utils.api.time.sleep"): + server = api.GalaxyAPI(mock.MagicMock(), "galaxy", "https://galaxy.example/api/", no_cache=no_cache) + try: + server._call_galaxy("https://galaxy.example/api/v3/collections/example/example/", cache=True) + # What it raised is the result under test: a KeyError on older ansible-core, an AnsibleError wrapping it on newer. + except (KeyError, AnsibleError) as error: + return calls, error + return calls, None + + def test_without_the_cache_a_retried_request_is_sent_again(self) -> None: + calls, error = self.call_after_one_bad_gateway(no_cache=True) + self.assertIsNone(error) + self.assertEqual(len(calls), 2) + + def test_with_the_cache_a_retried_request_reads_the_empty_entry(self) -> None: + # The behaviour --no-cache exists to avoid: the retry sends nothing and fails on the entry the failed request left. When an ansible-core release stops caching a failed request, this fails and the flag and this test can go. + calls, error = self.call_after_one_bad_gateway(no_cache=False) + self.assertEqual(len(calls), 1) + self.assertIsNotNone(error) + self.assertIn("results", str(error)) + +if __name__ == "__main__": + unittest.main()