test(mesh): wait for the Docker servers to serve, and install Galaxy collections without the cache

.github/workflows/arc-upgrade.yml

@@ -38,7 +38,8 @@ jobs:
           fi
           "${RUNNER_TEMP}/venv/bin/pip" install --quiet --prefer-binary ansible-core kubernetes
           echo "${RUNNER_TEMP}/venv/bin" >> "$GITHUB_PATH"
-          "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install kubernetes.core
+          # --no-cache: ansible-core's Galaxy response cache keeps an empty entry when a request it retries (a 429, 502 or 520) fails, and the retry then reads that entry and fails the install with "Missing expected 'results'" (_call_galaxy in ansible/galaxy/api.py); without the cache the retry asks the server again.
+          "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install --no-cache kubernetes.core
 
       - name: Run the ARC upgrade test
         run: tests/arc_upgrade/run.sh

.github/workflows/helm-apply.yml

@@ -48,9 +48,10 @@ jobs:
           echo "${RUNNER_TEMP}/venv/bin" >> "$GITHUB_PATH"
           # The newest kubernetes.core in the default path, and exactly the lowest one galaxy.yml accepts in a path of its own.
           floor="$(PYTHON="${RUNNER_TEMP}/venv/bin/python" tests/helm_apply/kubernetes-core-floor.sh)"
-          "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install "kubernetes.core>=${floor}"
+          # --no-cache: ansible-core's Galaxy response cache keeps an empty entry when a request it retries (a 429, 502 or 520) fails, and the retry then reads that entry and fails the install with "Missing expected 'results'" (_call_galaxy in ansible/galaxy/api.py); without the cache the retry asks the server again.
+          "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install --no-cache "kubernetes.core>=${floor}"
           # --force: otherwise ansible-galaxy counts the newest already in the default path as satisfying ==floor when the two are the same version, and installs nothing into the floor path.
-          "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install --force "kubernetes.core==${floor}" -p "${RUNNER_TEMP}/kubernetes-core-floor"
+          "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install --no-cache --force "kubernetes.core==${floor}" -p "${RUNNER_TEMP}/kubernetes-core-floor"
 
       - name: Download Helm 3 and Helm 4
         run: |

.github/workflows/litestream.yml

@@ -68,7 +68,8 @@ jobs:
             python3 -m venv "${RUNNER_TEMP}/venv"
           fi
           "${RUNNER_TEMP}/venv/bin/pip" install --quiet --prefer-binary ansible-core
-          "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install -r ansible/requirements.yml -p "${RUNNER_TEMP}/collections"
+          # --no-cache: ansible-core's Galaxy response cache keeps an empty entry when a request it retries (a 429, 502 or 520) fails, and the retry then reads that entry and fails the install with "Missing expected 'results'" (_call_galaxy in ansible/galaxy/api.py); without the cache the retry asks the server again.
+          "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install --no-cache -r ansible/requirements.yml -p "${RUNNER_TEMP}/collections"
           echo "${RUNNER_TEMP}/venv/bin" >> "$GITHUB_PATH"
 
       - name: Run the test

.github/workflows/mesh-integration.yml

@@ -80,7 +80,8 @@ jobs:
             python3 -m venv "${RUNNER_TEMP}/venv"
           fi
           "${RUNNER_TEMP}/venv/bin/pip" install --quiet --prefer-binary ansible-core
-          "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install -r ansible/requirements.yml -p "${RUNNER_TEMP}/collections"
+          # --no-cache: ansible-core's Galaxy response cache keeps an empty entry when a request it retries (a 429, 502 or 520) fails, and the retry then reads that entry and fails the install with "Missing expected 'results'" (_call_galaxy in ansible/galaxy/api.py); without the cache the retry asks the server again.
+          "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install --no-cache -r ansible/requirements.yml -p "${RUNNER_TEMP}/collections"
           echo "${RUNNER_TEMP}/venv/bin" >> "$GITHUB_PATH"
 
       - name: Run the ${{ matrix.scenario }} scenario

bootstrap.sh

@@ -19,7 +19,8 @@ if [ ! -x "${venv}/bin/ansible-playbook" ]; then
   # --prefer-binary: cryptography (an ansible-core dependency) stopped publishing Intel macOS wheels after 48.0.1, so without it pip picks the newest version on an Intel Mac and compiles it from source, which needs a working Rust and C toolchain.
   "${venv}/bin/pip" install --quiet --prefer-binary ansible-core
 fi
-"${venv}/bin/ansible-galaxy" collection install -r ansible/requirements.yml -p "${venv}/collections" >/dev/null
+# --no-cache: ansible-core's Galaxy response cache keeps an empty entry when a request it retries (a 429, 502 or 520) fails, and the retry then reads that entry and fails the install with "Missing expected 'results'" (_call_galaxy in ansible/galaxy/api.py); without the cache the retry asks the server again.
+"${venv}/bin/ansible-galaxy" collection install --no-cache -r ansible/requirements.yml -p "${venv}/collections" >/dev/null
 
 if [ -n "${K3S_AGENT_SERVER_URL:-}" ]; then
   node_role=agent

tests/mesh/run.sh

@@ -980,6 +980,15 @@ start_native_host() {
 
 native_exec() { docker exec -i "$native_host" "$@"; }
 native_systemd_up() { case "$(native_exec systemctl is-system-running 2>/dev/null)" in running | degraded) return 0 ;; *) return 1 ;; esac; }
+# Whether every Docker server's own API server reports ready (/readyz through its own container) and every Docker node is Ready. The role's first start of a server returns once Compose has started its container, which is before its API server serves, so a play that reads the cluster straight after (the native host's version check reads through every server, and refuses when none answers) waits for this first.
+docker_servers_serving() {
+  local index
+  for index in $(seq 1 "$node_count"); do
+    [ "$(kubectl_on "$index" get --raw /readyz 2>/dev/null </dev/null)" = ok ] || return 1
+  done
+  [ "$(kubectl_on 1 get nodes --no-headers 2>/dev/null </dev/null | grep -c ' Ready ')" = "$node_count" ]
+}
+
 # node_status names a Docker node by its index; the native host's node is named for the host.
 native_node_ready() { [ "$(kubectl_on 1 get node "$native_host" --no-headers 2>/dev/null | awk '{print $2}')" = Ready ]; }
 native_unit_is() { [ "$(native_exec systemctl is-active "$1" </dev/null 2>/dev/null)" = "$2" ]; }
@@ -1031,6 +1040,9 @@ scenario_native_server() {
   # A play runs a native host's tasks before any Docker host's (the role's first task includes them, and Ansible's linear strategy runs an include for its hosts while the others wait), so the servers a native node joins are brought up first, as the role README says.
   log "Bringing up three Docker servers, then the native host as an agent"
   run_role --limit "$(node_name 1),$(node_name 2),$(node_name 3)"
+  # Said either way, so the log shows whether this run's role returned before its servers served, which is the race the wait closes.
+  if docker_servers_serving; then log "The Docker servers already serve as the role returns"; else log "The Docker servers do not serve yet as the role returns; waiting"; fi
+  wait_for "every Docker server's API to serve and every Docker node to be Ready" 600 docker_servers_serving
   run_role --limit "$native_host"
   wait_for "${native_host} to be Ready as an agent" 300 native_node_ready
   native_unit_is k3s-agent active || fail "${native_host} does not run k3s-agent"

tests/unit/test_galaxy_installs.py

@@ -0,0 +1,74 @@
+"""Checks that every ansible-galaxy install that reaches a Galaxy server runs without the server response cache.
+
+ansible-core's _call_galaxy (ansible/galaxy/api.py) writes an empty cache entry before it sends a request, and fills it only when the request succeeds. The call is retried on a 429, 502 or 520, and the retry finds the empty entry, which is unexpired, and fails with "Missing expected 'results' in ansible-galaxy cache". --no-cache leaves the cache out, so the retry asks the server again.
+"""
+
+from __future__ import annotations
+
+import io
+import re
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+from urllib.error import HTTPError
+
+from ansible.errors import AnsibleError
+from ansible.galaxy import api
+
+ROOT = Path(__file__).resolve().parents[2]
+INSTALL = re.compile(r"ansible-galaxy\"? collection install(?P<args>[^\n]*)")
+
+
+def install_lines() -> list[tuple[str, str]]:
+    """Every ansible-galaxy collection install in the workflows and shell scripts that resolves from a server: (file, arguments). An install of a local tarball reaches no server."""
+    files = sorted((ROOT / ".github" / "workflows").glob("*.yml")) + sorted(ROOT.glob("*.sh"))
+    found = []
+    for path in files:
+        for match in INSTALL.finditer(path.read_text()):
+            if '"$tarball"' not in match.group("args"):
+                found.append((str(path.relative_to(ROOT)), match.group("args")))
+    return found
+
+
+class GalaxyInstallTest(unittest.TestCase):
+    def test_every_install_from_a_server_runs_without_the_response_cache(self) -> None:
+        lines = install_lines()
+        self.assertGreater(len(lines), 0)
+        for path, args in lines:
+            with self.subTest(path=path, args=args):
+                self.assertIn("--no-cache", args.split())
+
+    def call_after_one_bad_gateway(self, no_cache: bool) -> tuple[list[str], BaseException | None]:
+        """Call the Galaxy API through ansible-core's own _call_galaxy, whose first request answers 502: the requests it sent, and what it raised (None when it succeeded)."""
+        calls: list[str] = []
+
+        def open_url(url: str, **_: object) -> io.BytesIO:
+            calls.append(url)
+            if len(calls) == 1:
+                raise HTTPError(url, 502, "Bad Gateway", {}, io.BytesIO(b"{}"))  # type: ignore[arg-type]
+            return io.BytesIO(b'{"name": "example"}')
+
+        with tempfile.TemporaryDirectory() as cache, mock.patch.object(api, "open_url", open_url), mock.patch.object(api.C, "GALAXY_CACHE_DIR", cache), mock.patch("ansible.module_utils.api.time.sleep"):
+            server = api.GalaxyAPI(mock.MagicMock(), "galaxy", "https://galaxy.example/api/", no_cache=no_cache)
+            try:
+                server._call_galaxy("https://galaxy.example/api/v3/collections/example/example/", cache=True)
+            # What it raised is the result under test: a KeyError on older ansible-core, an AnsibleError wrapping it on newer.
+            except (KeyError, AnsibleError) as error:
+                return calls, error
+        return calls, None
+
+    def test_without_the_cache_a_retried_request_is_sent_again(self) -> None:
+        calls, error = self.call_after_one_bad_gateway(no_cache=True)
+        self.assertIsNone(error)
+        self.assertEqual(len(calls), 2)
+
+    def test_with_the_cache_a_retried_request_reads_the_empty_entry(self) -> None:
+        # The behaviour --no-cache exists to avoid: the retry sends nothing and fails on the entry the failed request left. When an ansible-core release stops caching a failed request, this fails and the flag and this test can go.
+        calls, error = self.call_after_one_bad_gateway(no_cache=False)
+        self.assertEqual(len(calls), 1)
+        self.assertIsNotNone(error)
+        self.assertIn("results", str(error))
+
+if __name__ == "__main__":
+    unittest.main()