fix(autoscaler): fail safe when no Ready node has a memory reading

scripts/autoscaler.sh

@@ -373,13 +373,17 @@ rm -f "$TOP_NODES_ERR_FILE"
   || fail_safe "kubectl top nodes${NODE_SELECTOR:+ -l ${NODE_SELECTOR}} failed (${TOP_NODES_ERR% }): the metrics API is missing or unreachable, or no node${NODE_SELECTOR:+ matching the selector} reports metrics"
 [ -n "$TOP_NODES_OUTPUT" ] || fail_safe "kubectl top nodes${NODE_SELECTOR:+ -l ${NODE_SELECTOR}} returned no nodes"
 mem_available_pct=100
+measured_nodes=0
 while read -r top_node _cpu _cpu_pct _mem mem_pct; do
   case $'\n'"$READY_NODES"$'\n' in *$'\n'"$top_node"$'\n'*) ;; *) continue ;; esac
+  measured_nodes=$(( measured_nodes + 1 ))
   mem_pct="${mem_pct%\%}"
   case "$mem_pct" in ''|*[!0-9]*) fail_safe "could not parse a node's own MEMORY% from kubectl top nodes ('$mem_pct')" ;; esac
   node_available_pct=$(( 100 - mem_pct ))
   [ "$node_available_pct" -lt "$mem_available_pct" ] && mem_available_pct=$node_available_pct
 done <<< "$TOP_NODES_OUTPUT"
+# With every listed node left out as not Ready, no figure was read and the 100 above is only its starting value, not a measurement of free memory.
+[ "$measured_nodes" -gt 0 ] || fail_safe "no Ready node${NODE_SELECTOR:+ matching ${NODE_SELECTOR}} has a memory reading in kubectl top nodes"
 
 pressure=false
 if [ "$mem_available_pct" -lt "$MEM_AVAILABLE_PRESSURE_PCT" ]; then

tests/unit/test_autoscaler.py

@@ -372,6 +372,15 @@ def test_a_ready_node_with_an_unreadable_figure_fails_safe_beside_a_not_ready_on
         self.assertEqual(result.returncode, 0, result.stderr)
         self.assertIn("FAIL-SAFE: could not parse a node's own MEMORY% from kubectl top nodes ('<unknown>')", result.stderr)
 
+    def test_every_listed_node_not_ready_fails_safe_instead_of_reading_full_availability(self) -> None:
+        # metrics-server can keep the last figure of a silent kubelet, so kubectl top still answers while no listed node is Ready.
+        nodes = [{"name": "a", "labels": [self.ELIGIBLE], "mem": "99%", "ready": "False"}, {"name": "b", "labels": [self.ELIGIBLE], "mem": "99%", "ready": "Unknown"}]
+        result = self.poll({"ns-a": [SCHEDULED_POD], "ns-b": [SCHEDULED_POD]}, ceiling=MAX_RUNNERS * len(NAMESPACES), nodes=nodes, node_selector=self.ELIGIBLE)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertIn("FAIL-SAFE: no Ready node", result.stderr)
+        for namespace in NAMESPACES:
+            self.assertIn(f"would patch {namespace}/release-{namespace}'s maxRunners to {RUNNING}", result.stdout)
+
     def test_a_node_list_that_cannot_be_read_fails_safe(self) -> None:
         result = self.poll({"ns-a": [SCHEDULED_POD], "ns-b": [SCHEDULED_POD]}, ceiling=MAX_RUNNERS * len(NAMESPACES), nodes_fail=True)
         self.assertEqual(result.returncode, 0, result.stderr)