diff --git a/scripts/autoscaler.sh b/scripts/autoscaler.sh index 1db55464..a04cc3f8 100755 --- a/scripts/autoscaler.sh +++ b/scripts/autoscaler.sh @@ -373,13 +373,17 @@ rm -f "$TOP_NODES_ERR_FILE" || fail_safe "kubectl top nodes${NODE_SELECTOR:+ -l ${NODE_SELECTOR}} failed (${TOP_NODES_ERR% }): the metrics API is missing or unreachable, or no node${NODE_SELECTOR:+ matching the selector} reports metrics" [ -n "$TOP_NODES_OUTPUT" ] || fail_safe "kubectl top nodes${NODE_SELECTOR:+ -l ${NODE_SELECTOR}} returned no nodes" mem_available_pct=100 +measured_nodes=0 while read -r top_node _cpu _cpu_pct _mem mem_pct; do case $'\n'"$READY_NODES"$'\n' in *$'\n'"$top_node"$'\n'*) ;; *) continue ;; esac + measured_nodes=$(( measured_nodes + 1 )) mem_pct="${mem_pct%\%}" case "$mem_pct" in ''|*[!0-9]*) fail_safe "could not parse a node's own MEMORY% from kubectl top nodes ('$mem_pct')" ;; esac node_available_pct=$(( 100 - mem_pct )) [ "$node_available_pct" -lt "$mem_available_pct" ] && mem_available_pct=$node_available_pct done <<< "$TOP_NODES_OUTPUT" +# With every listed node left out as not Ready, no figure was read and the 100 above is only its starting value, not a measurement of free memory. +[ "$measured_nodes" -gt 0 ] || fail_safe "no Ready node${NODE_SELECTOR:+ matching ${NODE_SELECTOR}} has a memory reading in kubectl top nodes" pressure=false if [ "$mem_available_pct" -lt "$MEM_AVAILABLE_PRESSURE_PCT" ]; then diff --git a/tests/unit/test_autoscaler.py b/tests/unit/test_autoscaler.py index 5b238a5d..dda7302e 100644 --- a/tests/unit/test_autoscaler.py +++ b/tests/unit/test_autoscaler.py @@ -372,6 +372,15 @@ def test_a_ready_node_with_an_unreadable_figure_fails_safe_beside_a_not_ready_on self.assertEqual(result.returncode, 0, result.stderr) self.assertIn("FAIL-SAFE: could not parse a node's own MEMORY% from kubectl top nodes ('')", result.stderr) + def test_every_listed_node_not_ready_fails_safe_instead_of_reading_full_availability(self) -> None: + # metrics-server can keep the last figure of a silent kubelet, so kubectl top still answers while no listed node is Ready. + nodes = [{"name": "a", "labels": [self.ELIGIBLE], "mem": "99%", "ready": "False"}, {"name": "b", "labels": [self.ELIGIBLE], "mem": "99%", "ready": "Unknown"}] + result = self.poll({"ns-a": [SCHEDULED_POD], "ns-b": [SCHEDULED_POD]}, ceiling=MAX_RUNNERS * len(NAMESPACES), nodes=nodes, node_selector=self.ELIGIBLE) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("FAIL-SAFE: no Ready node", result.stderr) + for namespace in NAMESPACES: + self.assertIn(f"would patch {namespace}/release-{namespace}'s maxRunners to {RUNNING}", result.stdout) + def test_a_node_list_that_cannot_be_read_fails_safe(self) -> None: result = self.poll({"ns-a": [SCHEDULED_POD], "ns-b": [SCHEDULED_POD]}, ceiling=MAX_RUNNERS * len(NAMESPACES), nodes_fail=True) self.assertEqual(result.returncode, 0, result.stderr)