From 0ee67953a9703bc504b3b85af075732513e79754 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sun, 1 Nov 2026 23:26:25 +0000 Subject: [PATCH 1/2] fix(autoscaler): fail safe when no Ready node has a memory reading With every listed node left out as not Ready, the worst-case available memory stayed at its starting value of 100 and read as no pressure. --- scripts/autoscaler.sh | 4 ++++ tests/unit/test_autoscaler.py | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/scripts/autoscaler.sh b/scripts/autoscaler.sh index 1db55464..a04cc3f8 100755 --- a/scripts/autoscaler.sh +++ b/scripts/autoscaler.sh @@ -373,13 +373,17 @@ rm -f "$TOP_NODES_ERR_FILE" || fail_safe "kubectl top nodes${NODE_SELECTOR:+ -l ${NODE_SELECTOR}} failed (${TOP_NODES_ERR% }): the metrics API is missing or unreachable, or no node${NODE_SELECTOR:+ matching the selector} reports metrics" [ -n "$TOP_NODES_OUTPUT" ] || fail_safe "kubectl top nodes${NODE_SELECTOR:+ -l ${NODE_SELECTOR}} returned no nodes" mem_available_pct=100 +measured_nodes=0 while read -r top_node _cpu _cpu_pct _mem mem_pct; do case $'\n'"$READY_NODES"$'\n' in *$'\n'"$top_node"$'\n'*) ;; *) continue ;; esac + measured_nodes=$(( measured_nodes + 1 )) mem_pct="${mem_pct%\%}" case "$mem_pct" in ''|*[!0-9]*) fail_safe "could not parse a node's own MEMORY% from kubectl top nodes ('$mem_pct')" ;; esac node_available_pct=$(( 100 - mem_pct )) [ "$node_available_pct" -lt "$mem_available_pct" ] && mem_available_pct=$node_available_pct done <<< "$TOP_NODES_OUTPUT" +# With every listed node left out as not Ready, no figure was read and the 100 above is only its starting value, not a measurement of free memory. +[ "$measured_nodes" -gt 0 ] || fail_safe "no Ready node${NODE_SELECTOR:+ matching ${NODE_SELECTOR}} has a memory reading in kubectl top nodes" pressure=false if [ "$mem_available_pct" -lt "$MEM_AVAILABLE_PRESSURE_PCT" ]; then diff --git a/tests/unit/test_autoscaler.py b/tests/unit/test_autoscaler.py index 5b238a5d..8502947b 100644 --- a/tests/unit/test_autoscaler.py +++ b/tests/unit/test_autoscaler.py @@ -372,6 +372,14 @@ def test_a_ready_node_with_an_unreadable_figure_fails_safe_beside_a_not_ready_on self.assertEqual(result.returncode, 0, result.stderr) self.assertIn("FAIL-SAFE: could not parse a node's own MEMORY% from kubectl top nodes ('')", result.stderr) + def test_every_listed_node_not_ready_fails_safe_instead_of_reading_full_availability(self) -> None: + nodes = [{"name": "a", "labels": [self.ELIGIBLE], "mem": "", "ready": "False"}, {"name": "b", "labels": [self.ELIGIBLE], "mem": "", "ready": "Unknown"}] + result = self.poll({"ns-a": [SCHEDULED_POD], "ns-b": [SCHEDULED_POD]}, ceiling=MAX_RUNNERS * len(NAMESPACES), nodes=nodes, node_selector=self.ELIGIBLE) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("FAIL-SAFE: no Ready node", result.stderr) + for namespace in NAMESPACES: + self.assertIn(f"would patch {namespace}/release-{namespace}'s maxRunners to {RUNNING}", result.stdout) + def test_a_node_list_that_cannot_be_read_fails_safe(self) -> None: result = self.poll({"ns-a": [SCHEDULED_POD], "ns-b": [SCHEDULED_POD]}, ceiling=MAX_RUNNERS * len(NAMESPACES), nodes_fail=True) self.assertEqual(result.returncode, 0, result.stderr) From bfbc2a1bb1538c73ed5e285453cc1b46a8c82e50 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sun, 1 Nov 2026 23:28:36 +0000 Subject: [PATCH 2/2] test(autoscaler): cover the guard with stale figures from silent kubelets kubectl top fails outright when no listed node has metrics, so the case that reaches the guard is a silent kubelet's last figure on a node that is not Ready. --- tests/unit/test_autoscaler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_autoscaler.py b/tests/unit/test_autoscaler.py index 8502947b..dda7302e 100644 --- a/tests/unit/test_autoscaler.py +++ b/tests/unit/test_autoscaler.py @@ -373,7 +373,8 @@ def test_a_ready_node_with_an_unreadable_figure_fails_safe_beside_a_not_ready_on self.assertIn("FAIL-SAFE: could not parse a node's own MEMORY% from kubectl top nodes ('')", result.stderr) def test_every_listed_node_not_ready_fails_safe_instead_of_reading_full_availability(self) -> None: - nodes = [{"name": "a", "labels": [self.ELIGIBLE], "mem": "", "ready": "False"}, {"name": "b", "labels": [self.ELIGIBLE], "mem": "", "ready": "Unknown"}] + # metrics-server can keep the last figure of a silent kubelet, so kubectl top still answers while no listed node is Ready. + nodes = [{"name": "a", "labels": [self.ELIGIBLE], "mem": "99%", "ready": "False"}, {"name": "b", "labels": [self.ELIGIBLE], "mem": "99%", "ready": "Unknown"}] result = self.poll({"ns-a": [SCHEDULED_POD], "ns-b": [SCHEDULED_POD]}, ceiling=MAX_RUNNERS * len(NAMESPACES), nodes=nodes, node_selector=self.ELIGIBLE) self.assertEqual(result.returncode, 0, result.stderr) self.assertIn("FAIL-SAFE: no Ready node", result.stderr)