fix(ci): bound each rollout public request and retry timeouts once

docs/operations/data-product-rollout-diagnostics.md

@@ -7,8 +7,14 @@ deadline, grant a retry, or relax an acceptance check.
 
 One failed resource read is read once more inside the shared deadline, because it
 says nothing about the rollout. An accepted receipt counts these in `readRetries`,
-as it counts dropped public connections in `publicRetries`. A second failure of the
-same read refuses with `read_incomplete`.
+as it counts dropped or timed-out public requests in `publicRetries`. A second
+failure of the same read refuses with `read_incomplete`.
+
+Each public request may use a tenth of the deadline, and at least five seconds,
+before it times out. So one stalled response cannot use the whole budget. A
+request that drops its connection, times out, or gets a 520 to 524 answer (the
+edge could not reach the cluster) is tried once more inside the shared deadline;
+any other public failure refuses at once.
 
 For snapshot failures, `phase` identifies `initial_readiness` or
 `rollout_stability`. `role` names a fixed resource role or check group, and
@@ -26,7 +32,10 @@ cause of an incident. `snapshot`, `workloads`, `backends`, `routes`, and
 `readiness` describe groups when no narrower completed result exists.
 
 Public-response failures use `phase: public_contract` and their fixed endpoint
-role and failure class. A public failure observed against a moving snapshot pair
+role and failure class. A public failure also reports `publicStartedAfterSeconds`,
+the seconds the verifier had used when that public pass began, and `attempt`, the
+snapshot pair it ran on. A large value means the readiness wait used the budget;
+a small one means the public requests did. A public failure observed against a moving snapshot pair
 is discarded. `changed` retains only the verifier's field names when a pair
 moved; no object values, resource identities, response bodies, addresses, or
 credentials appear in diagnostics.

scripts/tests/test-data-product-rollout.sh

@@ -387,6 +387,22 @@ case "${MODE:-}" in
       esac
     fi
     ;;
+  edge-origin-error-once)
+    if [[ ! -e "$FIXTURE/public-edge-failed-once" ]]; then
+      : >"$FIXTURE/public-edge-failed-once"
+      printf '520'
+      exit 0
+    fi
+    ;;
+  edge-origin-error) printf '522'; exit 0 ;;
+  request-timeout-once | stalled-once)
+    if [[ ! -e "$FIXTURE/public-timed-out-once" ]]; then
+      : >"$FIXTURE/public-timed-out-once"
+      [[ "$MODE" != stalled-once ]] || sleep "$transfer_seconds"
+      printf '000'
+      exit 28
+    fi
+    ;;
   dropped-at-deadline) : >"$FIXTURE/public-deadline"; printf '000'; exit 56 ;;
   slow-public) sleep 0.55 ;;
   connection-reset) printf '000'; printf 'PRIVATE-ERROR-CANARY: private upstream\n' >&2; exit 56 ;;
@@ -588,11 +604,15 @@ jq -e '.httpStatus == 0 and .curlExit == 56' "${scratch}/receipt-connection-rese
 [[ $(wc -l <"${scratch}/receipt-connection-reset/urls") -eq 2 ]] || fail 'a dropped connection was not retried exactly once'
 # One dropped connection recovers, and the accepted receipt still counts it.
 expected_urls=12
-for mode in connection-refused-once empty-reply-once connection-reset-once; do
+for mode in connection-refused-once empty-reply-once connection-reset-once request-timeout-once edge-origin-error-once; do
   run_case "$mode" pass "$mode"
   jq -e '.complete == true and .publicRetries == 1' "${scratch}/${mode}/stdout" >/dev/null ||
     fail "$mode: the accepted receipt lost the absorbed retry"
 done
+# A stalled response gives up at its own attempt limit, so the retry still fits the budget.
+run_case stalled-once pass stalled-once 30
+jq -e '.complete == true and .publicRetries == 1' "${scratch}/stalled-once/stdout" >/dev/null ||
+  fail 'stalled-once: a stalled response spent the budget its retry needed'
 expected_urls=11
 # The cluster-wide inventory reports counts only, and never reads an unread list as zero.
 run_case inventory-clean pass ''
@@ -673,13 +693,24 @@ run_case final-moving-public fail flap-public-failure 20
   fail 'final-moving-public: did not reach three moving pairs with failed public checks'
 jq -e '.failure == "rollout_changed" and .changed == ["apps.metadata.labels"] and (has("phase") or has("role") or has("reason")) == false' "${scratch}/final-moving-public/stdout" >/dev/null ||
   fail 'final-moving-public: final moving snapshot retained an unvalidated public failure'
-for mode in dns-failure tls-failure tls-handshake-failure request-timeout; do
+for mode in dns-failure tls-failure tls-handshake-failure; do
   run_case "$mode" fail "$mode" 15
   [[ $(wc -l <"${scratch}/${mode}/urls") -eq 1 ]] || fail "$mode: a non-retryable failure was retried"
 done
 # Spend the complete transfer budget, as a connected endpoint that never responds does.
 # The refusal must retain its sanitized transport cause even at the shared deadline.
+# An edge that keeps failing to reach the cluster is retried once, then refuses with its status.
+run_case edge-origin-error fail edge-origin-error 15
+[[ $(wc -l <"${scratch}/edge-origin-error/urls") -eq 2 ]] || fail 'edge-origin-error: an edge origin error was not retried exactly once'
+jq -e '.reason == "http_status" and .httpStatus == 522 and .curlExit == 0' "${scratch}/edge-origin-error/stdout" >/dev/null ||
+  fail 'edge-origin-error: the refusal lost its status'
+# A request that keeps timing out is retried once, then refuses with its cause.
+run_case request-timeout fail request-timeout 15
+[[ $(wc -l <"${scratch}/request-timeout/urls") -eq 2 ]] || fail 'request-timeout: a timed-out request was not retried exactly once'
+jq -e '.curlExit == 28' "${scratch}/request-timeout/stdout" >/dev/null || fail 'request-timeout: the refusal lost its curl exit'
 run_case transfer-timeout fail transfer-timeout 15
+jq -e '(.publicStartedAfterSeconds | type == "number" and . >= 0) and .attempt == 1' "${scratch}/transfer-timeout/stdout" >/dev/null ||
+  fail 'transfer-timeout: the refusal did not say when the public pass began'
 
 # mutate_case verifies that a dormant-profile mutation reaches snapshot rejection.
 # mutate_case_impl copies the immutable healthy seed and checks one isolated rollout mutation.

scripts/verify-data-product-rollout.sh

@@ -13,7 +13,7 @@ umask 077
 context='' domain='' image_digest='' chart_digest='' apps_digest='' apps_verify_file='' timeout_seconds=300
 runtime_digests=() scratch='' active='' watchdog='' changed='' started_at=$SECONDS
 probe_runtime_digests=()
-phase=''
+phase='' public_started_after=''
 readonly repository='ghcr.io/devantler-tech/khoist'
 readonly namespace='data-product-controller'
 expected_helm_file="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/k8s/bases/apps/data-product-controller/helm-release.yaml"
@@ -54,6 +54,10 @@ fail() {
     detail=$(printf ',"phase":"public_contract","role":"%s","reason":"%s"' "$role" "$reason")
     if [[ -n "$http_code" ]]; then detail+=$(printf ',"httpStatus":%d' "$http_code"); fi
     if [[ -n "$curl_code" ]]; then detail+=$(printf ',"curlExit":%d' "$curl_code"); fi
+    # Where the budget went: when the public pass that failed began, and on which snapshot pair.
+    if [[ "$public_started_after" =~ ^(0|[1-9][0-9]{0,3})$ ]]; then
+      detail+=$(printf ',"publicStartedAfterSeconds":%d,"attempt":%d' "$public_started_after" "${attempt:-0}")
+    fi
   elif [[ ("$1" == deadline_exceeded || "$1" == read_incomplete) &&
     ("$phase" == initial_readiness || "$phase" == rollout_stability) ]]; then
     role=snapshot
@@ -130,6 +134,10 @@ for tool in kubectl jq curl yq; do command -v "$tool" >/dev/null || fail missing
 scratch=$(mktemp -d) || fail private_storage_unavailable
 chmod 700 "$scratch"
 readonly deadline=$((started_at + timeout_seconds))
+# One public fetch attempt may use a tenth of the budget, never less than five seconds.
+fetch_seconds=$((timeout_seconds / 10))
+((fetch_seconds >= 5)) || fetch_seconds=5
+readonly fetch_seconds
 set -m
 # remaining enforces the shared deadline and records the time left for reads.
 remaining() {
@@ -697,24 +705,37 @@ refuse_public() {
 }
 
 # fetch retrieves one bounded HTTPS response without browser credentials.
-# A dropped connection (refused, empty reply or reset) is retried once inside the
-# shared deadline. Its cause is recorded first, so a retry the deadline forbids
-# still names it; every other failure refuses on its first attempt.
+# Each attempt has its own transfer limit, a tenth of the step budget (at least five
+# seconds), so one stalled response cannot spend the whole deadline. A dropped
+# connection (refused, empty reply or reset, or an edge 520-524) or a timed-out
+# attempt is retried once inside the shared deadline. Its cause is recorded first,
+# so a retry the deadline forbids still names it; every other failure refuses on
+# its first attempt.
 fetch() {
-  local url=$1 role=$2 result curl_exit reason=transport_failed retried=0
+  local url=$1 role=$2 result curl_exit reason retried=0
   while :; do
     remaining
     # Leave one second for curl's refusal to reach the parent before its watchdog fires.
     # An exhausted budget still refuses; it never grants extra transfer time.
     ((left > 1)) || fail deadline_exceeded
     local transfer_seconds=$((left - 1)) connect_seconds=$((left - 1))
+    ((transfer_seconds <= fetch_seconds)) || transfer_seconds=$fetch_seconds
     ((connect_seconds <= 5)) || connect_seconds=5
     curl_exit=0
     result=$(curl --disable --silent --show-error --proxy '' --connect-timeout "$connect_seconds" --max-time "$transfer_seconds" \
       --max-filesize 262144 --proto '=https' --header 'Cookie:' --header 'Authorization:' \
       --dump-header "$scratch/headers" --output "$scratch/body" --write-out '%{http_code}' --url "$url" 2>/dev/null) || curl_exit=$?
     case "$curl_exit" in
-      7 | 52 | 56) ;;
+      6) reason=dns_failed ;;
+      35 | 60) reason=tls_failed ;;
+      28) reason=request_timeout ;;
+      0) reason=http_status ;;
+      *) reason=transport_failed ;;
+    esac
+    # The edge answers 520-524 when its own connection to the cluster failed, which is
+    # the same dropped connection seen one hop further away.
+    case "$curl_exit:$result" in
+      7:* | 28:* | 52:* | 56:* | 0:52[0-4]) ;;
       *) break ;;
     esac
     ((retried == 0)) || break
@@ -725,11 +746,6 @@ fetch() {
   # A retry that succeeded must not leave its first cause on a later deadline refusal.
   rm -f "$scratch/public-failure"
   if ((curl_exit)); then
-    case "$curl_exit" in
-      6) reason=dns_failed ;;
-      35 | 60) reason=tls_failed ;;
-      28) reason=request_timeout ;;
-    esac
     refuse_public "$role" "$reason" "$result" "$curl_exit"
     return 1
   fi
@@ -885,6 +901,7 @@ while :; do
     sleep 0.2
   done
   public_ok=1
+  public_started_after=$((SECONDS - started_at))
   phase=public_contract
   bounded quiet_public_checks || public_ok=0
   ((public_ok || deadline - SECONDS > 0)) || fail public_contract_incomplete