diff --git a/docs/operations/data-product-rollout-diagnostics.md b/docs/operations/data-product-rollout-diagnostics.md index 53323c4f63..50026c4a82 100644 --- a/docs/operations/data-product-rollout-diagnostics.md +++ b/docs/operations/data-product-rollout-diagnostics.md @@ -7,8 +7,14 @@ deadline, grant a retry, or relax an acceptance check. One failed resource read is read once more inside the shared deadline, because it says nothing about the rollout. An accepted receipt counts these in `readRetries`, -as it counts dropped public connections in `publicRetries`. A second failure of the -same read refuses with `read_incomplete`. +as it counts dropped or timed-out public requests in `publicRetries`. A second +failure of the same read refuses with `read_incomplete`. + +Each public request may use a tenth of the deadline, and at least five seconds, +before it times out. So one stalled response cannot use the whole budget. A +request that drops its connection, times out, or gets a 520 to 524 answer (the +edge could not reach the cluster) is tried once more inside the shared deadline; +any other public failure refuses at once. For snapshot failures, `phase` identifies `initial_readiness` or `rollout_stability`. `role` names a fixed resource role or check group, and @@ -26,7 +32,10 @@ cause of an incident. `snapshot`, `workloads`, `backends`, `routes`, and `readiness` describe groups when no narrower completed result exists. Public-response failures use `phase: public_contract` and their fixed endpoint -role and failure class. A public failure observed against a moving snapshot pair +role and failure class. A public failure also reports `publicStartedAfterSeconds`, +the seconds the verifier had used when that public pass began, and `attempt`, the +snapshot pair it ran on. A large value means the readiness wait used the budget; +a small one means the public requests did. A public failure observed against a moving snapshot pair is discarded. `changed` retains only the verifier's field names when a pair moved; no object values, resource identities, response bodies, addresses, or credentials appear in diagnostics. diff --git a/scripts/tests/test-data-product-rollout.sh b/scripts/tests/test-data-product-rollout.sh index a0f948d459..d071392930 100755 --- a/scripts/tests/test-data-product-rollout.sh +++ b/scripts/tests/test-data-product-rollout.sh @@ -387,6 +387,22 @@ case "${MODE:-}" in esac fi ;; + edge-origin-error-once) + if [[ ! -e "$FIXTURE/public-edge-failed-once" ]]; then + : >"$FIXTURE/public-edge-failed-once" + printf '520' + exit 0 + fi + ;; + edge-origin-error) printf '522'; exit 0 ;; + request-timeout-once | stalled-once) + if [[ ! -e "$FIXTURE/public-timed-out-once" ]]; then + : >"$FIXTURE/public-timed-out-once" + [[ "$MODE" != stalled-once ]] || sleep "$transfer_seconds" + printf '000' + exit 28 + fi + ;; dropped-at-deadline) : >"$FIXTURE/public-deadline"; printf '000'; exit 56 ;; slow-public) sleep 0.55 ;; connection-reset) printf '000'; printf 'PRIVATE-ERROR-CANARY: private upstream\n' >&2; exit 56 ;; @@ -588,11 +604,15 @@ jq -e '.httpStatus == 0 and .curlExit == 56' "${scratch}/receipt-connection-rese [[ $(wc -l <"${scratch}/receipt-connection-reset/urls") -eq 2 ]] || fail 'a dropped connection was not retried exactly once' # One dropped connection recovers, and the accepted receipt still counts it. expected_urls=12 -for mode in connection-refused-once empty-reply-once connection-reset-once; do +for mode in connection-refused-once empty-reply-once connection-reset-once request-timeout-once edge-origin-error-once; do run_case "$mode" pass "$mode" jq -e '.complete == true and .publicRetries == 1' "${scratch}/${mode}/stdout" >/dev/null || fail "$mode: the accepted receipt lost the absorbed retry" done +# A stalled response gives up at its own attempt limit, so the retry still fits the budget. +run_case stalled-once pass stalled-once 30 +jq -e '.complete == true and .publicRetries == 1' "${scratch}/stalled-once/stdout" >/dev/null || + fail 'stalled-once: a stalled response spent the budget its retry needed' expected_urls=11 # The cluster-wide inventory reports counts only, and never reads an unread list as zero. run_case inventory-clean pass '' @@ -673,13 +693,24 @@ run_case final-moving-public fail flap-public-failure 20 fail 'final-moving-public: did not reach three moving pairs with failed public checks' jq -e '.failure == "rollout_changed" and .changed == ["apps.metadata.labels"] and (has("phase") or has("role") or has("reason")) == false' "${scratch}/final-moving-public/stdout" >/dev/null || fail 'final-moving-public: final moving snapshot retained an unvalidated public failure' -for mode in dns-failure tls-failure tls-handshake-failure request-timeout; do +for mode in dns-failure tls-failure tls-handshake-failure; do run_case "$mode" fail "$mode" 15 [[ $(wc -l <"${scratch}/${mode}/urls") -eq 1 ]] || fail "$mode: a non-retryable failure was retried" done # Spend the complete transfer budget, as a connected endpoint that never responds does. # The refusal must retain its sanitized transport cause even at the shared deadline. +# An edge that keeps failing to reach the cluster is retried once, then refuses with its status. +run_case edge-origin-error fail edge-origin-error 15 +[[ $(wc -l <"${scratch}/edge-origin-error/urls") -eq 2 ]] || fail 'edge-origin-error: an edge origin error was not retried exactly once' +jq -e '.reason == "http_status" and .httpStatus == 522 and .curlExit == 0' "${scratch}/edge-origin-error/stdout" >/dev/null || + fail 'edge-origin-error: the refusal lost its status' +# A request that keeps timing out is retried once, then refuses with its cause. +run_case request-timeout fail request-timeout 15 +[[ $(wc -l <"${scratch}/request-timeout/urls") -eq 2 ]] || fail 'request-timeout: a timed-out request was not retried exactly once' +jq -e '.curlExit == 28' "${scratch}/request-timeout/stdout" >/dev/null || fail 'request-timeout: the refusal lost its curl exit' run_case transfer-timeout fail transfer-timeout 15 +jq -e '(.publicStartedAfterSeconds | type == "number" and . >= 0) and .attempt == 1' "${scratch}/transfer-timeout/stdout" >/dev/null || + fail 'transfer-timeout: the refusal did not say when the public pass began' # mutate_case verifies that a dormant-profile mutation reaches snapshot rejection. # mutate_case_impl copies the immutable healthy seed and checks one isolated rollout mutation. diff --git a/scripts/verify-data-product-rollout.sh b/scripts/verify-data-product-rollout.sh index 557d03801e..e53fe42385 100755 --- a/scripts/verify-data-product-rollout.sh +++ b/scripts/verify-data-product-rollout.sh @@ -13,7 +13,7 @@ umask 077 context='' domain='' image_digest='' chart_digest='' apps_digest='' apps_verify_file='' timeout_seconds=300 runtime_digests=() scratch='' active='' watchdog='' changed='' started_at=$SECONDS probe_runtime_digests=() -phase='' +phase='' public_started_after='' readonly repository='ghcr.io/devantler-tech/khoist' readonly namespace='data-product-controller' expected_helm_file="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/k8s/bases/apps/data-product-controller/helm-release.yaml" @@ -54,6 +54,10 @@ fail() { detail=$(printf ',"phase":"public_contract","role":"%s","reason":"%s"' "$role" "$reason") if [[ -n "$http_code" ]]; then detail+=$(printf ',"httpStatus":%d' "$http_code"); fi if [[ -n "$curl_code" ]]; then detail+=$(printf ',"curlExit":%d' "$curl_code"); fi + # Where the budget went: when the public pass that failed began, and on which snapshot pair. + if [[ "$public_started_after" =~ ^(0|[1-9][0-9]{0,3})$ ]]; then + detail+=$(printf ',"publicStartedAfterSeconds":%d,"attempt":%d' "$public_started_after" "${attempt:-0}") + fi elif [[ ("$1" == deadline_exceeded || "$1" == read_incomplete) && ("$phase" == initial_readiness || "$phase" == rollout_stability) ]]; then role=snapshot @@ -130,6 +134,10 @@ for tool in kubectl jq curl yq; do command -v "$tool" >/dev/null || fail missing scratch=$(mktemp -d) || fail private_storage_unavailable chmod 700 "$scratch" readonly deadline=$((started_at + timeout_seconds)) +# One public fetch attempt may use a tenth of the budget, never less than five seconds. +fetch_seconds=$((timeout_seconds / 10)) +((fetch_seconds >= 5)) || fetch_seconds=5 +readonly fetch_seconds set -m # remaining enforces the shared deadline and records the time left for reads. remaining() { @@ -697,24 +705,37 @@ refuse_public() { } # fetch retrieves one bounded HTTPS response without browser credentials. -# A dropped connection (refused, empty reply or reset) is retried once inside the -# shared deadline. Its cause is recorded first, so a retry the deadline forbids -# still names it; every other failure refuses on its first attempt. +# Each attempt has its own transfer limit, a tenth of the step budget (at least five +# seconds), so one stalled response cannot spend the whole deadline. A dropped +# connection (refused, empty reply or reset, or an edge 520-524) or a timed-out +# attempt is retried once inside the shared deadline. Its cause is recorded first, +# so a retry the deadline forbids still names it; every other failure refuses on +# its first attempt. fetch() { - local url=$1 role=$2 result curl_exit reason=transport_failed retried=0 + local url=$1 role=$2 result curl_exit reason retried=0 while :; do remaining # Leave one second for curl's refusal to reach the parent before its watchdog fires. # An exhausted budget still refuses; it never grants extra transfer time. ((left > 1)) || fail deadline_exceeded local transfer_seconds=$((left - 1)) connect_seconds=$((left - 1)) + ((transfer_seconds <= fetch_seconds)) || transfer_seconds=$fetch_seconds ((connect_seconds <= 5)) || connect_seconds=5 curl_exit=0 result=$(curl --disable --silent --show-error --proxy '' --connect-timeout "$connect_seconds" --max-time "$transfer_seconds" \ --max-filesize 262144 --proto '=https' --header 'Cookie:' --header 'Authorization:' \ --dump-header "$scratch/headers" --output "$scratch/body" --write-out '%{http_code}' --url "$url" 2>/dev/null) || curl_exit=$? case "$curl_exit" in - 7 | 52 | 56) ;; + 6) reason=dns_failed ;; + 35 | 60) reason=tls_failed ;; + 28) reason=request_timeout ;; + 0) reason=http_status ;; + *) reason=transport_failed ;; + esac + # The edge answers 520-524 when its own connection to the cluster failed, which is + # the same dropped connection seen one hop further away. + case "$curl_exit:$result" in + 7:* | 28:* | 52:* | 56:* | 0:52[0-4]) ;; *) break ;; esac ((retried == 0)) || break @@ -725,11 +746,6 @@ fetch() { # A retry that succeeded must not leave its first cause on a later deadline refusal. rm -f "$scratch/public-failure" if ((curl_exit)); then - case "$curl_exit" in - 6) reason=dns_failed ;; - 35 | 60) reason=tls_failed ;; - 28) reason=request_timeout ;; - esac refuse_public "$role" "$reason" "$result" "$curl_exit" return 1 fi @@ -885,6 +901,7 @@ while :; do sleep 0.2 done public_ok=1 + public_started_after=$((SECONDS - started_at)) phase=public_contract bounded quiet_public_checks || public_ok=0 ((public_ok || deadline - SECONDS > 0)) || fail public_contract_incomplete