diff --git a/docs/SUPPORTED-PACKAGES.md b/docs/SUPPORTED-PACKAGES.md index 9bfee58..49da56a 100644 --- a/docs/SUPPORTED-PACKAGES.md +++ b/docs/SUPPORTED-PACKAGES.md @@ -205,7 +205,7 @@ aurtomator includes 14 version detection strategies. Choose by where your upstre ### Strategy limitations - **GitHub rate limit:** the GitHub strategies send `GITHUB_TOKEN` when it is set in the environment. The `check-updates` workflow sets it to the workflow token, which allows 1,000 requests per hour per repository. Without a token, the limit is 60 requests per hour. -- **No retry on API failure:** strategies make one attempt. A transient failure fails the package for that run. +- **Retries:** a strategy request tries a timeout, a refused connection and an HTTP 408, 429, 500, 502, 503, 504, 522 or 524 answer twice more, 5 seconds apart. A check that still fails within its time budget runs once more after the other packages and a pause of 60 seconds. A failure on both tries fails the package for that run. - **Tag strategies take only `v1.2.3` and `1.2.3` by default:** without `tag_pattern`, and without `tag_version_regex` for `github-tag`, `github-tag`, `gitlab-tag` and `gitea-tag` leave out every other tag, `latest`, `1.0.0-rc1` and `1.0.0rc1` included. A `tag_pattern` replaces that filter. They pick the highest tag with `sort -V`, which places `1.0.0-rc1` above `1.0.0`, so a pattern that lets pre-release tags through picks them. - **`gitea-tag` fetches at most 50 tags:** there is no pagination. - **`gitlab-tag` has no authentication:** private GitLab projects cannot be queried. diff --git a/docs/WORKFLOW.md b/docs/WORKFLOW.md index e512758..ee42a31 100644 --- a/docs/WORKFLOW.md +++ b/docs/WORKFLOW.md @@ -543,6 +543,8 @@ strategy: kde-tarball Outside `--update`, `check-all.sh` gives each package a time budget: 120 seconds for `check-update.sh`, and 300 seconds each for `update-pkg.sh`, `verify-pkg.sh` and `publish-pkg.sh`. `timeout` ends the script together with everything it started, makepkg, git and curl included, and the package gets `check_failed: timed out after 120 s` or `update_failed: timed out after 300 s`. `AURTOMATOR_CHECK_TIMEOUT` and `AURTOMATOR_UPDATE_TIMEOUT` change the budgets. `--update` has none, because it runs on a terminal, where git and ssh may ask for a passphrase, and a command under `timeout` cannot read the terminal. +Without a mode and with `--update`, a package whose check failed within its budget is checked once more after all the others, following a pause of 60 seconds, and the second result counts. A check that ran out of its budget is not repeated, so a run without network does not take twice as long. `AURTOMATOR_RECHECK_DELAY` changes the pause, and the tests set it to 0. + ### 4.1 Arguments ```text @@ -931,7 +933,7 @@ A strategy is a bash script in `strategies/` that finds the latest upstream vers - **Failure:** an error on stderr, a non-zero exit code. - **Version format:** `check-update.sh` rejects a version with any character outside `A-Za-z0-9._+`. -The script can rely on `yq` and `curl`. `scripts/lib.sh` provides YAML helpers and logging and sets `AURTOMATOR_DIR` to the project root. +The script can rely on `yq` and `curl`. `scripts/lib.sh` provides YAML helpers, `http_get` for requests and logging, and sets `AURTOMATOR_DIR` to the project root. ### 8.2 Template @@ -948,7 +950,6 @@ set -euo pipefail readonly NAME="${1:?Usage: my-strategy.sh }" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" readonly SCRIPT_DIR -readonly TIMEOUT=30 # shellcheck source=../scripts/lib.sh source "${SCRIPT_DIR}/../scripts/lib.sh" @@ -956,8 +957,7 @@ source "${SCRIPT_DIR}/../scripts/lib.sh" pkg_file=$(pkg_file "$NAME") my_field=$(pkg_get "$pkg_file" .upstream.my_field) -version=$(curl -sfL --max-time "$TIMEOUT" \ - "https://example.com/api/${my_field}" | +version=$(http_get "https://example.com/api/${my_field}" | yq -r '.version') || { echo "Failed to fetch version for ${my_field}" >&2 exit 1 @@ -975,7 +975,7 @@ echo "$version" 1. **Location:** `strategies/.sh`, executable (`chmod +x`). `` is the value of the YAML `strategy` field. 2. **Header:** `set -euo pipefail`, then source `lib.sh` for `pkg_file`, `pkg_get` and logging. -3. **HTTP:** `curl -sfL --max-time "$TIMEOUT"`. `-f` fails on HTTP errors, `-L` follows redirects, and `--max-time` prevents hangs. +3. **HTTP:** `http_get [curl option]...` from `lib.sh`. It runs `curl -sfL --max-time 30` and tries a timeout, a refused connection and an HTTP 408, 429, 500, 502, 503, 504, 522 or 524 answer twice more, 5 seconds apart. `-f` fails on HTTP errors and `-L` follows redirects. A test in `tests/strategies.bats` fails when a strategy calls `curl` itself. 4. **Result:** check for an empty string and for `null` from `yq`. 5. **Prefix:** strip a leading `v` with `${version#v}` when upstream tags look like `v1.0.0`. 6. **Validation:** add the name and its required fields to `scripts/validate-pkg.sh`. diff --git a/scripts/check-all.sh b/scripts/check-all.sh index 6b88b4f..005f70b 100755 --- a/scripts/check-all.sh +++ b/scripts/check-all.sh @@ -34,6 +34,9 @@ DRY_RUN="" CHECK_TIMEOUT="${AURTOMATOR_CHECK_TIMEOUT:-120}" UPDATE_TIMEOUT="${AURTOMATOR_UPDATE_TIMEOUT:-300}" +# Pause, in seconds, before a package whose check failed is checked again +RECHECK_DELAY="${AURTOMATOR_RECHECK_DELAY:-60}" + # Runs a per-package script within a budget. timeout runs it in a process # group of its own and ends the whole group, makepkg, git and curl below it # included. Such a group cannot read the terminal, where git and ssh may ask @@ -176,7 +179,6 @@ handed_failure() { updated=0 checked=0 -errors=0 # The version the check run found for a package, printed the way # check-update.sh prints it: nothing when up to date, an error on failure. @@ -199,9 +201,14 @@ checked_version() { esac } -for pkg_file in $(printf '%s\n' "${AURTOMATOR_DIR}"/packages/*.yml | sort); do - [[ -f "$pkg_file" ]] || continue - +# Checks one package, then updates, prepares or publishes it as the mode +# says, and writes its status. A check that fails fast in a mode that runs +# the strategies goes on the list for the second pass. +# Args: $1=package file, $2=pass, 1 or 2 +run_package() { + local pkg_file="$1" pass="$2" + local pkg_name found check_cmd err_file check_rc latest check_err current + local is_downgrade cmp available pushed cmd update_rc update_err pkg_name=$(pkg_get "$pkg_file" .name) # --publish keeps the status of the check run and publishes only a package # that run found outdated. Only package names from packages/ reach a path, @@ -212,28 +219,28 @@ for pkg_file in $(printf '%s\n' "${AURTOMATOR_DIR}"/packages/*.yml | sort); do "new_version: "* | "available_downgrade: "*) if ! validate_pkgver "${found#*: }" 2>/dev/null; then echo "check_failed: invalid status from the check run" >"${STATUS_DIR}/${pkg_name}" - continue + return fi if [[ -L "${WORK_DIR}/prepared" || ! -d "${WORK_DIR}/prepared/${pkg_name}" ]]; then handed_failure "$pkg_name" >"${STATUS_DIR}/${pkg_name}" - continue + return fi ;; up_to_date | check_failed:* | "downgrade_skipped: "*) printf '%s\n' "$found" >"${STATUS_DIR}/${pkg_name}" - continue + return ;; "") echo "check_failed: not checked before --publish" >"${STATUS_DIR}/${pkg_name}" - continue + return ;; *) echo "check_failed: invalid status from the check run" >"${STATUS_DIR}/${pkg_name}" - continue + return ;; esac fi - ((checked++)) || true + ((pass == 2)) || ((checked++)) || true if [[ "$MODE" == "prepare" || "$MODE" == "publish" ]]; then check_cmd=(checked_version "$pkg_name") @@ -250,14 +257,17 @@ for pkg_file in $(printf '%s\n' "${AURTOMATOR_DIR}"/packages/*.yml | sort); do rm -f "$err_file" log_err "Failed to check $pkg_name: $check_err" echo "check_failed: ${check_err}" >"${STATUS_DIR}/${pkg_name}" - ((errors++)) || true - continue + if ((pass == 1)) && [[ "${check_cmd[0]}" == bounded ]] && + ((check_rc != 124 && check_rc != 137)); then + RECHECK+=("$pkg_file") + fi + return fi rm -f "$err_file" if [[ -z "$latest" ]]; then echo "up_to_date" >"${STATUS_DIR}/${pkg_name}" - continue + return fi # Detect downgrade: current from YAML is newer than latest from upstream. @@ -278,7 +288,7 @@ for pkg_file in $(printf '%s\n' "${AURTOMATOR_DIR}"/packages/*.yml | sort); do [[ "$(pkg_get "$pkg_file" '.allow_downgrade // false')" != "true" ]]; then log_warn "$pkg_name: DOWNGRADE detected $current → $latest, skipped: allow_downgrade is not set" echo "downgrade_skipped: ${latest}" >"${STATUS_DIR}/${pkg_name}" - continue + return fi if [[ "$is_downgrade" == "true" ]]; then @@ -293,7 +303,7 @@ for pkg_file in $(printf '%s\n' "${AURTOMATOR_DIR}"/packages/*.yml | sort); do case "$MODE" in check) echo "$available" >"${STATUS_DIR}/${pkg_name}" - continue + return ;; update) log_info "Updating $pkg_name to $latest" @@ -335,22 +345,37 @@ for pkg_file in $(printf '%s\n' "${AURTOMATOR_DIR}"/packages/*.yml | sort); do *) log_err "Failed to update $pkg_name to $latest" echo "update_failed: ${update_err}" >"${STATUS_DIR}/${pkg_name}" - ((errors++)) || true ;; esac +} + +RECHECK=() +for pkg_file in $(printf '%s\n' "${AURTOMATOR_DIR}"/packages/*.yml | sort); do + [[ -f "$pkg_file" ]] || continue + run_package "$pkg_file" 1 done -# --publish counts the failures among the statuses it wrote -if [[ "$MODE" == "publish" ]]; then - errors=0 - for status_file in "$STATUS_DIR"/*; do - [[ -f "$status_file" ]] || continue - case "$(<"$status_file")" in - check_failed:* | update_failed:*) ((errors++)) || true ;; - esac +# A check that failed fast gets a second try after a pause, by when a +# failure of the upstream host can be over. A check that ran out of its +# budget gets none, so a run without network stays within the time limit +# of its job. +if ((${#RECHECK[@]} > 0)); then + log_info "Checking ${#RECHECK[@]} failed package(s) again in ${RECHECK_DELAY} s" + sleep "$RECHECK_DELAY" + for pkg_file in "${RECHECK[@]}"; do + run_package "$pkg_file" 2 done fi +# Count the failures among the statuses written +errors=0 +for status_file in "$STATUS_DIR"/*; do + [[ -f "$status_file" ]] || continue + case "$(<"$status_file")" in + check_failed:* | update_failed:*) ((errors++)) || true ;; + esac +done + echo "" log_info "Checked: $checked, Updated: $updated, Errors: $errors" diff --git a/scripts/lib.sh b/scripts/lib.sh index 56338fc..bd59796 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -79,9 +79,16 @@ warn_maintainer_line() { } # ============================================================================= -# GITHUB API +# HTTP # ============================================================================= +# GET a URL for a strategy. curl tries a timeout, a refused connection and +# an HTTP 408, 429, 500, 502, 503, 504, 522 or 524 answer twice more. +# Usage: http_get [curl option]... +http_get() { + curl -sfL --max-time 30 --retry 2 --retry-delay 5 --retry-connrefused "${@:2}" "$1" +} + # GET a GitHub API URL, with GITHUB_TOKEN when it is set. curl reads the # header from stdin: as an argument, the token is visible in ps to every # local user. @@ -91,7 +98,7 @@ github_api() { if [[ -n "${GITHUB_TOKEN:-}" ]]; then auth=(-H @-) fi - curl -sfL --max-time 30 "${auth[@]}" "$1" <<<"Authorization: Bearer ${GITHUB_TOKEN:-}" + http_get "$1" "${auth[@]}" <<<"Authorization: Bearer ${GITHUB_TOKEN:-}" } # ============================================================================= diff --git a/strategies/archpkg.sh b/strategies/archpkg.sh index e3f4a21..d566c6a 100755 --- a/strategies/archpkg.sh +++ b/strategies/archpkg.sh @@ -14,7 +14,6 @@ set -euo pipefail readonly NAME="${1:?Usage: archpkg.sh }" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" readonly SCRIPT_DIR -readonly TIMEOUT=30 # shellcheck source=../scripts/lib.sh source "${SCRIPT_DIR}/../scripts/lib.sh" @@ -22,8 +21,7 @@ source "${SCRIPT_DIR}/../scripts/lib.sh" pkg_file=$(pkg_file "$NAME") archpkg_name=$(pkg_get "$pkg_file" '.upstream.archpkg // .name') -result=$(curl -sfL --max-time "$TIMEOUT" \ - "https://archlinux.org/packages/search/json/?name=${archpkg_name}" | +result=$(http_get "https://archlinux.org/packages/search/json/?name=${archpkg_name}" | yq -r '.results[0]') || { echo "Failed to fetch Arch package info for ${archpkg_name}" >&2 exit 1 diff --git a/strategies/crates.sh b/strategies/crates.sh index 650a5e2..ffd74ec 100755 --- a/strategies/crates.sh +++ b/strategies/crates.sh @@ -12,7 +12,6 @@ set -euo pipefail readonly NAME="${1:?Usage: crates.sh }" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" readonly SCRIPT_DIR -readonly TIMEOUT=30 # shellcheck source=../scripts/lib.sh source "${SCRIPT_DIR}/../scripts/lib.sh" @@ -20,9 +19,8 @@ source "${SCRIPT_DIR}/../scripts/lib.sh" pkg_file=$(pkg_file "$NAME") registry=$(pkg_get "$pkg_file" .upstream.registry) -version=$(curl -sfL --max-time "$TIMEOUT" \ - -H "User-Agent: aurtomator/1.0" \ - "https://crates.io/api/v1/crates/${registry}" | +version=$(http_get "https://crates.io/api/v1/crates/${registry}" \ + -H "User-Agent: aurtomator/1.0" | yq -r '.crate.max_stable_version') || { echo "Failed to fetch crates.io info for ${registry}" >&2 exit 1 diff --git a/strategies/gitea-tag.sh b/strategies/gitea-tag.sh index 8ef08d5..8e9bedf 100755 --- a/strategies/gitea-tag.sh +++ b/strategies/gitea-tag.sh @@ -12,7 +12,6 @@ set -euo pipefail readonly NAME="${1:?Usage: gitea-tag.sh }" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" readonly SCRIPT_DIR -readonly TIMEOUT=30 # shellcheck source=../scripts/lib.sh source "${SCRIPT_DIR}/../scripts/lib.sh" @@ -22,8 +21,7 @@ host=$(pkg_get "$pkg_file" .upstream.host) owner_repo=$(pkg_get "$pkg_file" .upstream.project) tag_pattern=$(pkg_get "$pkg_file" '.upstream.tag_pattern // ""') -tags=$(curl -sfL --max-time "$TIMEOUT" \ - "https://${host}/api/v1/repos/${owner_repo}/tags?limit=50" | +tags=$(http_get "https://${host}/api/v1/repos/${owner_repo}/tags?limit=50" | yq -r '.[].name') || { echo "Failed to fetch tags from ${host}/${owner_repo}" >&2 exit 1 diff --git a/strategies/gitlab-tag.sh b/strategies/gitlab-tag.sh index 2191e42..046ea68 100755 --- a/strategies/gitlab-tag.sh +++ b/strategies/gitlab-tag.sh @@ -12,7 +12,6 @@ set -euo pipefail readonly NAME="${1:?Usage: gitlab-tag.sh }" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" readonly SCRIPT_DIR -readonly TIMEOUT=30 # shellcheck source=../scripts/lib.sh source "${SCRIPT_DIR}/../scripts/lib.sh" @@ -25,8 +24,7 @@ tag_pattern=$(pkg_get "$pkg_file" '.upstream.tag_pattern // ""') # URL-encode the project path (e.g., group/project → group%2Fproject) project_encoded="${project//\//%2F}" -tags=$(curl -sfL --max-time "$TIMEOUT" \ - "https://${host}/api/v4/projects/${project_encoded}/repository/tags?per_page=100" | +tags=$(http_get "https://${host}/api/v4/projects/${project_encoded}/repository/tags?per_page=100" | yq -r '.[].name') || { echo "Failed to fetch tags from ${host}/${project}" >&2 exit 1 diff --git a/strategies/kde-tarball.sh b/strategies/kde-tarball.sh index 4bd6da5..4d54006 100755 --- a/strategies/kde-tarball.sh +++ b/strategies/kde-tarball.sh @@ -11,9 +11,13 @@ set -euo pipefail readonly NAME="${1:?Usage: kde-tarball.sh }" readonly URL="https://download.kde.org/stable/${NAME}/" -readonly TIMEOUT=30 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR -page=$(curl -sfL --max-time "$TIMEOUT" "$URL") || { +# shellcheck source=../scripts/lib.sh +source "${SCRIPT_DIR}/../scripts/lib.sh" + +page=$(http_get "$URL") || { echo "Failed to fetch $URL" >&2 exit 1 } diff --git a/strategies/npm.sh b/strategies/npm.sh index 4c8b53e..c847f45 100755 --- a/strategies/npm.sh +++ b/strategies/npm.sh @@ -12,7 +12,6 @@ set -euo pipefail readonly NAME="${1:?Usage: npm.sh }" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" readonly SCRIPT_DIR -readonly TIMEOUT=30 # shellcheck source=../scripts/lib.sh source "${SCRIPT_DIR}/../scripts/lib.sh" @@ -20,8 +19,7 @@ source "${SCRIPT_DIR}/../scripts/lib.sh" pkg_file=$(pkg_file "$NAME") registry=$(pkg_get "$pkg_file" .upstream.registry) -version=$(curl -sfL --max-time "$TIMEOUT" \ - "https://registry.npmjs.org/${registry}/latest" | +version=$(http_get "https://registry.npmjs.org/${registry}/latest" | yq -r '.version') || { echo "Failed to fetch npm info for ${registry}" >&2 exit 1 diff --git a/strategies/pypi.sh b/strategies/pypi.sh index d74b8d8..3bb16b5 100755 --- a/strategies/pypi.sh +++ b/strategies/pypi.sh @@ -12,7 +12,6 @@ set -euo pipefail readonly NAME="${1:?Usage: pypi.sh }" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" readonly SCRIPT_DIR -readonly TIMEOUT=30 # shellcheck source=../scripts/lib.sh source "${SCRIPT_DIR}/../scripts/lib.sh" @@ -20,8 +19,7 @@ source "${SCRIPT_DIR}/../scripts/lib.sh" pkg_file=$(pkg_file "$NAME") registry=$(pkg_get "$pkg_file" .upstream.registry) -version=$(curl -sfL --max-time "$TIMEOUT" \ - "https://pypi.org/pypi/${registry}/json" | +version=$(http_get "https://pypi.org/pypi/${registry}/json" | yq -r '.info.version') || { echo "Failed to fetch PyPI info for ${registry}" >&2 exit 1 diff --git a/strategies/repology.sh b/strategies/repology.sh index bc4e61b..cfd19a3 100755 --- a/strategies/repology.sh +++ b/strategies/repology.sh @@ -12,7 +12,6 @@ set -euo pipefail readonly NAME="${1:?Usage: repology.sh }" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" readonly SCRIPT_DIR -readonly TIMEOUT=30 # shellcheck source=../scripts/lib.sh source "${SCRIPT_DIR}/../scripts/lib.sh" @@ -22,9 +21,8 @@ repology_name=$(pkg_get "$pkg_file" '.upstream.repology // .name') # Repology returns an array of package entries from various repos. # Filter for "newest" status to get the latest upstream version. -version=$(curl -sfL --max-time "$TIMEOUT" \ - -H "User-Agent: aurtomator/1.0" \ - "https://repology.org/api/v1/project/${repology_name}" | +version=$(http_get "https://repology.org/api/v1/project/${repology_name}" \ + -H "User-Agent: aurtomator/1.0" | yq -r '[.[] | select(.status == "newest")] | .[0].version') || { echo "Failed to fetch Repology info for ${repology_name}" >&2 exit 1 diff --git a/strategies/webpage-scrape.sh b/strategies/webpage-scrape.sh index 440e622..54c83fd 100755 --- a/strategies/webpage-scrape.sh +++ b/strategies/webpage-scrape.sh @@ -14,7 +14,6 @@ set -euo pipefail readonly NAME="${1:?Usage: webpage-scrape.sh }" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" readonly SCRIPT_DIR -readonly TIMEOUT=30 readonly MAX_SIZE=5242880 # 5MB # shellcheck source=../scripts/lib.sh @@ -37,8 +36,7 @@ if [[ "$url" != https://* ]]; then fi # Fetch page -page=$(curl -sfL --max-time "$TIMEOUT" --max-redirs 5 --max-filesize "$MAX_SIZE" \ - -A "Mozilla/5.0" "$url") || { +page=$(http_get "$url" --max-redirs 5 --max-filesize "$MAX_SIZE" -A "Mozilla/5.0") || { echo "Failed to fetch $url" >&2 exit 1 } diff --git a/tests/helpers/setup.sh b/tests/helpers/setup.sh index 0eb29fa..75efaaa 100644 --- a/tests/helpers/setup.sh +++ b/tests/helpers/setup.sh @@ -30,8 +30,8 @@ setup_test_env() { # Override AURTOMATOR_DIR so all scripts use our temp workspace export AURTOMATOR_DIR="$TEST_TMPDIR" - # No pause before an AUR clone or between attempts - export AURTOMATOR_CLONE_DELAY=0 AURTOMATOR_RETRY_DELAY=0 + # No pause before an AUR clone, between attempts or before a second check + export AURTOMATOR_CLONE_DELAY=0 AURTOMATOR_RETRY_DELAY=0 AURTOMATOR_RECHECK_DELAY=0 # git reads and writes no global config of the user running the tests export GIT_CONFIG_GLOBAL="$TEST_TMPDIR/gitconfig" diff --git a/tests/publish.bats b/tests/publish.bats index 68b924c..16d04c0 100644 --- a/tests/publish.bats +++ b/tests/publish.bats @@ -1135,6 +1135,97 @@ current: '1.0.0'" child_ended publish-pkg.sh } +# === check-all.sh second pass === + +# Helper: a check-update.sh that logs every package it checks to +# $TEST_TMPDIR/checks, exits for with the given codes, one per +# call, and finds every package up to date otherwise; a sleep that logs its +# argument to $TEST_TMPDIR/sleeps; the packages flaky and steady +# Usage: flaky_check [exit code]... +flaky_check() { + local pkg="$1" + shift + create_test_package "flaky" "github-release" "upstream: + project: owner/repo +current: '1.0.0'" + create_test_package "steady" "github-release" "upstream: + project: owner/repo +current: '1.0.0'" + printf '%s\n' "$@" >"$TEST_TMPDIR/codes" + rm -f "$TEST_TMPDIR/scripts/check-update.sh" + cat >"$TEST_TMPDIR/scripts/check-update.sh" <>"$TEST_TMPDIR/checks" +[[ "\$1" == "$pkg" ]] || exit 0 +rc=\$(head -1 "$TEST_TMPDIR/codes") +sed -i 1d "$TEST_TMPDIR/codes" +[[ -n "\$rc" ]] || exit 0 +echo "API down" >&2 +exit "\$rc" +EOF + chmod +x "$TEST_TMPDIR/scripts/check-update.sh" + printf '#!/usr/bin/env bash\necho "$1" >>"%s/sleeps"\n' "$TEST_TMPDIR" >"$MOCK_DIR/sleep" + chmod +x "$MOCK_DIR/sleep" +} + +@test "check-all: checks a package that failed again after a pause" { + flaky_check flaky 1 + AURTOMATOR_RECHECK_DELAY=9 run "$TEST_TMPDIR/scripts/check-all.sh" + [ "$status" -eq 0 ] + [ "$(cat "$TEST_TMPDIR/checks")" = $'flaky\nsteady\nflaky' ] + [ "$(cat "$TEST_TMPDIR/sleeps")" = "9" ] + [ "$(cat "$TEST_TMPDIR/.status/flaky")" = "up_to_date" ] + [ "$(cat "$TEST_TMPDIR/.status/_error_count")" = "0" ] + [[ "$output" == *"Checking 1 failed package(s) again in 9 s"* ]] + [[ "$output" == *"Checked: 2, Updated: 0, Errors: 0"* ]] +} + +@test "check-all: keeps a package that fails twice as failed" { + flaky_check flaky 1 1 + run "$TEST_TMPDIR/scripts/check-all.sh" + [ "$status" -eq 0 ] + [ "$(cat "$TEST_TMPDIR/checks")" = $'flaky\nsteady\nflaky' ] + [ "$(cat "$TEST_TMPDIR/.status/flaky")" = "check_failed: API down" ] + [ "$(cat "$TEST_TMPDIR/.status/_error_count")" = "1" ] +} + +# Without network every check runs out of its budget, and a second pass +# would double the time of the job +@test "check-all: checks a package that ran out of its budget once" { + flaky_check flaky 124 + run "$TEST_TMPDIR/scripts/check-all.sh" + [ "$status" -eq 0 ] + [ "$(cat "$TEST_TMPDIR/checks")" = $'flaky\nsteady' ] + [ ! -e "$TEST_TMPDIR/sleeps" ] + [ "$(cat "$TEST_TMPDIR/.status/flaky")" = "check_failed: timed out after 120 s" ] +} + +@test "check-all: makes no pause when every check passed" { + flaky_check flaky + run "$TEST_TMPDIR/scripts/check-all.sh" + [ "$status" -eq 0 ] + [ "$(cat "$TEST_TMPDIR/checks")" = $'flaky\nsteady' ] + [ ! -e "$TEST_TMPDIR/sleeps" ] +} + +@test "check-all --update: checks a package that failed again" { + flaky_check flaky 1 + run "$TEST_TMPDIR/scripts/check-all.sh" --update + [ "$status" -eq 0 ] + [ "$(cat "$TEST_TMPDIR/checks")" = $'flaky\nsteady\nflaky' ] + [ "$(cat "$TEST_TMPDIR/.status/flaky")" = "up_to_date" ] +} + +@test "check-all --prepare: checks no failed package again" { + flaky_check flaky + echo "check_failed: API down" >"$TEST_TMPDIR/.status/flaky" + echo "up_to_date" >"$TEST_TMPDIR/.status/steady" + run "$TEST_TMPDIR/scripts/check-all.sh" --prepare "$TEST_TMPDIR/out" + [ "$status" -eq 0 ] + [ ! -e "$TEST_TMPDIR/sleeps" ] + [ "$(cat "$TEST_TMPDIR/out/status/flaky")" = "check_failed: API down" ] +} + # === check-all.sh --verify === # Helper: one package with a prepared update and its handed over status diff --git a/tests/scripts.bats b/tests/scripts.bats index 44ddd1d..992469e 100644 --- a/tests/scripts.bats +++ b/tests/scripts.bats @@ -442,6 +442,17 @@ MOCKCURL [[ "$output" == *"HTML instead of"* ]] } +# === http_get === + +@test "lib.sh: http_get tries a transient failure again" { + printf '#!/usr/bin/env bash\nprintf "%%s\\n" "$*"\n' >"$MOCK_DIR/curl" + chmod +x "$MOCK_DIR/curl" + source "$TEST_TMPDIR/scripts/lib.sh" + run http_get https://example.invalid/a --max-redirs 5 + [ "$status" -eq 0 ] + [ "$output" = "-sfL --max-time 30 --retry 2 --retry-delay 5 --retry-connrefused --max-redirs 5 https://example.invalid/a" ] +} + # === validate_pkgver === @test "lib.sh: validate_pkgver accepts versions the strategies produce" { diff --git a/tests/strategies.bats b/tests/strategies.bats index 9b95291..4b11868 100644 --- a/tests/strategies.bats +++ b/tests/strategies.bats @@ -556,6 +556,21 @@ run_github_strategies() { ! grep -q -- '-H' "$TEST_TMPDIR/curl.args" } +@test "github strategies: try a failed request again" { + setup_github_strategies + run_github_strategies + calls=$(wc -l <"$TEST_TMPDIR/curl.args") + [ "$calls" -gt 0 ] + [ "$(grep -c -- '--retry 2 --retry-delay 5 --retry-connrefused' "$TEST_TMPDIR/curl.args")" -eq "$calls" ] +} + +# A request that goes around http_get makes one attempt +@test "strategies: send every request through http_get" { + run grep -nE '^[^#]*\bcurl -' "$PROJECT_DIR"/strategies/*.sh + [ "$status" -eq 1 ] + [ -z "$output" ] +} + # === Error handling: unknown package === @test "strategy fails gracefully on unknown package" {