fix(scripts): retry failed strategy checks

docs/SUPPORTED-PACKAGES.md

@@ -205,7 +205,7 @@ aurtomator includes 14 version detection strategies. Choose by where your upstre
 ### Strategy limitations
 
 - **GitHub rate limit:** the GitHub strategies send `GITHUB_TOKEN` when it is set in the environment. The `check-updates` workflow sets it to the workflow token, which allows 1,000 requests per hour per repository. Without a token, the limit is 60 requests per hour.
-- **No retry on API failure:** strategies make one attempt. A transient failure fails the package for that run.
+- **Retries:** a strategy request tries a timeout, a refused connection and an HTTP 408, 429, 500, 502, 503, 504, 522 or 524 answer twice more, 5 seconds apart. A check that still fails within its time budget runs once more after the other packages and a pause of 60 seconds. A failure on both tries fails the package for that run.
 - **Tag strategies take only `v1.2.3` and `1.2.3` by default:** without `tag_pattern`, and without `tag_version_regex` for `github-tag`, `github-tag`, `gitlab-tag` and `gitea-tag` leave out every other tag, `latest`, `1.0.0-rc1` and `1.0.0rc1` included. A `tag_pattern` replaces that filter. They pick the highest tag with `sort -V`, which places `1.0.0-rc1` above `1.0.0`, so a pattern that lets pre-release tags through picks them.
 - **`gitea-tag` fetches at most 50 tags:** there is no pagination.
 - **`gitlab-tag` has no authentication:** private GitLab projects cannot be queried.

docs/WORKFLOW.md

@@ -543,6 +543,8 @@ strategy: kde-tarball
 
 Outside `--update`, `check-all.sh` gives each package a time budget: 120 seconds for `check-update.sh`, and 300 seconds each for `update-pkg.sh`, `verify-pkg.sh` and `publish-pkg.sh`. `timeout` ends the script together with everything it started, makepkg, git and curl included, and the package gets `check_failed: timed out after 120 s` or `update_failed: timed out after 300 s`. `AURTOMATOR_CHECK_TIMEOUT` and `AURTOMATOR_UPDATE_TIMEOUT` change the budgets. `--update` has none, because it runs on a terminal, where git and ssh may ask for a passphrase, and a command under `timeout` cannot read the terminal.
 
+Without a mode and with `--update`, a package whose check failed within its budget is checked once more after all the others, following a pause of 60 seconds, and the second result counts. A check that ran out of its budget is not repeated, so a run without network does not take twice as long. `AURTOMATOR_RECHECK_DELAY` changes the pause, and the tests set it to 0.
+
 ### 4.1 Arguments
 
 ```text
@@ -931,7 +933,7 @@ A strategy is a bash script in `strategies/` that finds the latest upstream vers
 - **Failure:** an error on stderr, a non-zero exit code.
 - **Version format:** `check-update.sh` rejects a version with any character outside `A-Za-z0-9._+`.
 
-The script can rely on `yq` and `curl`. `scripts/lib.sh` provides YAML helpers and logging and sets `AURTOMATOR_DIR` to the project root.
+The script can rely on `yq` and `curl`. `scripts/lib.sh` provides YAML helpers, `http_get` for requests and logging, and sets `AURTOMATOR_DIR` to the project root.
 
 ### 8.2 Template
 
@@ -948,16 +950,14 @@ set -euo pipefail
 readonly NAME="${1:?Usage: my-strategy.sh <package-name>}"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly SCRIPT_DIR
-readonly TIMEOUT=30
 
 # shellcheck source=../scripts/lib.sh
 source "${SCRIPT_DIR}/../scripts/lib.sh"
 
 pkg_file=$(pkg_file "$NAME")
 my_field=$(pkg_get "$pkg_file" .upstream.my_field)
 
-version=$(curl -sfL --max-time "$TIMEOUT" \
-  "https://example.com/api/${my_field}" |
+version=$(http_get "https://example.com/api/${my_field}" |
   yq -r '.version') || {
   echo "Failed to fetch version for ${my_field}" >&2
   exit 1
@@ -975,7 +975,7 @@ echo "$version"
 
 1. **Location:** `strategies/<name>.sh`, executable (`chmod +x`). `<name>` is the value of the YAML `strategy` field.
 2. **Header:** `set -euo pipefail`, then source `lib.sh` for `pkg_file`, `pkg_get` and logging.
-3. **HTTP:** `curl -sfL --max-time "$TIMEOUT"`. `-f` fails on HTTP errors, `-L` follows redirects, and `--max-time` prevents hangs.
+3. **HTTP:** `http_get <url> [curl option]...` from `lib.sh`. It runs `curl -sfL --max-time 30` and tries a timeout, a refused connection and an HTTP 408, 429, 500, 502, 503, 504, 522 or 524 answer twice more, 5 seconds apart. `-f` fails on HTTP errors and `-L` follows redirects. A test in `tests/strategies.bats` fails when a strategy calls `curl` itself.
 4. **Result:** check for an empty string and for `null` from `yq`.
 5. **Prefix:** strip a leading `v` with `${version#v}` when upstream tags look like `v1.0.0`.
 6. **Validation:** add the name and its required fields to `scripts/validate-pkg.sh`.

scripts/check-all.sh

@@ -34,6 +34,9 @@ DRY_RUN=""
 CHECK_TIMEOUT="${AURTOMATOR_CHECK_TIMEOUT:-120}"
 UPDATE_TIMEOUT="${AURTOMATOR_UPDATE_TIMEOUT:-300}"
 
+# Pause, in seconds, before a package whose check failed is checked again
+RECHECK_DELAY="${AURTOMATOR_RECHECK_DELAY:-60}"
+
 # Runs a per-package script within a budget. timeout runs it in a process
 # group of its own and ends the whole group, makepkg, git and curl below it
 # included. Such a group cannot read the terminal, where git and ssh may ask
@@ -176,7 +179,6 @@ handed_failure() {
 
 updated=0
 checked=0
-errors=0
 
 # The version the check run found for a package, printed the way
 # check-update.sh prints it: nothing when up to date, an error on failure.
@@ -199,9 +201,14 @@ checked_version() {
   esac
 }
 
-for pkg_file in $(printf '%s\n' "${AURTOMATOR_DIR}"/packages/*.yml | sort); do
-  [[ -f "$pkg_file" ]] || continue
-
+# Checks one package, then updates, prepares or publishes it as the mode
+# says, and writes its status. A check that fails fast in a mode that runs
+# the strategies goes on the list for the second pass.
+# Args: $1=package file, $2=pass, 1 or 2
+run_package() {
+  local pkg_file="$1" pass="$2"
+  local pkg_name found check_cmd err_file check_rc latest check_err current
+  local is_downgrade cmp available pushed cmd update_rc update_err
   pkg_name=$(pkg_get "$pkg_file" .name)
   # --publish keeps the status of the check run and publishes only a package
   # that run found outdated. Only package names from packages/ reach a path,
@@ -212,28 +219,28 @@ for pkg_file in $(printf '%s\n' "${AURTOMATOR_DIR}"/packages/*.yml | sort); do
       "new_version: "* | "available_downgrade: "*)
         if ! validate_pkgver "${found#*: }" 2>/dev/null; then
           echo "check_failed: invalid status from the check run" >"${STATUS_DIR}/${pkg_name}"
-          continue
+          return
         fi
         if [[ -L "${WORK_DIR}/prepared" || ! -d "${WORK_DIR}/prepared/${pkg_name}" ]]; then
           handed_failure "$pkg_name" >"${STATUS_DIR}/${pkg_name}"
-          continue
+          return
         fi
         ;;
       up_to_date | check_failed:* | "downgrade_skipped: "*)
         printf '%s\n' "$found" >"${STATUS_DIR}/${pkg_name}"
-        continue
+        return
         ;;
       "")
         echo "check_failed: not checked before --publish" >"${STATUS_DIR}/${pkg_name}"
-        continue
+        return
         ;;
       *)
         echo "check_failed: invalid status from the check run" >"${STATUS_DIR}/${pkg_name}"
-        continue
+        return
         ;;
     esac
   fi
-  ((checked++)) || true
+  ((pass == 2)) || ((checked++)) || true
 
   if [[ "$MODE" == "prepare" || "$MODE" == "publish" ]]; then
     check_cmd=(checked_version "$pkg_name")
@@ -250,14 +257,17 @@ for pkg_file in $(printf '%s\n' "${AURTOMATOR_DIR}"/packages/*.yml | sort); do
     rm -f "$err_file"
     log_err "Failed to check $pkg_name: $check_err"
     echo "check_failed: ${check_err}" >"${STATUS_DIR}/${pkg_name}"
-    ((errors++)) || true
-    continue
+    if ((pass == 1)) && [[ "${check_cmd[0]}" == bounded ]] &&
+      ((check_rc != 124 && check_rc != 137)); then
+      RECHECK+=("$pkg_file")
+    fi
+    return
   fi
   rm -f "$err_file"
 
   if [[ -z "$latest" ]]; then
     echo "up_to_date" >"${STATUS_DIR}/${pkg_name}"
-    continue
+    return
   fi
 
   # Detect downgrade: current from YAML is newer than latest from upstream.
@@ -278,7 +288,7 @@ for pkg_file in $(printf '%s\n' "${AURTOMATOR_DIR}"/packages/*.yml | sort); do
     [[ "$(pkg_get "$pkg_file" '.allow_downgrade // false')" != "true" ]]; then
     log_warn "$pkg_name: DOWNGRADE detected $current → $latest, skipped: allow_downgrade is not set"
     echo "downgrade_skipped: ${latest}" >"${STATUS_DIR}/${pkg_name}"
-    continue
+    return
   fi
 
   if [[ "$is_downgrade" == "true" ]]; then
@@ -293,7 +303,7 @@ for pkg_file in $(printf '%s\n' "${AURTOMATOR_DIR}"/packages/*.yml | sort); do
   case "$MODE" in
     check)
       echo "$available" >"${STATUS_DIR}/${pkg_name}"
-      continue
+      return
       ;;
     update)
       log_info "Updating $pkg_name to $latest"
@@ -335,22 +345,37 @@ for pkg_file in $(printf '%s\n' "${AURTOMATOR_DIR}"/packages/*.yml | sort); do
     *)
       log_err "Failed to update $pkg_name to $latest"
       echo "update_failed: ${update_err}" >"${STATUS_DIR}/${pkg_name}"
-      ((errors++)) || true
       ;;
   esac
+}
+
+RECHECK=()
+for pkg_file in $(printf '%s\n' "${AURTOMATOR_DIR}"/packages/*.yml | sort); do
+  [[ -f "$pkg_file" ]] || continue
+  run_package "$pkg_file" 1
 done
 
-# --publish counts the failures among the statuses it wrote
-if [[ "$MODE" == "publish" ]]; then
-  errors=0
-  for status_file in "$STATUS_DIR"/*; do
-    [[ -f "$status_file" ]] || continue
-    case "$(<"$status_file")" in
-      check_failed:* | update_failed:*) ((errors++)) || true ;;
-    esac
+# A check that failed fast gets a second try after a pause, by when a
+# failure of the upstream host can be over. A check that ran out of its
+# budget gets none, so a run without network stays within the time limit
+# of its job.
+if ((${#RECHECK[@]} > 0)); then
+  log_info "Checking ${#RECHECK[@]} failed package(s) again in ${RECHECK_DELAY} s"
+  sleep "$RECHECK_DELAY"
+  for pkg_file in "${RECHECK[@]}"; do
+    run_package "$pkg_file" 2
   done
 fi
 
+# Count the failures among the statuses written
+errors=0
+for status_file in "$STATUS_DIR"/*; do
+  [[ -f "$status_file" ]] || continue
+  case "$(<"$status_file")" in
+    check_failed:* | update_failed:*) ((errors++)) || true ;;
+  esac
+done
+
 echo ""
 log_info "Checked: $checked, Updated: $updated, Errors: $errors"
 

scripts/lib.sh

@@ -79,9 +79,16 @@ warn_maintainer_line() {
 }
 
 # =============================================================================
-# GITHUB API
+# HTTP
 # =============================================================================
 
+# GET a URL for a strategy. curl tries a timeout, a refused connection and
+# an HTTP 408, 429, 500, 502, 503, 504, 522 or 524 answer twice more.
+# Usage: http_get <url> [curl option]...
+http_get() {
+  curl -sfL --max-time 30 --retry 2 --retry-delay 5 --retry-connrefused "${@:2}" "$1"
+}
+
 # GET a GitHub API URL, with GITHUB_TOKEN when it is set. curl reads the
 # header from stdin: as an argument, the token is visible in ps to every
 # local user.
@@ -91,7 +98,7 @@ github_api() {
   if [[ -n "${GITHUB_TOKEN:-}" ]]; then
     auth=(-H @-)
   fi
-  curl -sfL --max-time 30 "${auth[@]}" "$1" <<<"Authorization: Bearer ${GITHUB_TOKEN:-}"
+  http_get "$1" "${auth[@]}" <<<"Authorization: Bearer ${GITHUB_TOKEN:-}"
 }
 
 # =============================================================================

strategies/archpkg.sh

@@ -14,16 +14,14 @@ set -euo pipefail
 readonly NAME="${1:?Usage: archpkg.sh <package-name>}"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly SCRIPT_DIR
-readonly TIMEOUT=30
 
 # shellcheck source=../scripts/lib.sh
 source "${SCRIPT_DIR}/../scripts/lib.sh"
 
 pkg_file=$(pkg_file "$NAME")
 archpkg_name=$(pkg_get "$pkg_file" '.upstream.archpkg // .name')
 
-result=$(curl -sfL --max-time "$TIMEOUT" \
-  "https://archlinux.org/packages/search/json/?name=${archpkg_name}" |
+result=$(http_get "https://archlinux.org/packages/search/json/?name=${archpkg_name}" |
   yq -r '.results[0]') || {
   echo "Failed to fetch Arch package info for ${archpkg_name}" >&2
   exit 1

strategies/crates.sh

@@ -12,17 +12,15 @@ set -euo pipefail
 readonly NAME="${1:?Usage: crates.sh <package-name>}"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly SCRIPT_DIR
-readonly TIMEOUT=30
 
 # shellcheck source=../scripts/lib.sh
 source "${SCRIPT_DIR}/../scripts/lib.sh"
 
 pkg_file=$(pkg_file "$NAME")
 registry=$(pkg_get "$pkg_file" .upstream.registry)
 
-version=$(curl -sfL --max-time "$TIMEOUT" \
-  -H "User-Agent: aurtomator/1.0" \
-  "https://crates.io/api/v1/crates/${registry}" |
+version=$(http_get "https://crates.io/api/v1/crates/${registry}" \
+  -H "User-Agent: aurtomator/1.0" |
   yq -r '.crate.max_stable_version') || {
   echo "Failed to fetch crates.io info for ${registry}" >&2
   exit 1

strategies/gitea-tag.sh

@@ -12,7 +12,6 @@ set -euo pipefail
 readonly NAME="${1:?Usage: gitea-tag.sh <package-name>}"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly SCRIPT_DIR
-readonly TIMEOUT=30
 
 # shellcheck source=../scripts/lib.sh
 source "${SCRIPT_DIR}/../scripts/lib.sh"
@@ -22,8 +21,7 @@ host=$(pkg_get "$pkg_file" .upstream.host)
 owner_repo=$(pkg_get "$pkg_file" .upstream.project)
 tag_pattern=$(pkg_get "$pkg_file" '.upstream.tag_pattern // ""')
 
-tags=$(curl -sfL --max-time "$TIMEOUT" \
-  "https://${host}/api/v1/repos/${owner_repo}/tags?limit=50" |
+tags=$(http_get "https://${host}/api/v1/repos/${owner_repo}/tags?limit=50" |
   yq -r '.[].name') || {
   echo "Failed to fetch tags from ${host}/${owner_repo}" >&2
   exit 1

strategies/gitlab-tag.sh

@@ -12,7 +12,6 @@ set -euo pipefail
 readonly NAME="${1:?Usage: gitlab-tag.sh <package-name>}"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly SCRIPT_DIR
-readonly TIMEOUT=30
 
 # shellcheck source=../scripts/lib.sh
 source "${SCRIPT_DIR}/../scripts/lib.sh"
@@ -25,8 +24,7 @@ tag_pattern=$(pkg_get "$pkg_file" '.upstream.tag_pattern // ""')
 # URL-encode the project path (e.g., group/project → group%2Fproject)
 project_encoded="${project//\//%2F}"
 
-tags=$(curl -sfL --max-time "$TIMEOUT" \
-  "https://${host}/api/v4/projects/${project_encoded}/repository/tags?per_page=100" |
+tags=$(http_get "https://${host}/api/v4/projects/${project_encoded}/repository/tags?per_page=100" |
   yq -r '.[].name') || {
   echo "Failed to fetch tags from ${host}/${project}" >&2
   exit 1

strategies/kde-tarball.sh

@@ -11,9 +11,13 @@ set -euo pipefail
 
 readonly NAME="${1:?Usage: kde-tarball.sh <package-name>}"
 readonly URL="https://download.kde.org/stable/${NAME}/"
-readonly TIMEOUT=30
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly SCRIPT_DIR
 
-page=$(curl -sfL --max-time "$TIMEOUT" "$URL") || {
+# shellcheck source=../scripts/lib.sh
+source "${SCRIPT_DIR}/../scripts/lib.sh"
+
+page=$(http_get "$URL") || {
   echo "Failed to fetch $URL" >&2
   exit 1
 }

strategies/npm.sh

@@ -12,16 +12,14 @@ set -euo pipefail
 readonly NAME="${1:?Usage: npm.sh <package-name>}"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly SCRIPT_DIR
-readonly TIMEOUT=30
 
 # shellcheck source=../scripts/lib.sh
 source "${SCRIPT_DIR}/../scripts/lib.sh"
 
 pkg_file=$(pkg_file "$NAME")
 registry=$(pkg_get "$pkg_file" .upstream.registry)
 
-version=$(curl -sfL --max-time "$TIMEOUT" \
-  "https://registry.npmjs.org/${registry}/latest" |
+version=$(http_get "https://registry.npmjs.org/${registry}/latest" |
   yq -r '.version') || {
   echo "Failed to fetch npm info for ${registry}" >&2
   exit 1

strategies/pypi.sh

@@ -12,16 +12,14 @@ set -euo pipefail
 readonly NAME="${1:?Usage: pypi.sh <package-name>}"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly SCRIPT_DIR
-readonly TIMEOUT=30
 
 # shellcheck source=../scripts/lib.sh
 source "${SCRIPT_DIR}/../scripts/lib.sh"
 
 pkg_file=$(pkg_file "$NAME")
 registry=$(pkg_get "$pkg_file" .upstream.registry)
 
-version=$(curl -sfL --max-time "$TIMEOUT" \
-  "https://pypi.org/pypi/${registry}/json" |
+version=$(http_get "https://pypi.org/pypi/${registry}/json" |
   yq -r '.info.version') || {
   echo "Failed to fetch PyPI info for ${registry}" >&2
   exit 1

strategies/repology.sh

@@ -12,7 +12,6 @@ set -euo pipefail
 readonly NAME="${1:?Usage: repology.sh <package-name>}"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly SCRIPT_DIR
-readonly TIMEOUT=30
 
 # shellcheck source=../scripts/lib.sh
 source "${SCRIPT_DIR}/../scripts/lib.sh"
@@ -22,9 +21,8 @@ repology_name=$(pkg_get "$pkg_file" '.upstream.repology // .name')
 
 # Repology returns an array of package entries from various repos.
 # Filter for "newest" status to get the latest upstream version.
-version=$(curl -sfL --max-time "$TIMEOUT" \
-  -H "User-Agent: aurtomator/1.0" \
-  "https://repology.org/api/v1/project/${repology_name}" |
+version=$(http_get "https://repology.org/api/v1/project/${repology_name}" \
+  -H "User-Agent: aurtomator/1.0" |
   yq -r '[.[] | select(.status == "newest")] | .[0].version') || {
   echo "Failed to fetch Repology info for ${repology_name}" >&2
   exit 1