From 8951afbfe2a563710697d23d199b3e55c79f9af0 Mon Sep 17 00:00:00 2001 From: Andres Cera Date: Fri, 21 Aug 2026 18:46:46 -0500 Subject: [PATCH 1/7] feat(packaging): per-source change detection --- packaging/ci/detect-changed-sources.sh | 358 ++++++++++++++++++++ packaging/ci/test-detect-changed-sources.sh | 290 ++++++++++++++++ 2 files changed, 648 insertions(+) create mode 100755 packaging/ci/detect-changed-sources.sh create mode 100755 packaging/ci/test-detect-changed-sources.sh diff --git a/packaging/ci/detect-changed-sources.sh b/packaging/ci/detect-changed-sources.sh new file mode 100755 index 0000000..89284eb --- /dev/null +++ b/packaging/ci/detect-changed-sources.sh @@ -0,0 +1,358 @@ +#!/usr/bin/env bash +# detect-changed-sources.sh — which upstream packaging sources actually changed since the +# previous release, and therefore need rebuilding. +# +# WHAT THIS IS +# The FIRST script of the differential-release pipeline. A `vX.Y.Z` release should rebuild +# only the sources whose inputs moved; every other source's .debs are carried forward +# byte-identically from the release that last built them (builds are NOT reproducible, so an +# unchanged source must reuse recorded bytes and can never be rebuilt at its old version). +# This script makes ONLY the verdict; staging, building and manifest generation are elsewhere. +# +# OUTPUT (stdout — the machine contract; consumers grep these exact lines) +# libqrtr-glib=changed|unchanged +# libmbim=changed|unchanged +# libqmi=changed|unchanged +# modemmanager=changed|unchanged +# mode=differential|force-all +# +# Source names are the upstream-pins.yaml PIN KEYS (lowercase `modemmanager`), not the +# packaging directory names (`ModemManager`) — the mapping is NON-IDENTITY and is the same +# one build-bookworm.sh's `pin_key()` and read-pin.sh's `recipe_dir()` carry. Lines are +# emitted in BOOTSTRAP ORDER, so a consumer can build the selected set by reading top to +# bottom. Every human-readable reason goes to STDERR, so stdout stays parseable. +# +# THE VERDICT +# A source is `changed` when the diff `..HEAD` touched either +# * `packaging//**` — its checked-in debian/ recipe, or +# * that source's own block in `packaging/upstream-pins.yaml` — compared BLOCK-SCOPED, so a +# comment edit or a neighbouring source's pin bump does not implicate it. +# +# FORCE-ALL (every source `changed`, `mode=force-all`, reason logged) +# 1. A SHARED INPUT changed — `packaging/ci/**` (which subsumes `ci/expected-packages.txt`) +# or `packaging/BOOKWORM-ADAPTATIONS.md`. These feed every source's build, so a change to +# one of them invalidates every carried artifact. +# 2. The previous release is ABSENT, or carries no manifest asset. With no manifest there is +# nothing to carry forward from, so rebuilding everything is the only honest answer. +# 3. The previous manifest is V1-SHAPED — it carries no `closure_version:` header. An ABSENT +# header IS closure version 1 (that default is apt-worker's backward-compatibility +# mechanism, `modem_manifest_closure_version`), and a v1 manifest predates the companion +# row this pipeline depends on. +# 4. `FORCE_REBUILD=all` — the operator's escape hatch. +# Fail-SAFE here means REBUILD EVERYTHING, never "assume unchanged": a wrong `unchanged` +# ships stale bytes, a wrong `changed` only costs build time. +# +# THE COMPANION IS NOT PART OF DETECTION +# `ceralive-modem-support` is never enumerated and never verdicted — it ALWAYS rebuilds +# (apt-worker's health check greps `^Version: $`, which only the companion's bare +# version can satisfy). A companion-only change therefore leaves all four sources +# `unchanged`; that is correct, not a miss. +# +# PREVIOUS-RELEASE RESOLUTION — `gh release list`, NEVER `git describe` +# `git describe` answers "nearest tag in this history", which is not the same question: a tag +# can exist with no release, a release can be a draft or a pre-release, and a release can +# carry no manifest asset. The previous release is the latest PUBLISHED release, so it is +# resolved through `gh`. `PREV_TAG` overrides the resolution outright. +# +# SEAMS (all optional; they exist so the contract test runs offline, with no gh and no network) +# PREV_TAG Use this tag as the previous release; skips `gh release list`. +# PREV_MANIFEST_FILE Read the previous release manifest from this local path; skips +# `gh release download`. Set-but-unreadable is a hard error (exit 2), not +# a force-all — a named seam pointing at nothing is operator error, and +# silently force-alling would hide it. +# FORCE_REBUILD=all Force-all (rule 4). +# HEAD_REF The current side of the diff (default `HEAD`). +# GH_REPO Passed to `gh --repo` when resolving/downloading (default: gh's own +# repo inference from the checkout). +# +# USAGE +# detect-changed-sources.sh [--out ] +# --out additionally write the stdout contract lines to (for a later job +# step to source); the file is written atomically-enough for CI (truncate+write). +# +# EXIT +# 0 verdicts printed (differential OR force-all — force-all is a normal outcome, not an error). +# 2 usage / environment error: not a git repo, unreadable pin manifest, unresolvable HEAD ref, +# a set-but-unreadable PREV_MANIFEST_FILE, or a pinned-source set that no longer matches the +# declared bootstrap order. Every one names the offending field. +set -euo pipefail + +LOG_PREFIX="detect-changed-sources" +log() { printf '%s: %s\n' "$LOG_PREFIX" "$*" >&2; } +die() { + printf '%s: %s\n' "$LOG_PREFIX" "$*" >&2 + exit 2 +} + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PKG_ROOT="$(cd "$HERE/.." && pwd)" + +# ---- arguments ----------------------------------------------------------------------------- +OUT_FILE="" +while [ $# -gt 0 ]; do + case "$1" in + --out) + shift + [ $# -gt 0 ] || die "--out requires a file path" + OUT_FILE="$1" + ;; + --out=*) OUT_FILE="${1#--out=}" ;; + -h | --help) + echo "usage: detect-changed-sources.sh [--out ]" + exit 0 + ;; + *) die "unknown argument '$1' (usage: detect-changed-sources.sh [--out ])" ;; + esac + shift +done + +# ---- repository + packaging paths ---------------------------------------------------------- +PINS="$PKG_ROOT/upstream-pins.yaml" +[ -r "$PINS" ] || die "cannot read pin manifest '$PINS'" + +repo_top="$(git -C "$PKG_ROOT" rev-parse --show-toplevel 2>/dev/null)" || + die "packaging root '$PKG_ROOT' is not inside a git repository (a diff needs one)" +REPO_ROOT="$(cd "$repo_top" && pwd)" +cd "$REPO_ROOT" + +# Repo-RELATIVE packaging prefix, derived rather than hardcoded, so the same script works from +# any checkout layout (and from the contract test's throwaway fixture repo). +PKG_REL="${PKG_ROOT#"$REPO_ROOT"/}" +[ "$PKG_REL" != "$PKG_ROOT" ] || die "packaging root '$PKG_ROOT' is not under repository root '$REPO_ROOT'" +PINS_REL="$PKG_REL/upstream-pins.yaml" + +# ---- contract constants (mirrored from build-bookworm.sh:41,46) ----------------------------- +# Bootstrap order — the order a differential build must walk the selected set in. +BUILD_ORDER=(libqrtr-glib libmbim libqmi ModemManager) +# packaging dir -> upstream-pins.yaml source key. NON-IDENTITY: only ModemManager differs. +pin_key() { case "$1" in ModemManager) echo modemmanager ;; *) echo "$1" ;; esac; } + +# The list of source names under `sources:` — byte-identical to read-pin.sh's / +# verify-upstream-pins.sh's reader, deliberately the SAME parser rather than a second one. +yaml_sources() { + awk ' + /^sources:[ \t]*$/ { ins=1; next } + ins && /^[^ ]/ { ins=0 } + ins && /^ [^ ]+:[ \t]*$/ { s=$0; sub(/^ /, "", s); sub(/:[ \t]*$/, "", s); print s } + ' "$PINS" +} + +# One source's block of the pin manifest, read from stdin. Block-scoped so a comment edit or a +# neighbouring source's bump cannot implicate this source. +pins_block() { # (manifest text on stdin) + awk -v src="$1" ' + $0 ~ "^ " src ":[ \t]*$" { inblk=1; print; next } + inblk && /^ [^ ]/ { inblk=0 } + inblk && /^[^ ]/ { inblk=0 } + inblk { print } + ' +} + +# The pinned set and the declared bootstrap set must agree. A fifth pinned source added without +# updating this detector would otherwise be silently un-detected — i.e. carried forward forever. +pinned_sorted="$(yaml_sources | LC_ALL=C sort)" +declared_sorted="$(for d in "${BUILD_ORDER[@]}"; do pin_key "$d"; done | LC_ALL=C sort)" +if [ "$pinned_sorted" != "$declared_sorted" ]; then + die "pinned source set in '$PINS_REL' != declared BUILD_ORDER set — pinned: [$(echo "$pinned_sorted" | tr '\n' ' ')] declared: [$(echo "$declared_sorted" | tr '\n' ' ')]" +fi + +# ---- shared-input classifier --------------------------------------------------------------- +# `packaging/ci/**` subsumes ci/expected-packages.txt; both are named in the header on purpose. +is_shared_input() { + case "$1" in + "$PKG_REL"/ci/*) return 0 ;; + "$PKG_REL"/BOOKWORM-ADAPTATIONS.md) return 0 ;; + esac + return 1 +} + +# ---- resolution ---------------------------------------------------------------------------- +FORCED=0 +FORCE_REASON="" +PREV_TAG_RESOLVED="" +HEAD_REF_RESOLVED="${HEAD_REF:-HEAD}" +CHANGED_FILES="" +MANIFEST_TMP="" + +cleanup() { [ -n "$MANIFEST_TMP" ] && rm -rf "$MANIFEST_TMP"; return 0; } +trap cleanup EXIT + +force() { + FORCED=1 + FORCE_REASON="$*" +} + +gh_args() { + if [ -n "${GH_REPO:-}" ]; then printf '%s\n%s\n' "--repo" "$GH_REPO"; fi +} + +resolve() { + # (4) Operator override, checked first so it needs no network and no previous release. + if [ "${FORCE_REBUILD:-}" = "all" ]; then + force "FORCE_REBUILD=all — operator override" + return + fi + if [ -n "${FORCE_REBUILD:-}" ]; then + die "FORCE_REBUILD='$FORCE_REBUILD' is not a recognized value (the only accepted value is 'all')" + fi + + # --- previous release tag --------------------------------------------------------------- + if [ -n "${PREV_TAG:-}" ]; then + PREV_TAG_RESOLVED="$PREV_TAG" + log "previous release tag: $PREV_TAG_RESOLVED (PREV_TAG seam)" + else + if ! command -v gh >/dev/null 2>&1; then + force "previous-release-unresolved — PREV_TAG is unset and 'gh' is not installed (the previous release is the latest PUBLISHED release, resolved via 'gh release list'; 'git describe' is never used)" + return + fi + local ghargs=() tag="" + mapfile -t ghargs < <(gh_args) + if ! tag="$(gh release list "${ghargs[@]}" --limit 1 --exclude-drafts --exclude-pre-releases --json tagName --jq '.[0].tagName' 2>/dev/null)"; then + force "previous-release-unresolved — 'gh release list' failed" + return + fi + tag="$(printf '%s' "$tag" | tr -d '[:space:]')" + if [ -z "$tag" ] || [ "$tag" = "null" ]; then + force "previous-release-absent — 'gh release list' reports no published release to diff against" + return + fi + PREV_TAG_RESOLVED="$tag" + log "previous release tag: $PREV_TAG_RESOLVED (gh release list)" + fi + + # --- previous release manifest ------------------------------------------------------------ + local manifest="" + if [ -n "${PREV_MANIFEST_FILE:-}" ]; then + [ -r "$PREV_MANIFEST_FILE" ] || + die "PREV_MANIFEST_FILE='$PREV_MANIFEST_FILE' is set but not readable (a named seam pointing at nothing is operator error, not a force-all)" + manifest="$PREV_MANIFEST_FILE" + log "previous release manifest: $manifest (PREV_MANIFEST_FILE seam)" + else + if ! command -v gh >/dev/null 2>&1; then + force "previous-manifest-absent — 'gh' is not installed, so release '$PREV_TAG_RESOLVED' manifest cannot be fetched" + return + fi + local ghargs=() + mapfile -t ghargs < <(gh_args) + MANIFEST_TMP="$(mktemp -d "${TMPDIR:-/tmp}/detect-changed-sources.XXXXXX")" + if ! gh release download "$PREV_TAG_RESOLVED" "${ghargs[@]}" --pattern 'release-manifest*.txt' --dir "$MANIFEST_TMP" >/dev/null 2>&1; then + force "previous-manifest-absent — release '$PREV_TAG_RESOLVED' has no downloadable release-manifest asset" + return + fi + manifest="$(find "$MANIFEST_TMP" -maxdepth 1 -type f -name 'release-manifest*.txt' | LC_ALL=C sort | head -n1)" + if [ -z "$manifest" ]; then + force "previous-manifest-absent — 'gh release download' produced no release-manifest*.txt for '$PREV_TAG_RESOLVED'" + return + fi + log "previous release manifest: $manifest (gh release download from $PREV_TAG_RESOLVED)" + fi + + # (3) An ABSENT `closure_version:` header IS closure version 1 — apt-worker's + # `modem_manifest_closure_version` treats it exactly that way, and that default is the + # backward-compatibility mechanism for every pre-header release. + if ! grep -q '^closure_version:' "$manifest"; then + force "previous-manifest-v1-shaped — '$manifest' carries no 'closure_version:' header (an absent header IS closure version 1), so nothing may be carried forward from it" + return + fi + + # --- the diff ----------------------------------------------------------------------------- + git rev-parse --verify --quiet "$HEAD_REF_RESOLVED^{commit}" >/dev/null || + die "HEAD ref '$HEAD_REF_RESOLVED' does not resolve to a commit in '$REPO_ROOT'" + if ! git rev-parse --verify --quiet "$PREV_TAG_RESOLVED^{commit}" >/dev/null; then + force "previous-ref-unresolvable — '$PREV_TAG_RESOLVED' does not resolve to a commit in this checkout (fetch the tag, or the diff cannot be taken)" + return + fi + if ! CHANGED_FILES="$(git diff --name-only "$PREV_TAG_RESOLVED..$HEAD_REF_RESOLVED")"; then + force "diff-unavailable — 'git diff --name-only $PREV_TAG_RESOLVED..$HEAD_REF_RESOLVED' failed" + return + fi + log "diff $PREV_TAG_RESOLVED..$HEAD_REF_RESOLVED touched $(printf '%s' "$CHANGED_FILES" | grep -c . || true) file(s)" + + # (1) Shared inputs feed every source's build; one of them moving invalidates every + # carried artifact, so it is a force-all rather than a per-source verdict. + local f + while IFS= read -r f; do + [ -n "$f" ] || continue + if is_shared_input "$f"; then + force "shared-input-changed — '$f' is a SHARED build input (packaging/ci/** or packaging/BOOKWORM-ADAPTATIONS.md); it feeds every source, so no source may be carried forward" + return + fi + done <<<"$CHANGED_FILES" +} + +resolve + +# ---- per-source verdicts -------------------------------------------------------------------- +declare -A VERDICT=() + +pins_block_changed() { # -> 0 when this source's pin block differs prev..HEAD + local key="$1" prev="" cur="" + prev="$(git show "$PREV_TAG_RESOLVED:$PINS_REL" 2>/dev/null | pins_block "$key")" || prev="" + cur="$(git show "$HEAD_REF_RESOLVED:$PINS_REL" 2>/dev/null | pins_block "$key")" || cur="" + [ "$prev" != "$cur" ] +} + +if [ "$FORCED" -eq 1 ]; then + for dir in "${BUILD_ORDER[@]}"; do VERDICT["$(pin_key "$dir")"]=changed; done + log "mode=force-all — $FORCE_REASON" + log "every source rebuilds; nothing is carried forward" +else + pins_touched=0 + while IFS= read -r f; do + if [ "$f" = "$PINS_REL" ]; then pins_touched=1; fi + done <<<"$CHANGED_FILES" + + for dir in "${BUILD_ORDER[@]}"; do + key="$(pin_key "$dir")" + verdict=unchanged + why="" + while IFS= read -r f; do + [ -n "$f" ] || continue + case "$f" in + "$PKG_REL/$dir"/*) + verdict=changed + why="recipe '$f'" + break + ;; + esac + done <<<"$CHANGED_FILES" + + if [ "$verdict" = unchanged ] && [ "$pins_touched" -eq 1 ] && pins_block_changed "$key"; then + verdict=changed + why="pin block [$key] in '$PINS_REL'" + fi + + VERDICT["$key"]="$verdict" + if [ "$verdict" = changed ]; then + log " $key=changed ($why)" + else + log " $key=unchanged (no recipe change under $PKG_REL/$dir/, no [$key] pin-block change)" + fi + done + + changed_n=0 + for dir in "${BUILD_ORDER[@]}"; do + if [ "${VERDICT["$(pin_key "$dir")"]}" = changed ]; then changed_n=$((changed_n + 1)); fi + done + log "mode=differential — $changed_n of ${#BUILD_ORDER[@]} source(s) changed since $PREV_TAG_RESOLVED; the rest carry forward" + log "the companion ceralive-modem-support is outside detection and always rebuilds" +fi + +MODE=differential +if [ "$FORCED" -eq 1 ]; then MODE=force-all; fi + +emit() { + local dir + for dir in "${BUILD_ORDER[@]}"; do + printf '%s=%s\n' "$(pin_key "$dir")" "${VERDICT["$(pin_key "$dir")"]}" + done + printf 'mode=%s\n' "$MODE" +} + +if [ -n "$OUT_FILE" ]; then + mkdir -p "$(dirname "$OUT_FILE")" + emit | tee "$OUT_FILE" + log "wrote verdicts to '$OUT_FILE'" +else + emit +fi diff --git a/packaging/ci/test-detect-changed-sources.sh b/packaging/ci/test-detect-changed-sources.sh new file mode 100755 index 0000000..d6b9445 --- /dev/null +++ b/packaging/ci/test-detect-changed-sources.sh @@ -0,0 +1,290 @@ +#!/usr/bin/env bash +# test-detect-changed-sources.sh — the change-detector's behaviour contract. +# +# HOST-RUNNABLE, OFFLINE, NO DOCKER. Two seams make that possible, and the choice of seam is +# deliberate in each case: +# +# 1. THE DIFF IS REAL. There is no stubbed `git diff` and no injected changed-file list. +# Each case runs inside a THROWAWAY GIT REPO built here — real commits, a real `v1.1.0` +# tag, real `git diff --name-only ..HEAD` and real `git show :` for the +# pin-block comparison. A stubbed file list would prove the classifier and leave the two +# things most likely to break — the diff invocation and the block-scoped pin comparison — +# completely uncovered. The fixture repo mirrors the real layout (`packaging/ci/`, +# `packaging//debian/`, `packaging/upstream-pins.yaml`) and carries a COPY of the +# REAL `upstream-pins.yaml`, so the detector's pinned-set-vs-BUILD_ORDER contract check +# runs against the real manifest. +# +# 2. `gh` IS STUBBED ON PATH. The previous-release resolution is `gh release list` / +# `gh release download` by contract (never `git describe`), and both are network calls. +# A stub dir is prepended to PATH; it shadows any real `gh`, answers per `STUB_GH`, and +# exits 97 on an invocation no case expects — so a test can never silently reach the +# network. The FORCE-ALL RULES THEMSELVES ARE NEVER STUBBED: case (d) forces because the +# manifest genuinely could not be fetched, and the non-vacuity control right after it +# proves the very same fixture goes `differential` once a manifest exists. +# +# Cases: (a) recipe change (b) pin-block change (c) shared input (d) no release / no manifest +# (e) v1-shaped manifest (f) FORCE_REBUILD=all + companion / non-packaging / no-change / +# block-scoping controls. +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PKG_ROOT="$(cd "$HERE/.." && pwd)" +SCRIPT="$HERE/detect-changed-sources.sh" +REAL_PINS="$PKG_ROOT/upstream-pins.yaml" + +[ -f "$SCRIPT" ] || { echo "missing: $SCRIPT" >&2; exit 1; } +[ -r "$REAL_PINS" ] || { echo "missing: $REAL_PINS" >&2; exit 1; } + +pass=0; fail=0 +ok() { printf ' ok %s\n' "$1"; pass=$((pass + 1)); } +bad() { printf ' FAIL %s\n' "$1"; fail=$((fail + 1)); } + +ROOT="$(mktemp -d "${TMPDIR:-/tmp}/ceralive-detect-changed.XXXXXX")" +trap 'rm -rf "$ROOT"' EXIT + +REPO="$ROOT/repo" +STUB_BIN="$ROOT/bin" +ERR="$ROOT/stderr.txt" +mkdir -p "$STUB_BIN" + +# ---- the offline `gh` stub ------------------------------------------------------------------- +cat >"$STUB_BIN/gh" <<'STUB' +#!/usr/bin/env bash +# Offline `gh` stub for test-detect-changed-sources.sh. Behaviour is selected by STUB_GH. +# Anything unexpected exits 97 so a case can never silently reach the real gh / the network. +set -uo pipefail +sub="${1:-}/${2:-}" +case "${STUB_GH:-unset}:${sub}" in + no-release:release/list) exit 0 ;; # a repo with no published release + have-release:release/list) echo "v1.1.0"; exit 0 ;; # a release exists ... + *:release/download) echo "gh(stub): no asset matched the pattern" >&2; exit 1 ;; + *) echo "gh(stub): unexpected invocation: $*" >&2; exit 97 ;; +esac +STUB +chmod +x "$STUB_BIN/gh" + +# ---- the throwaway fixture repo --------------------------------------------------------------- +mkdir -p "$REPO/packaging/ci" "$REPO/packaging/ceralive-modem-support/debian" "$REPO/control/src" +cp "$SCRIPT" "$REPO/packaging/ci/detect-changed-sources.sh" +cp "$REAL_PINS" "$REPO/packaging/upstream-pins.yaml" +printf 'shared build inputs live here\n' >"$REPO/packaging/BOOKWORM-ADAPTATIONS.md" +printf '# stand-in for the real builder\n' >"$REPO/packaging/ci/build-bookworm.sh" +printf '[libqmi all-artifact]\nlibqmi-glib5\n' >"$REPO/packaging/ci/expected-packages.txt" +printf 'Source: ceralive-modem-support\n' >"$REPO/packaging/ceralive-modem-support/debian/control" +printf 'export const x = 1;\n' >"$REPO/control/src/index.ts" +for d in libqrtr-glib libmbim libqmi ModemManager; do + mkdir -p "$REPO/packaging/$d/debian" + printf '#!/usr/bin/make -f\n%%:\n\tdh $@\n' >"$REPO/packaging/$d/debian/rules" + printf '%s (0.0.0-1) unstable; urgency=medium\n' "$d" >"$REPO/packaging/$d/debian/changelog" +done + +git -C "$REPO" init -q +git -C "$REPO" config user.email "ci@ceralive.tv" +git -C "$REPO" config user.name "CeraLive CI" +git -C "$REPO" config commit.gpgsign false +git -C "$REPO" add -A +git -C "$REPO" commit -q -m "fixture: release v1.1.0 state" +git -C "$REPO" tag v1.1.0 + +# ---- fixture previous-release manifests -------------------------------------------------------- +M_V2="$ROOT/release-manifest-v2.txt" +M_V1="$ROOT/release-manifest-v1.txt" +cat >"$M_V2" <<'EOF' +# CeraLive modem-stack release manifest (fixture) +tag: v1.1.0 +version: 1.1.0 +closure_version: 2 +amd64 libqmi-glib5 libqmi 1.38.0-1~ceralive1.1.0 runtime libqmi-glib5_1.38.0-1~ceralive1.1.0_amd64.deb 0000 +EOF +cat >"$M_V1" <<'EOF' +# CeraLive modem-stack release manifest (fixture) — v1 shape: NO closure_version header. +tag: v1.0.0 +version: 1.0.0 +amd64 libqmi-glib5 libqmi 1.38.0-1~ceralive1.0.0 runtime libqmi-glib5_1.38.0-1~ceralive1.0.0_amd64.deb 0000 +EOF + +# ---- harness ----------------------------------------------------------------------------------- +reset_repo() { + git -C "$REPO" reset --hard -q v1.1.0 + git -C "$REPO" clean -qfd +} +commit_all() { + git -C "$REPO" add -A + git -C "$REPO" commit -q -m "$1" +} +# Runs the detector inside the fixture repo with the stub `gh` shadowing any real one. +detect() { (cd "$REPO" && PATH="$STUB_BIN:$PATH" bash packaging/ci/detect-changed-sources.sh 2>"$ERR"); } + +assert_has() { if printf '%s\n' "$2" | grep -qxF "$1"; then ok "$3"; else bad "$3 — stdout has no '$1'"; fi; } +assert_err() { if grep -qE "$1" "$ERR"; then ok "$2"; else bad "$2 — stderr has no /$1/"; fi; } +assert_rc() { if [ "$1" -eq "$2" ]; then ok "$3"; else bad "$3 — exit $1, expected $2"; fi; } +# All four sources at one verdict, in one assertion. +assert_all() { + local want="$1" out="$2" label="$3" s + for s in libqrtr-glib libmbim libqmi modemmanager; do + if ! printf '%s\n' "$out" | grep -qxF "$s=$want"; then bad "$label — $s is not '$want'"; return; fi + done + ok "$label" +} + +echo "== (a) a recipe change implicates ONLY its own source ==" +reset_repo +printf '\n# a real recipe edit\n' >>"$REPO/packaging/libqmi/debian/rules" +commit_all "touch libqmi recipe" +out="$(PREV_TAG=v1.1.0 PREV_MANIFEST_FILE="$M_V2" detect)"; rc=$? +assert_rc "$rc" 0 "exit 0" +assert_has "libqmi=changed" "$out" "libqmi=changed" +assert_has "libmbim=unchanged" "$out" "libmbim=unchanged" +assert_has "modemmanager=unchanged" "$out" "modemmanager=unchanged" +assert_has "libqrtr-glib=unchanged" "$out" "libqrtr-glib=unchanged" +assert_has "mode=differential" "$out" "mode=differential" +assert_err "libqmi=changed .*recipe" "the reason names the changed recipe file" +# The stdout contract is exactly five lines — a consumer greps it, so stray output is a break. +if [ "$(printf '%s\n' "$out" | grep -c .)" -eq 5 ]; then ok "stdout is exactly 4 source lines + mode"; else bad "stdout is not 5 lines: $out"; fi + +echo "== (b) editing a source's pin block implicates ONLY that source ==" +reset_repo +sed -i 's|^ upstream_tag: "1.38.0"| upstream_tag: "1.38.1"|' "$REPO/packaging/upstream-pins.yaml" +if grep -q '1.38.1' "$REPO/packaging/upstream-pins.yaml"; then ok "fixture: libqmi pin block edited"; else bad "fixture: pin edit did not apply"; fi +commit_all "bump the libqmi pin" +out="$(PREV_TAG=v1.1.0 PREV_MANIFEST_FILE="$M_V2" detect)"; rc=$? +assert_rc "$rc" 0 "exit 0" +assert_has "libqmi=changed" "$out" "libqmi=changed" +assert_has "libmbim=unchanged" "$out" "libmbim=unchanged" +assert_has "modemmanager=unchanged" "$out" "modemmanager=unchanged" +assert_has "libqrtr-glib=unchanged" "$out" "libqrtr-glib=unchanged" +assert_has "mode=differential" "$out" "mode=differential" +assert_err "libqmi=changed .*pin block" "the reason names the pin block" + +echo "== (b-control) the pin comparison is BLOCK-SCOPED, not whole-file ==" +reset_repo +printf '\n# a comment outside every source block\n' >>"$REPO/packaging/upstream-pins.yaml" +commit_all "comment-only pins edit" +out="$(PREV_TAG=v1.1.0 PREV_MANIFEST_FILE="$M_V2" detect)"; rc=$? +assert_rc "$rc" 0 "exit 0" +assert_all unchanged "$out" "a comment-only pins edit implicates no source" +assert_has "mode=differential" "$out" "mode=differential" + +echo "== (b-control-2) the NON-IDENTITY dir->pin-key mapping is exercised ==" +reset_repo +printf '\n# a real recipe edit\n' >>"$REPO/packaging/ModemManager/debian/rules" +commit_all "touch ModemManager recipe" +out="$(PREV_TAG=v1.1.0 PREV_MANIFEST_FILE="$M_V2" detect)"; rc=$? +assert_rc "$rc" 0 "exit 0" +assert_has "modemmanager=changed" "$out" "dir 'ModemManager' reports as pin key 'modemmanager'" +if printf '%s\n' "$out" | grep -q '^ModemManager='; then bad "the directory name leaked into stdout"; else ok "no directory name leaked into stdout"; fi +assert_has "libqmi=unchanged" "$out" "libqmi=unchanged" + +echo "== (c) a shared build input forces every source ==" +for shared in packaging/ci/build-bookworm.sh packaging/ci/expected-packages.txt packaging/BOOKWORM-ADAPTATIONS.md; do + reset_repo + printf '\n# shared input edit\n' >>"$REPO/$shared" + commit_all "touch $shared" + out="$(PREV_TAG=v1.1.0 PREV_MANIFEST_FILE="$M_V2" detect)"; rc=$? + assert_rc "$rc" 0 "exit 0 ($shared)" + assert_has "mode=force-all" "$out" "mode=force-all ($shared)" + assert_all changed "$out" "every source changed ($shared)" + assert_err "shared-input-changed" "the reason is shared-input-changed ($shared)" +done + +echo "== (d) previous release absent, or carrying no manifest, forces every source ==" +reset_repo +printf '\n# a real recipe edit\n' >>"$REPO/packaging/libqmi/debian/rules" +commit_all "touch libqmi recipe" +# (d1) no published release at all — PREV_TAG unset, `gh release list` answers nothing. +out="$(STUB_GH=no-release detect)"; rc=$? +assert_rc "$rc" 0 "exit 0 (no published release)" +assert_has "mode=force-all" "$out" "mode=force-all (no published release)" +assert_all changed "$out" "every source changed (no published release)" +assert_err "previous-release-absent" "the reason is previous-release-absent" +# (d2) a release exists but carries no manifest asset — `gh release download` fails. +out="$(STUB_GH=have-release detect)"; rc=$? +assert_rc "$rc" 0 "exit 0 (release without a manifest asset)" +assert_has "mode=force-all" "$out" "mode=force-all (release without a manifest asset)" +assert_all changed "$out" "every source changed (release without a manifest asset)" +assert_err "previous-manifest-absent" "the reason is previous-manifest-absent" + +echo "== (d-non-vacuity) the SAME commit goes differential once a manifest exists ==" +# Nothing about the force-all rules is stubbed anywhere in (d); the only difference here is that +# a manifest is reachable. If this went force-all too, (d) would have proven nothing. +out="$(PREV_TAG=v1.1.0 PREV_MANIFEST_FILE="$M_V2" detect)"; rc=$? +assert_rc "$rc" 0 "exit 0" +assert_has "mode=differential" "$out" "mode=differential with a reachable v2 manifest" +assert_has "libqmi=changed" "$out" "libqmi=changed" +assert_has "libmbim=unchanged" "$out" "libmbim=unchanged" + +echo "== (e) a v1-shaped previous manifest (no closure_version header) forces every source ==" +out="$(PREV_TAG=v1.1.0 PREV_MANIFEST_FILE="$M_V1" detect)"; rc=$? +assert_rc "$rc" 0 "exit 0" +assert_has "mode=force-all" "$out" "mode=force-all" +assert_all changed "$out" "every source changed" +assert_err "previous-manifest-v1-shaped" "the reason is previous-manifest-v1-shaped" + +echo "== (f) FORCE_REBUILD=all forces every source with no release lookup at all ==" +# STUB_GH is deliberately left unset: the stub exits 97 on any call, so this case also proves +# the override short-circuits BEFORE any previous-release resolution. +out="$(FORCE_REBUILD=all detect)"; rc=$? +assert_rc "$rc" 0 "exit 0" +assert_has "mode=force-all" "$out" "mode=force-all" +assert_all changed "$out" "every source changed" +assert_err "FORCE_REBUILD=all" "the reason names the operator override" +out="$(FORCE_REBUILD=libqmi PREV_TAG=v1.1.0 PREV_MANIFEST_FILE="$M_V2" detect)"; rc=$? +assert_rc "$rc" 2 "an unrecognized FORCE_REBUILD value fails closed" +assert_err "FORCE_REBUILD='libqmi' is not a recognized value" "the error names the bad value" + +echo "== (g) the companion is never part of detection ==" +reset_repo +printf 'Description: a companion-only change\n' >>"$REPO/packaging/ceralive-modem-support/debian/control" +commit_all "touch the companion" +out="$(PREV_TAG=v1.1.0 PREV_MANIFEST_FILE="$M_V2" detect)"; rc=$? +assert_rc "$rc" 0 "exit 0" +assert_all unchanged "$out" "a companion-only change leaves all four sources unchanged" +assert_has "mode=differential" "$out" "mode=differential" +if printf '%s\n' "$out" | grep -q 'ceralive-modem-support'; then bad "the companion was verdicted"; else ok "the companion is never verdicted"; fi + +echo "== (h) a TypeScript-only release changes no packaging source ==" +reset_repo +printf 'export const y = 2;\n' >>"$REPO/control/src/index.ts" +commit_all "touch control/" +out="$(PREV_TAG=v1.1.0 PREV_MANIFEST_FILE="$M_V2" detect)"; rc=$? +assert_rc "$rc" 0 "exit 0" +assert_all unchanged "$out" "all four sources unchanged" +assert_has "mode=differential" "$out" "mode=differential" + +echo "== (i) an empty diff is differential with nothing changed ==" +reset_repo +out="$(PREV_TAG=v1.1.0 PREV_MANIFEST_FILE="$M_V2" detect)"; rc=$? +assert_rc "$rc" 0 "exit 0" +assert_all unchanged "$out" "all four sources unchanged" +assert_has "mode=differential" "$out" "mode=differential" + +echo "== (j) --out writes the same contract lines to a file ==" +outfile="$ROOT/verdicts.txt" +out="$(cd "$REPO" && PATH="$STUB_BIN:$PATH" PREV_TAG=v1.1.0 PREV_MANIFEST_FILE="$M_V2" \ + bash packaging/ci/detect-changed-sources.sh --out "$outfile" 2>"$ERR")"; rc=$? +assert_rc "$rc" 0 "exit 0" +if [ -f "$outfile" ] && [ "$(cat "$outfile")" = "$out" ]; then ok "--out file matches stdout"; else bad "--out file differs from stdout"; fi + +echo "== (k) seam misuse fails closed rather than silently forcing ==" +out="$(PREV_TAG=v1.1.0 PREV_MANIFEST_FILE="$ROOT/does-not-exist.txt" detect)"; rc=$? +assert_rc "$rc" 2 "an unreadable PREV_MANIFEST_FILE exits 2" +assert_err "PREV_MANIFEST_FILE=.*is set but not readable" "the error names the seam" +out="$(PREV_TAG=v1.1.0 PREV_MANIFEST_FILE="$M_V2" HEAD_REF=refs/heads/nope detect)"; rc=$? +assert_rc "$rc" 2 "an unresolvable HEAD_REF exits 2" +assert_err "HEAD ref 'refs/heads/nope' does not resolve" "the error names the ref" + +echo "== (l) an unfetched previous tag forces every source rather than guessing ==" +out="$(PREV_TAG=v9.9.9 PREV_MANIFEST_FILE="$M_V2" detect)"; rc=$? +assert_rc "$rc" 0 "exit 0" +assert_has "mode=force-all" "$out" "mode=force-all" +assert_all changed "$out" "every source changed" +assert_err "previous-ref-unresolvable" "the reason is previous-ref-unresolvable" + +echo +echo "passed: $pass failed: $fail" +if [ "$fail" -eq 0 ]; then + echo "PASS: per-source change detection + force-all bootstrap rules hold" + exit 0 +fi +echo "FAIL: detect-changed-sources contract violated" +exit 1 From 0070f1950bd1394d5913a6f9dd98c202f5c2dcf8 Mon Sep 17 00:00:00 2001 From: Andres Cera Date: Fri, 21 Aug 2026 18:46:53 -0500 Subject: [PATCH 2/7] feat(packaging): carry-forward staging of unchanged-source debs --- packaging/ci/stage-carryforward-debs.sh | 429 +++++++++++++++++++ packaging/ci/test-stage-carryforward-debs.sh | 344 +++++++++++++++ 2 files changed, 773 insertions(+) create mode 100755 packaging/ci/stage-carryforward-debs.sh create mode 100755 packaging/ci/test-stage-carryforward-debs.sh diff --git a/packaging/ci/stage-carryforward-debs.sh b/packaging/ci/stage-carryforward-debs.sh new file mode 100755 index 0000000..b980f4f --- /dev/null +++ b/packaging/ci/stage-carryforward-debs.sh @@ -0,0 +1,429 @@ +#!/usr/bin/env bash +# stage-carryforward-debs.sh — stage every UNCHANGED source's .debs from the PREVIOUS release +# into packaging/build//, byte-verified, so the differential build never rebuilds them. +# +# WHAT THIS IS +# The SECOND script of the differential-release pipeline. `detect-changed-sources.sh` decides +# WHICH sources moved; this one carries the bytes of the ones that did not. Builds are NOT +# reproducible, so an unchanged source can never be rebuilt at its old version — the only +# honest way to keep a release self-contained is to reuse the exact artifacts the release that +# last built them recorded. Those bytes are a build INPUT too: `build-bookworm.sh` seeds its +# temporary local apt repo from `build//`, so a changed source resolves its build-deps +# against the carried `-dev`/`gir1.2-*` packages rather than stock bookworm. +# +# INPUT — the verdict lines from detect-changed-sources.sh, on STDIN by default +# The caller pipes that script's stdout straight in (or points `--verdicts ` at the file +# it wrote with `--out`). Reading a stream rather than re-running the detector is deliberate: +# detection resolves the previous release over the network and must happen exactly ONCE per +# run, and release.yml already holds that result. +# +# libqrtr-glib=unchanged +# libmbim=unchanged +# libqmi=changed +# modemmanager=unchanged +# mode=differential +# +# Source names are upstream-pins.yaml PIN KEYS (`modemmanager`, not `ModemManager`). The +# `mode=` line is REQUIRED — a verdict stream without it is malformed, not a default. +# +# WHAT IS CARRIED +# EVERY row of an `unchanged` source — runtime AND aux (`-dbgsym`, `-dev`, `gir1.2-*`). The +# `role` column is deliberately NOT filtered on: the release manifest is manifest-complete and +# a release that re-attached only the 9-package runtime closure would silently drop ~36 debs. +# Rows of a `changed` source are skipped (the build produces them fresh). +# +# THE COMPANION IS NEVER CARRIED +# `ceralive-modem-support` ALWAYS rebuilds — apt-worker's health check greps `^Version: $`, +# which only the companion's bare tag version can satisfy. Its row is skipped explicitly, and +# that skip is checked BEFORE the unknown-source rule below, so a previous manifest carrying it +# (every closure_version-2 manifest does) is normal input rather than an error. +# +# GITHUB ASSET-NAME RECONCILIATION (the inverse of reconcile-release-assets.sh) +# The uploader stages every asset under a sanitized basename — `target="${src//\~/.}"` — so the +# canonical `libqmi-glib5_1.38.0-1~ceralive1.1.0_amd64.deb` is STORED on the release as +# `libqmi-glib5_1.38.0-1.ceralive1.1.0_amd64.deb`. Going back the other way cannot simply +# substitute `.`→`~` (the name is full of legitimate dots), so each `~` becomes a single-char +# glob wildcard `?` and every other character stays anchored — apt-worker's `modem_asset_glob` +# discipline. EXACTLY ONE asset must match: zero means the release does not carry it, and two +# (both the `~` and the `.` spelling present) means the name is ambiguous. Either is fail-closed. +# The staged file always lands under the CANONICAL `~` name, which is what dpkg orders on and +# what the next manifest must report. +# +# FAIL CLOSED, ALWAYS NAMING THE OFFENDING ROW/FILE +# * a manifest row's asset is missing, or its name is ambiguous; +# * a staged asset's sha256 differs from the manifest row; +# * a row names a source that is neither the companion nor any verdicted source (defense in +# depth: a source the caller never adjudicated must never be carried on this script's guess); +# * an `unchanged` source has ZERO rows in the previous manifest (nothing would be carried and +# nothing would be built, so the merged set would be quietly incomplete); +# * the previous manifest is V1-SHAPED — no `closure_version:` header. An ABSENT header IS +# closure version 1 (apt-worker's backward-compatibility default), which predates the +# companion row this pipeline depends on. In practice detection force-alls on a v1 manifest, +# so nothing is ever carried from one; if this script is nevertheless ASKED to, it refuses +# rather than guessing which shape it is reading. +# * a destination file already exists with DIFFERENT bytes (a carried deb never overwrites a +# freshly built one — reconcile-release-assets.sh's integrity-compare stance). +# +# NOTHING TO CARRY IS A NORMAL OUTCOME +# `mode=force-all` (or simply no `unchanged` source) stages nothing and exits 0 WITHOUT reading +# a manifest at all — under force-all there is nothing to carry from by definition. +# +# STAGING LAYOUT +# `//` — the same gitignored tree +# `build-bookworm.sh` writes freshly built debs into and `generate-release-manifest.sh` reads. +# `build_arch` is taken verbatim from manifest column 1, so an `all` row would land in +# `build/all/`. Only the companion is ever `Architecture: all`, and the companion is never +# carried — an upstream source's rows are always arch-dependent — so `build/all/` should never +# be created here. It is handled rather than assumed, and loudly logged if it ever happens. +# +# SEAMS (all optional; they exist so the contract test runs offline, with no gh and no network) +# PREV_MANIFEST_FILE Read the previous release manifest from this local path; skips +# `gh release download`. Set-but-unreadable is exit 2, not a silent skip. +# CARRYFORWARD_ASSET_DIR Read the release assets from this local directory instead of +# downloading them. Names in it may be either the GitHub-mangled `.` +# form or the canonical `~` form — the glob resolves both. +# PREV_TAG The release to download the manifest/assets from (required when +# neither seam above is set). +# GH_REPO Passed to `gh --repo`. +# BUILD_ROOT Staging root (default: packaging/build). `--build-root` overrides it. +# +# OUTPUT (stdout — the machine contract) +# One `/` line per staged deb, in manifest order. Every human-readable +# line goes to stderr, so stdout stays parseable. +# +# USAGE +# detect-changed-sources.sh | stage-carryforward-debs.sh +# stage-carryforward-debs.sh --verdicts verdicts.txt --build-root packaging/build +# +# EXIT +# 0 staged (or nothing to carry). +# 2 usage / environment error: bad argument, malformed verdict stream, a set-but-unreadable +# seam, or no way to reach the previous manifest. +# 3 fail closed: the carry cannot be performed honestly (see the list above). Every message +# names the row, the file, or the source at fault. +set -euo pipefail + +LOG_PREFIX="stage-carryforward-debs" +log() { printf '%s: %s\n' "$LOG_PREFIX" "$*" >&2; } +die() { + printf '%s: %s\n' "$LOG_PREFIX" "$*" >&2 + exit 2 +} +refuse() { + printf '%s: FAIL CLOSED — %s\n' "$LOG_PREFIX" "$*" >&2 + exit 3 +} + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PKG_ROOT="$(cd "$HERE/.." && pwd)" + +# The one source that is never carried. It always rebuilds (apt-worker's health check greps +# `^Version: $`, which only its bare tag version satisfies). +COMPANION_SOURCE="ceralive-modem-support" + +# ---- arguments ----------------------------------------------------------------------------- +VERDICTS_FILE="" +BUILD_ROOT="${BUILD_ROOT:-$PKG_ROOT/build}" +while [ $# -gt 0 ]; do + case "$1" in + --verdicts) + shift + [ $# -gt 0 ] || die "--verdicts requires a file path" + VERDICTS_FILE="$1" + ;; + --verdicts=*) VERDICTS_FILE="${1#--verdicts=}" ;; + --build-root) + shift + [ $# -gt 0 ] || die "--build-root requires a directory path" + BUILD_ROOT="$1" + ;; + --build-root=*) BUILD_ROOT="${1#--build-root=}" ;; + -h | --help) + echo "usage: stage-carryforward-debs.sh [--verdicts ] [--build-root ]" + echo " (verdict lines are read from stdin when --verdicts is absent)" + exit 0 + ;; + *) die "unknown argument '$1' (usage: stage-carryforward-debs.sh [--verdicts ] [--build-root ])" ;; + esac + shift +done + +if [ -n "$VERDICTS_FILE" ]; then + [ -r "$VERDICTS_FILE" ] || + die "--verdicts '$VERDICTS_FILE' is not readable (a named input pointing at nothing is operator error, not an empty verdict set)" + VERDICTS_SRC="$VERDICTS_FILE" +else + VERDICTS_SRC="/dev/stdin" +fi + +# ---- the verdict stream --------------------------------------------------------------------- +declare -A VERDICT=() +VERDICT_ORDER=() +MODE="" +while IFS= read -r line || [ -n "$line" ]; do + [ -n "$line" ] || continue + case "$line" in + \#*) continue ;; + mode=*) MODE="${line#mode=}" ;; + *=changed | *=unchanged) + key="${line%%=*}" + val="${line#*=}" + [ -n "$key" ] || die "verdict line '$line' has an empty source name" + if [ -n "${VERDICT[$key]:-}" ] && [ "${VERDICT[$key]}" != "$val" ]; then + die "source '$key' is verdicted twice and inconsistently ('${VERDICT[$key]}' then '$val')" + fi + if [ -z "${VERDICT[$key]:-}" ]; then VERDICT_ORDER+=("$key"); fi + VERDICT["$key"]="$val" + ;; + *) die "unparseable verdict line '$line' (expected '=changed|unchanged' or 'mode=differential|force-all')" ;; + esac +done <"$VERDICTS_SRC" + +[ "${#VERDICT_ORDER[@]}" -gt 0 ] || + die "the verdict stream carried no '=changed|unchanged' line (is detect-changed-sources.sh's stdout actually piped in?)" +case "$MODE" in +differential | force-all) ;; +"") die "the verdict stream carried no 'mode=' line — a stream without it is malformed, not a default" ;; +*) die "unrecognized mode '$MODE' (the only accepted values are 'differential' and 'force-all')" ;; +esac + +UNCHANGED=() +for key in "${VERDICT_ORDER[@]}"; do + if [ "${VERDICT[$key]}" = unchanged ]; then UNCHANGED+=("$key"); fi +done + +# A force-all run rebuilds EVERYTHING by definition, so an `unchanged` verdict under it is a +# contract violation in the caller's wiring rather than an instruction to carry something. +if [ "$MODE" = force-all ] && [ "${#UNCHANGED[@]}" -gt 0 ]; then + die "mode=force-all but source(s) [${UNCHANGED[*]}] are verdicted 'unchanged' — under force-all every source rebuilds; refusing to carry anything from a self-contradicting verdict stream" +fi + +log "verdicts: mode=$MODE, ${#UNCHANGED[@]} of ${#VERDICT_ORDER[@]} source(s) unchanged" + +if [ "${#UNCHANGED[@]}" -eq 0 ]; then + log "nothing to carry — every source rebuilds; no previous manifest is read and no deb is staged" + exit 0 +fi +log "carrying: ${UNCHANGED[*]}" + +is_unchanged() { # -> 0 when the caller verdicted it `unchanged` + local s + for s in "${UNCHANGED[@]}"; do [ "$s" = "$1" ] && return 0; done + return 1 +} +is_verdicted() { [ -n "${VERDICT[$1]:-}" ]; } + +# ---- the previous release manifest ------------------------------------------------------------ +TMP_DIR="" +cleanup() { + [ -n "$TMP_DIR" ] && rm -rf "$TMP_DIR" + return 0 +} +trap cleanup EXIT + +gh_args() { + if [ -n "${GH_REPO:-}" ]; then printf '%s\n%s\n' "--repo" "$GH_REPO"; fi +} +need_tmp() { + if [ -z "$TMP_DIR" ]; then TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/stage-carryforward.XXXXXX")"; fi +} + +MANIFEST="" +if [ -n "${PREV_MANIFEST_FILE:-}" ]; then + [ -r "$PREV_MANIFEST_FILE" ] || + die "PREV_MANIFEST_FILE='$PREV_MANIFEST_FILE' is set but not readable (a named seam pointing at nothing is operator error)" + MANIFEST="$PREV_MANIFEST_FILE" + log "previous release manifest: $MANIFEST (PREV_MANIFEST_FILE seam)" +else + [ -n "${PREV_TAG:-}" ] || + die "no previous manifest: set PREV_MANIFEST_FILE, or PREV_TAG so the manifest can be downloaded from that release" + command -v gh >/dev/null 2>&1 || + die "no previous manifest: 'gh' is not installed and PREV_MANIFEST_FILE is unset, so release '$PREV_TAG' cannot be read" + need_tmp + mkdir -p "$TMP_DIR/manifest" + ghargs=() + mapfile -t ghargs < <(gh_args) + gh release download "$PREV_TAG" "${ghargs[@]}" --pattern 'release-manifest*.txt' --dir "$TMP_DIR/manifest" >/dev/null 2>&1 || + refuse "release '$PREV_TAG' has no downloadable release-manifest asset, so there is nothing to carry forward from" + MANIFEST="$(find "$TMP_DIR/manifest" -maxdepth 1 -type f -name 'release-manifest*.txt' | LC_ALL=C sort | head -n1)" + [ -n "$MANIFEST" ] || + refuse "'gh release download' produced no release-manifest*.txt for '$PREV_TAG'" + log "previous release manifest: $MANIFEST (gh release download from $PREV_TAG)" +fi + +# An ABSENT `closure_version:` header IS closure version 1 — apt-worker's +# `modem_manifest_closure_version` treats it exactly that way. A v1 manifest predates the +# companion row this pipeline depends on, so carrying from one would be a guess. +grep -q '^closure_version:' "$MANIFEST" || + refuse "previous manifest '$MANIFEST' carries no 'closure_version:' header (an absent header IS closure version 1); refusing to carry from a v1-shaped manifest rather than guessing its shape" + +# Data rows are the 7-column `build_arch package source version role filename sha256` lines. +# Header lines are `key: value` and are excluded by the trailing-colon test on column 1, so a +# multi-token header (`sources: [...]`) can never be mistaken for a row. +ROWS="$(awk '$0 !~ /^#/ && NF==7 && $1 !~ /:$/ { print }' "$MANIFEST")" +[ -n "$ROWS" ] || + refuse "previous manifest '$MANIFEST' contains no deb rows" + +# ---- release assets --------------------------------------------------------------------------- +ASSET_NAMES=() +ASSET_DIR="" +if [ -n "${CARRYFORWARD_ASSET_DIR:-}" ]; then + [ -d "$CARRYFORWARD_ASSET_DIR" ] || + die "CARRYFORWARD_ASSET_DIR='$CARRYFORWARD_ASSET_DIR' is set but is not a directory" + ASSET_DIR="$CARRYFORWARD_ASSET_DIR" + mapfile -t ASSET_NAMES < <(find "$ASSET_DIR" -maxdepth 1 -type f -printf '%f\n' | LC_ALL=C sort) + log "release assets: ${#ASSET_NAMES[@]} local file(s) in $ASSET_DIR (CARRYFORWARD_ASSET_DIR seam)" +else + [ -n "${PREV_TAG:-}" ] || + die "no release assets: set CARRYFORWARD_ASSET_DIR, or PREV_TAG so the assets can be downloaded" + command -v gh >/dev/null 2>&1 || + die "no release assets: 'gh' is not installed and CARRYFORWARD_ASSET_DIR is unset" + log "release assets: downloaded per row from release '$PREV_TAG'" +fi + +# Canonical `~ceralive…` filename -> the glob that matches whatever name the release stores it +# under. The uploader sanitizes `~`→`.`, so each `~` becomes a single-char wildcard while every +# other character stays anchored. INVERSE of reconcile-release-assets.sh's `${src//\~/.}`. +asset_glob() { printf '%s\n' "${1//\~/?}"; } + +# Put the row's asset at , resolving the stored (possibly mangled) name into the global +# RESOLVED_ASSET so callers can report which asset a failure came from. Deliberately NOT a +# command-substitution helper: every failure here is a `refuse`, and an `exit 3` inside `$( )` +# would leave the main shell to infer the outcome from a status. +RESOLVED_ASSET="" +fetch_asset() { # + local canonical="$1" dest="$2" glob name matches=() dl ghargs=() + glob="$(asset_glob "$canonical")" + RESOLVED_ASSET="" + + if [ -n "$ASSET_DIR" ]; then + for name in "${ASSET_NAMES[@]}"; do + # shellcheck disable=SC2053 # $glob is a glob PATTERN here, quoting would break it + if [[ "$name" == $glob ]]; then matches+=("$name"); fi + done + if [ "${#matches[@]}" -eq 0 ]; then + refuse "asset for '$canonical' is missing — no file in '$ASSET_DIR' matches '$glob' (the previous release does not carry it, so it cannot be carried forward)" + fi + if [ "${#matches[@]}" -gt 1 ]; then + refuse "asset for '$canonical' is ambiguous — ${#matches[@]} files match '$glob': ${matches[*]}" + fi + cp "$ASSET_DIR/${matches[0]}" "$dest" + RESOLVED_ASSET="${matches[0]}" + return 0 + fi + + need_tmp + dl="$TMP_DIR/dl" + rm -rf "$dl" + mkdir -p "$dl" + mapfile -t ghargs < <(gh_args) + gh release download "$PREV_TAG" "${ghargs[@]}" --pattern "$glob" --dir "$dl" >/dev/null 2>&1 || + refuse "asset for '$canonical' could not be downloaded from release '$PREV_TAG' (pattern '$glob')" + mapfile -t matches < <(find "$dl" -maxdepth 1 -type f -printf '%f\n' | LC_ALL=C sort) + if [ "${#matches[@]}" -ne 1 ]; then + refuse "asset for '$canonical' resolved to ${#matches[@]} release asset(s) via '$glob' (expected exactly 1): ${matches[*]:-none}" + fi + mv "$dl/${matches[0]}" "$dest" + RESOLVED_ASSET="${matches[0]}" +} + +sha_of() { sha256sum "$1" | awk '{print $1}'; } + +# ---- the carry -------------------------------------------------------------------------------- +declare -A STAGED_PER_SOURCE=() +for key in "${UNCHANGED[@]}"; do STAGED_PER_SOURCE["$key"]=0; done + +need_tmp +STAGE_TMP="$TMP_DIR/stage" +mkdir -p "$STAGE_TMP" + +staged_total=0 +skipped_changed=0 +skipped_companion=0 +staged_lines=() + +while IFS= read -r row; do + [ -n "$row" ] || continue + # shellcheck disable=SC2086 # deliberate word-splitting of a fixed 7-column row + set -- $row + build_arch="$1" + package="$2" + source_name="$3" + version="$4" + role="$5" + filename="$6" + want_sha="$7" + + # The companion is checked FIRST: it is never verdicted, so the unknown-source rule below + # would otherwise reject a perfectly ordinary closure_version-2 manifest. + if [ "$source_name" = "$COMPANION_SOURCE" ]; then + skipped_companion=$((skipped_companion + 1)) + log " skip $filename — the companion '$COMPANION_SOURCE' always rebuilds and is never carried" + continue + fi + + if ! is_verdicted "$source_name"; then + refuse "manifest row '$filename' names source '$source_name', which the caller never verdicted (verdicted: ${VERDICT_ORDER[*]}) — a source the caller did not adjudicate is never carried" + fi + + if ! is_unchanged "$source_name"; then + skipped_changed=$((skipped_changed + 1)) + continue + fi + + case "$filename" in + */* | "") refuse "manifest row for package '$package' has an unusable filename '$filename'" ;; + esac + if [ "$build_arch" = all ]; then + log " NOTE: upstream source '$source_name' has a build_arch 'all' row ($filename) — only the companion is ever Architecture: all; staging it into '$BUILD_ROOT/all/' verbatim" + fi + + tmp_asset="$STAGE_TMP/$filename" + fetch_asset "$filename" "$tmp_asset" + stored="$RESOLVED_ASSET" + + got_sha="$(sha_of "$tmp_asset")" + if [ "$got_sha" != "$want_sha" ]; then + refuse "carried deb '$filename' (release asset '$stored') sha256 $got_sha != previous manifest $want_sha — the recorded bytes are the only thing that may be carried" + fi + + dest_dir="$BUILD_ROOT/$build_arch" + mkdir -p "$dest_dir" + dest="$dest_dir/$filename" + if [ -e "$dest" ]; then + dest_sha="$(sha_of "$dest")" + if [ "$dest_sha" != "$want_sha" ]; then + refuse "'$dest' already exists with different bytes (staged $dest_sha != manifest $want_sha); a carried deb never overwrites what is already in the build tree" + fi + log " keep $build_arch/$filename ($package $version, $role) — already staged, integrity matches" + else + cp "$tmp_asset" "$dest" + # Re-hash the DESTINATION: a short write must never look like a successful carry. + dest_sha="$(sha_of "$dest")" + if [ "$dest_sha" != "$want_sha" ]; then + refuse "staged copy '$dest' hashes $dest_sha, not the manifest's $want_sha" + fi + log " stage $build_arch/$filename ($package $version, $role) <- release asset '$stored'" + fi + + rm -f "$tmp_asset" + per_source="${STAGED_PER_SOURCE[$source_name]}" + STAGED_PER_SOURCE["$source_name"]=$((per_source + 1)) + staged_total=$((staged_total + 1)) + staged_lines+=("$build_arch/$filename") +done <<<"$ROWS" + +# An unchanged source with no rows in the previous manifest would be neither carried nor built — +# a silently incomplete merged set, which is exactly what this pipeline exists to prevent. +for key in "${UNCHANGED[@]}"; do + if [ "${STAGED_PER_SOURCE[$key]}" -eq 0 ]; then + refuse "source '$key' is verdicted 'unchanged' but the previous manifest '$MANIFEST' carries ZERO rows for it — it would be neither carried nor rebuilt" + fi +done + +for key in "${UNCHANGED[@]}"; do + log " carried $key: ${STAGED_PER_SOURCE[$key]} deb(s)" +done +log "staged $staged_total deb(s) into '$BUILD_ROOT' (skipped $skipped_changed row(s) of changed sources, $skipped_companion companion row(s))" + +if [ "${#staged_lines[@]}" -gt 0 ]; then printf '%s\n' "${staged_lines[@]}"; fi diff --git a/packaging/ci/test-stage-carryforward-debs.sh b/packaging/ci/test-stage-carryforward-debs.sh new file mode 100755 index 0000000..aa3b6ff --- /dev/null +++ b/packaging/ci/test-stage-carryforward-debs.sh @@ -0,0 +1,344 @@ +#!/usr/bin/env bash +# test-stage-carryforward-debs.sh — the carry-forward stager's behaviour contract. +# +# HOST-RUNNABLE, OFFLINE, NO DOCKER, NO `gh`. Two seams make that possible: +# +# 1. `PREV_MANIFEST_FILE` supplies a fixture previous-release manifest, generated here in the +# real 7-column `build_arch package source version role filename sha256` shape (closure +# version 2 — four upstream sources at runtime AND aux roles, plus the companion row every +# real v2 manifest carries). +# +# 2. `CARRYFORWARD_ASSET_DIR` supplies the release assets as local files. Every upstream +# fixture asset is written under the GITHUB-MANGLED name (`~` -> `.`), never the canonical +# one, and a control asserts the directory contains no `~` at all — so the glob-based +# reconciliation is the ONLY way any of them can be found. If the inverse mapping regressed, +# every case would fail with a missing asset rather than passing by accident. +# +# The assets are tiny placeholder files rather than real archives on purpose: this script's +# contract is sha256 + name reconciliation + placement. It never invokes `dpkg-deb` (unlike +# apt-worker's `verify_staged_deb`, which additionally reads the control stanza), so a real +# archive would prove nothing extra here and would make the fixture non-deterministic — +# `dpkg-deb --build` output is not bit-reproducible. +# +# Cases: (a) mangled assets -> canonical staging, aux rows included (a2) zero-upstream-build +# shape (b) flipped byte (c) missing asset (d) v1-shaped manifest (e) unverdicted +# source (f) companion never carried (g) nothing to carry (h) unchanged source with no +# rows (i) stdin == --verdicts (k) ambiguous asset name (l/m) malformed verdict stream +# (n) idempotent re-run (o) destination collision. +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT="$HERE/stage-carryforward-debs.sh" +[ -f "$SCRIPT" ] || { echo "missing: $SCRIPT" >&2; exit 1; } + +pass=0; fail=0 +ok() { printf ' ok %s\n' "$1"; pass=$((pass + 1)); } +bad() { printf ' FAIL %s\n' "$1"; fail=$((fail + 1)); } + +ROOT="$(mktemp -d "${TMPDIR:-/tmp}/ceralive-stage-carryforward.XXXXXX")" +trap 'rm -rf "$ROOT"' EXIT + +ASSETS="$ROOT/gh-assets" +OUT="$ROOT/stdout.txt" +ERR="$ROOT/stderr.txt" +mkdir -p "$ASSETS" + +# ---- the fixture previous release -------------------------------------------------------------- +# Two arches x four sources x {runtime, aux}. The aux members are deliberately one dbgsym, one +# -dev and one gir1.2-* — the three classes a runtime-only carry would silently drop. +ROWS_SPEC=( + "libqrtr-glib|libqrtr-glib0|1.4.0-1~ceralive1.1.0|runtime" + "libqrtr-glib|gir1.2-qrtr-1.0|1.4.0-1~ceralive1.1.0|aux" + "libmbim|libmbim-glib4|1.34.0-1~ceralive1.1.0|runtime" + "libmbim|libmbim-glib4-dbgsym|1.34.0-1~ceralive1.1.0|aux" + "libqmi|libqmi-glib5|1.38.0-1~ceralive1.1.0|runtime" + "libqmi|libqmi-glib-dev|1.38.0-1~ceralive1.1.0|aux" + "modemmanager|modemmanager|1.24.2-2~ceralive1.1.0|runtime" + "modemmanager|modemmanager-dev|1.24.2-2~ceralive1.1.0|aux" +) +COMPANION_DEB="ceralive-modem-support_1.1.0_all.deb" + +M_V2="$ROOT/release-manifest-v2.txt" +cat >"$M_V2" <<'EOF' +# CeraLive modem-stack release manifest (fixture) +tag: v1.1.0 +version: 1.1.0 +sources: [ceralive-modem-support libmbim libqmi libqrtr-glib modemmanager] +closure_version: 2 +runtime_closure_size: 9 +arch_all_closure_size: 1 +index_arches: [arm64 amd64] +# columns: build_arch package source version role filename sha256 +EOF + +sha_of() { sha256sum "$1" | awk '{print $1}'; } + +for arch in amd64 arm64; do + for spec in "${ROWS_SPEC[@]}"; do + IFS='|' read -r f_src f_pkg f_ver f_role <<<"$spec" + canonical="${f_pkg}_${f_ver}_${arch}.deb" + stored="${canonical//\~/.}" # exactly what the release uploader stores + printf 'fixture payload for %s\n' "$canonical" >"$ASSETS/$stored" + printf '%s %s %s %s %s %s %s\n' \ + "$arch" "$f_pkg" "$f_src" "$f_ver" "$f_role" "$canonical" "$(sha_of "$ASSETS/$stored")" >>"$M_V2" + done +done +printf 'fixture payload for %s\n' "$COMPANION_DEB" >"$ASSETS/$COMPANION_DEB" +printf 'all ceralive-modem-support ceralive-modem-support 1.1.0 runtime %s %s\n' \ + "$COMPANION_DEB" "$(sha_of "$ASSETS/$COMPANION_DEB")" >>"$M_V2" +echo "# build_dirs: all amd64 arm64" >>"$M_V2" + +# v1 SHAPE: the same manifest with no `closure_version:` header (an absent header IS version 1). +M_V1="$ROOT/release-manifest-v1.txt" +grep -v '^closure_version:' "$M_V2" >"$M_V1" + +# A v2 manifest carrying ZERO rows for libqrtr-glib. +M_NOQRTR="$ROOT/release-manifest-no-qrtr.txt" +awk '$3 != "libqrtr-glib"' "$M_V2" >"$M_NOQRTR" + +# ---- fixture controls --------------------------------------------------------------------------- +echo "== fixture controls ==" +if find "$ASSETS" -maxdepth 1 -name '*~*' | grep -q .; then + bad "the asset dir must hold ONLY GitHub-mangled names — a '~' name would let the glob pass vacuously" +else + ok "every upstream asset is stored under its mangled '.' name (the glob is the only way to find it)" +fi +if [ "$(grep -c 'ceralive-modem-support' "$M_V2")" -ge 2 ] && [ -f "$ASSETS/$COMPANION_DEB" ]; then + ok "the companion is present in BOTH the manifest and the asset dir (its skip cannot be vacuous)" +else + bad "fixture: the companion row/asset is missing" +fi +if [ "$(awk 'NF==7 && $1 !~ /:$/ && $0 !~ /^#/' "$M_V2" | wc -l)" -eq 17 ]; then + ok "fixture manifest carries 17 deb rows (8 packages x 2 arches + the companion)" +else + bad "fixture manifest row count is $(awk 'NF==7 && $1 !~ /:$/ && $0 !~ /^#/' "$M_V2" | wc -l), expected 17" +fi + +# ---- harness ------------------------------------------------------------------------------------ +verdicts() { # ... + local out="$1" mode="$2"; shift 2 + : >"$out" + local v + for v in "$@"; do printf '%s\n' "$v" >>"$out"; done + printf 'mode=%s\n' "$mode" >>"$out" +} + +stage() { # + PREV_MANIFEST_FILE="$1" CARRYFORWARD_ASSET_DIR="$2" \ + bash "$SCRIPT" --build-root "$3" --verdicts "$4" >"$OUT" 2>"$ERR" +} + +staged_count() { find "$1" -type f -name '*.deb' 2>/dev/null | wc -l; } +assert_rc() { if [ "$1" -eq "$2" ]; then ok "$3"; else bad "$3 — exit $1, expected $2"; fi; } +assert_nz() { if [ "$1" -ne 0 ]; then ok "$2"; else bad "$2 — expected a non-zero exit, got 0"; fi; } +assert_err() { if grep -qF -- "$1" "$ERR"; then ok "$2"; else bad "$2 — stderr does not name '$1'"; fi; } +assert_file() { if [ -f "$1" ]; then ok "$2"; else bad "$2 — missing $1"; fi; } +assert_no_file() { if [ -e "$1" ]; then bad "$2 — $1 exists"; else ok "$2"; fi; } +assert_eq() { if [ "$1" = "$2" ]; then ok "$3"; else bad "$3 — got '$1', expected '$2'"; fi; } + +# One byte overwritten IN PLACE (not appended, which would also change the length). The new byte +# is derived from the existing one so a file whose last byte already equalled it cannot make the +# case vacuous; the sha256 change is asserted before the case runs. +corrupt_one_byte() { # + local f="$1" sz cur next + sz="$(stat -c%s "$f")" + cur="$(dd if="$f" bs=1 skip=$((sz - 1)) count=1 2>/dev/null | od -An -tu1 | tr -d ' ')" + next=$(((cur + 1) % 256)) + printf '%b' "\\$(printf '%03o' "$next")" | dd of="$f" bs=1 seek=$((sz - 1)) count=1 conv=notrunc 2>/dev/null +} + +V_THREE="$ROOT/verdicts-three.txt" +V_ALL="$ROOT/verdicts-all.txt" +V_NONE="$ROOT/verdicts-none.txt" +verdicts "$V_THREE" differential libqrtr-glib=unchanged libmbim=unchanged libqmi=changed modemmanager=unchanged +verdicts "$V_ALL" differential libqrtr-glib=unchanged libmbim=unchanged libqmi=unchanged modemmanager=unchanged +verdicts "$V_NONE" force-all libqrtr-glib=changed libmbim=changed libqmi=changed modemmanager=changed + +echo +echo "== (a) mangled release assets stage under CANONICAL names, runtime AND aux ==" +BR_A="$ROOT/build-a" +stage "$M_V2" "$ASSETS" "$BR_A" "$V_THREE"; rc=$? +assert_rc "$rc" 0 "exit 0" +assert_eq "$(staged_count "$BR_A")" 12 "12 debs staged (3 unchanged sources x 2 packages x 2 arches)" +assert_file "$BR_A/amd64/libqrtr-glib0_1.4.0-1~ceralive1.1.0_amd64.deb" "runtime row staged (amd64)" +assert_file "$BR_A/arm64/libqrtr-glib0_1.4.0-1~ceralive1.1.0_arm64.deb" "runtime row staged (arm64)" +assert_file "$BR_A/arm64/gir1.2-qrtr-1.0_1.4.0-1~ceralive1.1.0_arm64.deb" "AUX gir1.2-* row staged" +assert_file "$BR_A/amd64/libmbim-glib4-dbgsym_1.34.0-1~ceralive1.1.0_amd64.deb" "AUX dbgsym row staged" +assert_file "$BR_A/arm64/modemmanager-dev_1.24.2-2~ceralive1.1.0_arm64.deb" "AUX -dev row staged" +assert_no_file "$BR_A/amd64/libqmi-glib5_1.38.0-1~ceralive1.1.0_amd64.deb" "a CHANGED source's runtime deb is not carried" +assert_no_file "$BR_A/amd64/libqmi-glib-dev_1.38.0-1~ceralive1.1.0_amd64.deb" "a CHANGED source's aux deb is not carried" +if find "$BR_A" -type f -name '*.ceralive1.1.0_*' | grep -q .; then + bad "a GitHub-mangled name survived into the build tree" +else + ok "no mangled '.ceralive' name reached the build tree" +fi +if [ "$(find "$BR_A" -type f -name '*~ceralive1.1.0*' | wc -l)" -eq 12 ]; then + ok "every staged deb carries the canonical '~ceralive' name dpkg orders on" +else + bad "not every staged deb carries a '~ceralive' name" +fi +if cmp -s "$ASSETS/libmbim-glib4_1.34.0-1.ceralive1.1.0_amd64.deb" \ + "$BR_A/amd64/libmbim-glib4_1.34.0-1~ceralive1.1.0_amd64.deb"; then + ok "the staged bytes are the release asset's bytes, verbatim" +else + bad "the staged deb does not match the release asset byte-for-byte" +fi +assert_eq "$(grep -c . "$OUT")" 12 "stdout carries one / line per staged deb" +if grep -qxF 'amd64/libmbim-glib4_1.34.0-1~ceralive1.1.0_amd64.deb' "$OUT"; then + ok "stdout names staged debs by canonical /" +else + bad "stdout does not carry the expected / line" +fi + +echo "== (a2) the zero-upstream-build shape: all four sources carried ==" +BR_A2="$ROOT/build-a2" +stage "$M_V2" "$ASSETS" "$BR_A2" "$V_ALL"; rc=$? +assert_rc "$rc" 0 "exit 0" +assert_eq "$(staged_count "$BR_A2")" 16 "all 16 upstream rows staged (8 packages x 2 arches)" +assert_file "$BR_A2/amd64/libqmi-glib5_1.38.0-1~ceralive1.1.0_amd64.deb" "libqmi is carried once it is unchanged" + +echo "== (f) the companion is NEVER carried, even with all sources unchanged ==" +if find "$BR_A2" -type f -name 'ceralive-modem-support*' | grep -q .; then + bad "the companion was staged — it must always rebuild" +else + ok "the companion deb is never staged" +fi +assert_no_file "$BR_A2/all" "no build/all directory is created for a carry" +assert_err "the companion 'ceralive-modem-support' always rebuilds" "the skip is logged with its reason" + +echo "== (b) one flipped byte fails closed, naming the file ==" +ASSETS_B="$ROOT/assets-b" +cp -a "$ASSETS" "$ASSETS_B" +VICTIM="$ASSETS_B/libmbim-glib4_1.34.0-1.ceralive1.1.0_amd64.deb" +before="$(sha_of "$VICTIM")" +corrupt_one_byte "$VICTIM" +if [ "$(sha_of "$VICTIM")" != "$before" ]; then + ok "fixture: the flipped byte actually changed the sha256" +else + bad "fixture: corruption did not change the sha256 — the case would be vacuous" +fi +BR_B="$ROOT/build-b" +stage "$M_V2" "$ASSETS_B" "$BR_B" "$V_ALL"; rc=$? +assert_nz "$rc" "a corrupted carried deb exits non-zero" +assert_err "libmbim-glib4_1.34.0-1~ceralive1.1.0_amd64.deb" "stderr NAMES the corrupted file" +assert_err "sha256" "stderr says the failure is a sha256 mismatch" +assert_no_file "$BR_B/amd64/libmbim-glib4_1.34.0-1~ceralive1.1.0_amd64.deb" "the corrupted deb is never staged" +printf ' evidence (exit %s): %s\n' "$rc" "$(grep -m1 'FAIL CLOSED' "$ERR")" + +echo "== (c) a missing asset fails closed, naming the file ==" +ASSETS_C="$ROOT/assets-c" +cp -a "$ASSETS" "$ASSETS_C" +rm -f "$ASSETS_C/modemmanager-dev_1.24.2-2.ceralive1.1.0_arm64.deb" +BR_C="$ROOT/build-c" +stage "$M_V2" "$ASSETS_C" "$BR_C" "$V_ALL"; rc=$? +assert_nz "$rc" "a manifest row with no release asset exits non-zero" +assert_err "modemmanager-dev_1.24.2-2~ceralive1.1.0_arm64.deb" "stderr NAMES the missing file" +assert_err "is missing" "stderr says the asset is missing" + +echo "== (d) a v1-shaped previous manifest is REFUSED, not guessed at ==" +BR_D="$ROOT/build-d" +stage "$M_V1" "$ASSETS" "$BR_D" "$V_ALL"; rc=$? +assert_nz "$rc" "carrying from a v1-shaped manifest exits non-zero" +assert_err "closure_version" "stderr names the absent closure_version header" +assert_err "refusing to carry from a v1-shaped manifest" "stderr says it refuses rather than guessing" +assert_eq "$(staged_count "$BR_D")" 0 "nothing is staged from a v1-shaped manifest" + +echo "== (e) a row naming an UNVERDICTED source fails closed ==" +V_MISSING="$ROOT/verdicts-missing-source.txt" +verdicts "$V_MISSING" differential libqrtr-glib=unchanged libqmi=changed modemmanager=unchanged +BR_E="$ROOT/build-e" +stage "$M_V2" "$ASSETS" "$BR_E" "$V_MISSING"; rc=$? +assert_nz "$rc" "a source the caller never adjudicated exits non-zero" +assert_err "libmbim" "stderr NAMES the unverdicted source" +assert_err "never verdicted" "stderr explains the row was never adjudicated" + +echo "== (g) nothing to carry needs no manifest at all ==" +BR_G="$ROOT/build-g" +bash "$SCRIPT" --build-root "$BR_G" --verdicts "$V_NONE" >"$OUT" 2>"$ERR"; rc=$? +assert_rc "$rc" 0 "force-all with every source changed exits 0" +assert_eq "$(staged_count "$BR_G")" 0 "nothing is staged" +assert_err "nothing to carry" "stderr says nothing was carried" +if grep -q 'previous release manifest:' "$ERR"; then + bad "a previous manifest was resolved when there was nothing to carry" +else + ok "no previous manifest is resolved when there is nothing to carry" +fi + +echo "== (h) an unchanged source with ZERO rows in the previous manifest fails closed ==" +BR_H="$ROOT/build-h" +stage "$M_NOQRTR" "$ASSETS" "$BR_H" "$V_ALL"; rc=$? +assert_nz "$rc" "an unchanged source absent from the manifest exits non-zero" +assert_err "libqrtr-glib" "stderr NAMES the source with no rows" +assert_err "ZERO rows" "stderr says the manifest carries no rows for it" + +echo "== (i) verdicts on STDIN are identical to --verdicts ==" +BR_I="$ROOT/build-i" +PREV_MANIFEST_FILE="$M_V2" CARRYFORWARD_ASSET_DIR="$ASSETS" \ + bash "$SCRIPT" --build-root "$BR_I" <"$V_THREE" >"$OUT" 2>"$ERR"; rc=$? +assert_rc "$rc" 0 "exit 0 reading the verdict stream from stdin" +assert_eq "$(staged_count "$BR_I")" 12 "stdin staged the same 12 debs" +if diff -q <(cd "$BR_A" && find . -type f | LC_ALL=C sort) <(cd "$BR_I" && find . -type f | LC_ALL=C sort) >/dev/null; then + ok "the stdin run staged exactly the same tree as the --verdicts run" +else + bad "stdin and --verdicts produced different trees" +fi + +echo "== (k) an AMBIGUOUS asset name fails closed rather than picking one ==" +ASSETS_K="$ROOT/assets-k" +cp -a "$ASSETS" "$ASSETS_K" +cp "$ASSETS_K/libqmi-glib5_1.38.0-1.ceralive1.1.0_amd64.deb" \ + "$ASSETS_K/libqmi-glib5_1.38.0-1~ceralive1.1.0_amd64.deb" +BR_K="$ROOT/build-k" +stage "$M_V2" "$ASSETS_K" "$BR_K" "$V_ALL"; rc=$? +assert_nz "$rc" "two assets matching one canonical name exits non-zero" +assert_err "is ambiguous" "stderr says the asset name is ambiguous" + +echo "== (l/m) a malformed verdict stream fails closed rather than assuming a default ==" +V_NOMODE="$ROOT/verdicts-no-mode.txt" +printf 'libqmi=unchanged\n' >"$V_NOMODE" +BR_L="$ROOT/build-l" +stage "$M_V2" "$ASSETS" "$BR_L" "$V_NOMODE"; rc=$? +assert_rc "$rc" 2 "a verdict stream with no mode= line exits 2" +assert_err "no 'mode=' line" "stderr names the missing mode line" +V_CONTRA="$ROOT/verdicts-contradiction.txt" +verdicts "$V_CONTRA" force-all libqrtr-glib=changed libmbim=unchanged libqmi=changed modemmanager=changed +stage "$M_V2" "$ASSETS" "$BR_L" "$V_CONTRA"; rc=$? +assert_rc "$rc" 2 "force-all carrying an 'unchanged' verdict exits 2" +assert_err "mode=force-all but source(s)" "stderr names the self-contradicting stream" +V_JUNK="$ROOT/verdicts-junk.txt" +printf 'libqmi=maybe\nmode=differential\n' >"$V_JUNK" +stage "$M_V2" "$ASSETS" "$BR_L" "$V_JUNK"; rc=$? +assert_rc "$rc" 2 "an unparseable verdict line exits 2" +assert_err "unparseable verdict line" "stderr names the bad line" +stage "$ROOT/no-such-manifest.txt" "$ASSETS" "$BR_L" "$V_ALL"; rc=$? +assert_rc "$rc" 2 "a set-but-unreadable PREV_MANIFEST_FILE exits 2" +assert_err "PREV_MANIFEST_FILE=" "stderr names the seam" + +echo "== (n) a re-run over an already-staged tree is idempotent ==" +stage "$M_V2" "$ASSETS" "$BR_A" "$V_THREE"; rc=$? +assert_rc "$rc" 0 "the second run exits 0" +assert_eq "$(staged_count "$BR_A")" 12 "the tree still holds exactly 12 debs" +assert_err "already staged, integrity matches" "the re-run reports the existing debs as integrity-matched" + +echo "== (o) a destination holding DIFFERENT bytes is never overwritten ==" +BR_O="$ROOT/build-o" +mkdir -p "$BR_O/amd64" +printf 'a freshly built deb that must not be clobbered\n' \ + >"$BR_O/amd64/libmbim-glib4_1.34.0-1~ceralive1.1.0_amd64.deb" +stage "$M_V2" "$ASSETS" "$BR_O" "$V_ALL"; rc=$? +assert_nz "$rc" "a differing destination file exits non-zero" +assert_err "already exists with different bytes" "stderr says the destination differs" +if grep -q 'must not be clobbered' "$BR_O/amd64/libmbim-glib4_1.34.0-1~ceralive1.1.0_amd64.deb"; then + ok "the pre-existing file was left untouched" +else + bad "the pre-existing file was overwritten" +fi + +echo +echo "passed: $pass failed: $fail" +if [ "$fail" -eq 0 ]; then + echo "PASS: carry-forward staging reconciles GitHub-mangled names, verifies bytes, and fails closed" + exit 0 +fi +echo "FAIL: stage-carryforward-debs contract violated" +exit 1 From 8b2f0a2b3407f26222d21eed3936b8c6607eb5f0 Mon Sep 17 00:00:00 2001 From: Andres Cera Date: Fri, 21 Aug 2026 18:46:58 -0500 Subject: [PATCH 3/7] feat(packaging): differential source builds with carried-forward bootstrap seeding --- packaging/ci/build-bookworm.sh | 587 ++++++++++++------ packaging/ci/inject-deb-version.sh | 186 ++++-- .../ci/test-build-bookworm-differential.sh | 325 ++++++++++ 3 files changed, 876 insertions(+), 222 deletions(-) create mode 100755 packaging/ci/test-build-bookworm-differential.sh diff --git a/packaging/ci/build-bookworm.sh b/packaging/ci/build-bookworm.sh index 73abde9..fb8c50c 100755 --- a/packaging/ci/build-bookworm.sh +++ b/packaging/ci/build-bookworm.sh @@ -1,150 +1,417 @@ #!/usr/bin/env bash -# build-bookworm.sh — rebuild the ModemManager stack for bookworm. +# build-bookworm.sh — differential ModemManager-stack rebuild for bookworm. # -# WHAT IT DOES -# Builds the four provenance-pinned sources (upstream-pins.yaml) in the mandatory -# bootstrap order libqrtr-glib -> libmbim -> libqmi -> modemmanager inside a -# `debian:bookworm` container. Each source's freshly built .debs are dropped into a -# temporary LOCAL apt repo (dpkg-scanpackages + `deb [trusted=yes] file:` line) so the -# NEXT source's build-deps resolve against the just-built package, not the older -# bookworm-main version. Real `dpkg-buildpackage` — no faking. +# BUILD SET +# BUILD_SOURCES may be an explicitly set space/comma-separated list of upstream-pins.yaml +# pin keys: libqrtr-glib libmbim libqmi modemmanager. An explicitly empty value selects zero +# sources. Alternatively, VERDICTS_FILE may point directly at detect-changed-sources.sh's +# five-line `=changed|unchanged` + `mode=` output. Supplying neither preserves the +# local-development default and builds all four sources. Supplying both fails closed. # -# ARCHES (native — never cross) -# amd64 -> `--platform linux/amd64`; arm64 -> `--platform linux/arm64` (full-system QEMU -# via the host's binfmt_misc `qemu-aarch64` handler — a genuine aarch64 userland, not a -# cross-compile). +# DIFFERENTIAL INVARIANT +# Selected sources build in the mandatory bootstrap order. Before the first build, every deb +# belonging to each skipped source is copied from the already-staged output directory into the +# temporary Pin-Priority 1001 local apt repo. A selected source therefore resolves stack +# build-deps and gir typelibs against carried CeraLive packages, never stock bookworm merely +# because its dependency source was skipped. # -# INPUTS (all under packaging/, this script's parent) -# /debian/ the pinned salsa debian/ dir + the bookworm adaptations (ModemManager -# only: debhelper relax, systemd-dev->udev, and the systemd/udev -# install-dir rules pins — see packaging/BOOKWORM-ADAPTATIONS.md). -# upstream-pins.yaml orig_tar_url / orig_tar_name / orig_tar_sha256 per source. -# ci/inject-deb-version.sh writes ~ceralive0.0.0~dev (dev build) into each changelog. +# OUTPUT + ASSERTIONS +# Carried .debs already staged by stage-carryforward-debs.sh are preserved. Only stale +# *.changes/*.buildinfo and artifacts belonging to sources selected for this run are removed. +# Freshly built sources retain the existing exact check-package-sets.sh verification from their +# .changes. The full runtime closure is asserted from the MERGED staged deb set (built + carried) +# at the end. A zero-source run starts no container and still performs that merged assertion. # -# OUTPUT -# Binary .debs + the four *.changes into $OUT (default: packaging/build/, gitignored). -# The runtime closure is asserted from the *.changes: EXACTLY the 9 runtime packages -# (modemmanager libmm-glib0 libmbim-{glib4,proxy,utils} libqmi-{glib5,proxy,utils} -# libqrtr-glib0). Any drift => exit 3 (STOP-and-surface). +# VERSIONING +# Release builds invoke inject-deb-version.sh once per selected source. It consumes the caller's +# already-resolved PREV_MANIFEST_FILE and derives that source's next ~ceralive.N counter; this +# script never fetches the previous manifest. Dev builds retain ~ceralive0.0.0~dev. # -# USAGE -# packaging/ci/build-bookworm.sh amd64 -# packaging/ci/build-bookworm.sh arm64 -# OUT=/some/dir packaging/ci/build-bookworm.sh amd64 # override output dir +# TEST SEAM +# BUILD_BOOKWORM_STUB_DIR is used only by test-build-bookworm-differential.sh. It replaces the +# expensive build body with source-keyed fixture artifacts while exercising production build-set +# parsing, seeding, dispatch, check-package-sets.sh, and merged-closure logic. # # EXIT -# 0 success (all 4 built, closure == the 9). 2 usage/env. 3 closure drift. non-zero build fail. +# 0 success. 2 usage/environment. 3 fail-closed package/carry/closure drift. Other non-zero is a +# real tooling or source-build failure. set -euo pipefail -# ------------------------------------------------------------------------------------------ -# Bootstrap order + the 9-package runtime closure are contract constants. BUILD_ORDER=(libqrtr-glib libmbim libqmi ModemManager) EXPECTED_RUNTIME=(libmbim-glib4 libmbim-proxy libmbim-utils libmm-glib0 libqmi-glib5 \ libqmi-proxy libqmi-utils libqrtr-glib0 modemmanager) -# Map a packaging dir name -> its upstream-pins.yaml source key (only ModemManager differs). +LOG_PREFIX="build-bookworm" +build_log() { printf '%s: %s\n' "$LOG_PREFIX" "$*" >&2; } +die() { + printf '%s: %s\n' "$LOG_PREFIX" "$*" >&2 + exit 2 +} +refuse() { + printf '%s: FAIL CLOSED — %s\n' "$LOG_PREFIX" "$*" >&2 + exit 3 +} + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT_PKG_ROOT="$(cd "$HERE/.." && pwd)" + +# Map a packaging directory name to its upstream-pins.yaml source key. Only ModemManager differs. pin_key() { case "$1" in ModemManager) echo modemmanager ;; *) echo "$1" ;; esac; } +is_known_key() { + local wanted="$1" dir + for dir in "${BUILD_ORDER[@]}"; do + [ "$(pin_key "$dir")" = "$wanted" ] && return 0 + done + return 1 +} + +expected_source_set() { # + awk -v want="[$1 all-artifact]" ' + /^\[/ { h=$0; sub(/[ \t]*#.*$/, "", h); insec=(h==want)?1:0; next } + insec { l=$0; sub(/#.*$/, "", l); gsub(/[ \t]+/, "", l); if (l!="") print l } + ' "$2" | LC_ALL=C sort -u +} + +declare -A SELECTED=() +BUILD_DIRS=() +BUILD_SOURCE_KEYS=() + +select_key() { + local key="$1" + is_known_key "$key" || die "unknown build source '$key' (expected pin-key names: libqrtr-glib libmbim libqmi modemmanager)" + [ -z "${SELECTED[$key]:-}" ] || die "build source '$key' was selected more than once" + SELECTED["$key"]=1 +} + +resolve_verdicts_file() { # + local verdicts_file="$1" line key value mode="" dir + local -A verdict=() + + [ -r "$verdicts_file" ] || die "VERDICTS_FILE='$verdicts_file' is set but not readable" + while IFS= read -r line || [ -n "$line" ]; do + [ -n "$line" ] || continue + case "$line" in + \#*) continue ;; + mode=*) + [ -z "$mode" ] || die "VERDICTS_FILE='$verdicts_file' contains more than one mode= line" + mode="${line#mode=}" + ;; + *=changed | *=unchanged) + key="${line%%=*}" + value="${line#*=}" + is_known_key "$key" || die "VERDICTS_FILE='$verdicts_file' names unknown source '$key'" + [ -z "${verdict[$key]:-}" ] || die "VERDICTS_FILE='$verdicts_file' verdicts source '$key' more than once" + verdict["$key"]="$value" + ;; + *) die "VERDICTS_FILE='$verdicts_file' has unparseable line '$line'" ;; + esac + done <"$verdicts_file" + + case "$mode" in + differential | force-all) ;; + "") die "VERDICTS_FILE='$verdicts_file' contains no mode= line" ;; + *) die "VERDICTS_FILE='$verdicts_file' has unrecognized mode '$mode'" ;; + esac + + for dir in "${BUILD_ORDER[@]}"; do + key="$(pin_key "$dir")" + [ -n "${verdict[$key]:-}" ] || die "VERDICTS_FILE='$verdicts_file' contains no verdict for source '$key'" + if [ "$mode" = force-all ] && [ "${verdict[$key]}" != changed ]; then + die "VERDICTS_FILE='$verdicts_file' says mode=force-all but source '$key' is '${verdict[$key]}'" + fi + if [ "${verdict[$key]}" = changed ]; then select_key "$key"; fi + done + build_log "build set read from VERDICTS_FILE='$verdicts_file' (mode=$mode)" +} + +resolve_build_set() { + local normalized token dir key + local requested=() + + if [ -n "${VERDICTS_FILE:-}" ] && [ "${BUILD_SOURCES+x}" = x ]; then + die "BUILD_SOURCES and VERDICTS_FILE are mutually exclusive; pass one build-set contract" + fi + + if [ -n "${VERDICTS_FILE:-}" ]; then + resolve_verdicts_file "$VERDICTS_FILE" + elif [ "${BUILD_SOURCES+x}" = x ]; then + normalized="${BUILD_SOURCES//,/ }" + if [[ "$normalized" =~ [^[:space:]] ]]; then + read -r -a requested <<<"$normalized" + fi + for token in "${requested[@]}"; do select_key "$token"; done + build_log "build set read from BUILD_SOURCES='${BUILD_SOURCES}'" + else + for dir in "${BUILD_ORDER[@]}"; do select_key "$(pin_key "$dir")"; done + build_log "no build-set input supplied; defaulting to all sources" + fi + + for dir in "${BUILD_ORDER[@]}"; do + key="$(pin_key "$dir")" + if [ -n "${SELECTED[$key]:-}" ]; then + BUILD_DIRS+=("$dir") + BUILD_SOURCE_KEYS+=("$key") + fi + done +} + +is_selected() { [ -n "${SELECTED[$1]:-}" ]; } + +assert_merged_runtime_closure() { # + local staged_dir="$1" deb filename package expected got + local debs=() runtime=() runtime_sorted=() + local -A seen=() + + shopt -s nullglob + debs=("$staged_dir"/*.deb) + shopt -u nullglob + for deb in "${debs[@]}"; do + filename="$(basename "$deb")" + case "$filename" in + *_*.deb) package="${filename%%_*}" ;; + *) refuse "merged staged file '$filename' is not a canonical '__.deb' name" ;; + esac + case "$package" in + *-dev | *-doc | *-dbgsym | gir1.2-*) continue ;; + esac + if [ -n "${seen[$package]:-}" ]; then + refuse "merged runtime set contains more than one staged deb for package '$package' ('$filename' and '${seen[$package]}')" + fi + seen["$package"]="$filename" + runtime+=("$package") + done + + if [ "${#runtime[@]}" -gt 0 ]; then + mapfile -t runtime_sorted < <(printf '%s\n' "${runtime[@]}" | LC_ALL=C sort -u) + fi + expected="$(printf '%s\n' "${EXPECTED_RUNTIME[@]}" | LC_ALL=C sort -u)" + got="$(printf '%s\n' "${runtime_sorted[@]:-}" | sed '/^$/d')" + + if [ "$expected" != "$got" ]; then + printf '%s: FAIL CLOSED — merged runtime closure drift in %s\n' "$LOG_PREFIX" "$staged_dir" >&2 + comm -23 <(printf '%s\n' "$expected") <(printf '%s\n' "$got") \ + | sed 's/^/ MISSING (expected, not staged): /' >&2 || true + comm -13 <(printf '%s\n' "$expected") <(printf '%s\n' "$got") \ + | sed 's/^/ UNEXPECTED (staged, not expected): /' >&2 || true + exit 3 + fi + build_log "MERGED RUNTIME CLOSURE OK: ${#EXPECTED_RUNTIME[@]} expected packages across built + carried debs in '$staged_dir'" +} + +clean_run_outputs() { # + local out_dir="$1" expected_file="$2" key package source_set + rm -f "$out_dir"/*.changes "$out_dir"/*.buildinfo 2>/dev/null || true + for key in "${BUILD_SOURCE_KEYS[@]}"; do + source_set="$(expected_source_set "$key" "$expected_file")" + [ -n "$source_set" ] || die "source '$key' has no [$key all-artifact] block in '$expected_file'" + while IFS= read -r package; do + [ -n "$package" ] || continue + rm -f "$out_dir/${package}_"*.deb + done <<<"$source_set" + build_log "cleared stale artifacts for selected source '$key'; carried debs for skipped sources were preserved" + done +} + +resolve_build_set +CANONICAL_BUILD_SOURCES="${BUILD_SOURCE_KEYS[*]}" + # ========================================================================================== -# HOST ROLE — arg parse, launch the container, then post-process the results it wrote. +# HOST ROLE — preserve staged carries, launch selected builds, or prove a zero-build closure. # ========================================================================================== -if [ "${BUILD_IN_CONTAINER:-0}" != "1" ]; then +if [ "${BUILD_IN_CONTAINER:-0}" != 1 ]; then ARCH="${1:-}" case "$ARCH" in - amd64) PLATFORM="linux/amd64" ;; - arm64) PLATFORM="linux/arm64" ;; - *) echo "usage: build-bookworm.sh " >&2; exit 2 ;; + amd64) PLATFORM="linux/amd64" ;; + arm64) PLATFORM="linux/arm64" ;; + *) echo "usage: build-bookworm.sh " >&2; exit 2 ;; esac - command -v docker >/dev/null 2>&1 || { echo "build-bookworm: docker not found" >&2; exit 2; } - - HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - PKG_ROOT="$(cd "$HERE/.." && pwd)" + PKG_ROOT="$SCRIPT_PKG_ROOT" OUT="${OUT:-$PKG_ROOT/build/$ARCH}" mkdir -p "$OUT" - # Clean any prior artifacts so the closure check sees only this run's output. - rm -f "$OUT"/*.deb "$OUT"/*.changes "$OUT"/*.buildinfo 2>/dev/null || true - - echo "build-bookworm: arch=$ARCH platform=$PLATFORM" - echo "build-bookworm: packaging root=$PKG_ROOT" - echo "build-bookworm: output=$OUT" - - # Mount packaging/ read-only at /pkg; output read-write at /out. Re-invoke self in-container. - # RELEASE_VERSION (optional): a vX.Y.Z tag → build ~ceraliveX.Y.Z debs. Unset → dev build - # (~ceralive0.0.0~dev). release.yml passes the release tag here; local/CI dev builds omit it. - docker run --rm --platform "$PLATFORM" \ - -e BUILD_IN_CONTAINER=1 \ - -e ARCH="$ARCH" \ - -e RELEASE_VERSION="${RELEASE_VERSION:-}" \ - -v "$PKG_ROOT":/pkg:ro \ - -v "$OUT":/out \ - debian:bookworm \ + clean_run_outputs "$OUT" "$PKG_ROOT/ci/expected-packages.txt" + + build_log "arch=$ARCH platform=$PLATFORM" + build_log "packaging root=$PKG_ROOT" + build_log "output=$OUT" + build_log "selected sources (${#BUILD_SOURCE_KEYS[@]}): ${CANONICAL_BUILD_SOURCES:-none}" + + if [ "${#BUILD_SOURCE_KEYS[@]}" -eq 0 ]; then + build_log "zero-build path: no source selected; no container will be started" + assert_merged_runtime_closure "$OUT" + build_log "PASS [$ARCH]: zero sources built; carried-only merged runtime closure is complete" + exit 0 + fi + + command -v docker >/dev/null 2>&1 || die "docker not found" + if [ -n "${PREV_MANIFEST_FILE:-}" ] && [ ! -r "$PREV_MANIFEST_FILE" ]; then + die "PREV_MANIFEST_FILE='$PREV_MANIFEST_FILE' is set but not readable" + fi + if [ -n "${RELEASE_VERSION:-}" ] && [ -z "${PREV_MANIFEST_FILE:-}" ] && + [ "${#BUILD_SOURCE_KEYS[@]}" -ne "${#BUILD_ORDER[@]}" ]; then + die "release differential build selects ${#BUILD_SOURCE_KEYS[@]} source(s) but PREV_MANIFEST_FILE is absent; only a force-all bootstrap may initialize counters without a previous manifest" + fi + + docker_args=( + run --rm --platform "$PLATFORM" + -e BUILD_IN_CONTAINER=1 + -e ARCH="$ARCH" + -e BUILD_SOURCES="$CANONICAL_BUILD_SOURCES" + -e RELEASE_VERSION="${RELEASE_VERSION:-}" + ) + if [ -n "${PREV_MANIFEST_FILE:-}" ]; then + docker_args+=( + -e PREV_MANIFEST_FILE=/previous-release-manifest.txt + -v "$PREV_MANIFEST_FILE":/previous-release-manifest.txt:ro + ) + fi + docker_args+=( + -v "$PKG_ROOT":/pkg:ro + -v "$OUT":/out + debian:bookworm bash /pkg/ci/build-bookworm.sh "$ARCH" + ) + docker "${docker_args[@]}" - echo "build-bookworm: container finished; artifacts in $OUT" + build_log "container finished; merged artifacts are in '$OUT'" exit 0 fi # ========================================================================================== -# CONTAINER ROLE — the real build, inside debian:bookworm. +# CONTAINER ROLE — seed skipped sources, then build only selected sources in bootstrap order. # ========================================================================================== -ARCH="${ARCH:-$(dpkg --print-architecture)}" -echo "== in-container build (arch=$(dpkg --print-architecture), target=$ARCH, $(uname -m)) ==" +PKG_MOUNT="${BUILD_BOOKWORM_PKG_ROOT:-/pkg}" +OUT_DIR="${BUILD_BOOKWORM_OUT_DIR:-/out}" +STUB_BUILD_DIR="${BUILD_BOOKWORM_STUB_DIR:-}" +[ -d "$PKG_MOUNT" ] || die "container packaging root '$PKG_MOUNT' is not a directory" +[ -d "$OUT_DIR" ] || die "container output '$OUT_DIR' is not a directory" +if [ -n "$STUB_BUILD_DIR" ] && [ ! -d "$STUB_BUILD_DIR" ]; then + die "BUILD_BOOKWORM_STUB_DIR='$STUB_BUILD_DIR' is set but not a directory" +fi -export DEBIAN_FRONTEND=noninteractive -# nocheck: skip the upstream test phase (needs a live session bus; that is A5.2 daemon-smoke, -# not a build-time concern). nodoc: skip gtk-doc (arch:all -doc pkgs are not in the -# runtime closure). Both are standard for a binary rebuild. -NPROC="$(nproc)" -export DEB_BUILD_OPTIONS="nocheck nodoc parallel=$NPROC" -# dch (version injection) needs a maintainer identity; the container has none by default. -export DEBEMAIL="ci@ceralive.tv" -export DEBFULLNAME="CeraLive CI" +ARCH="${ARCH:-}" +if [ -z "$ARCH" ]; then + [ -z "$STUB_BUILD_DIR" ] || die "ARCH is required with BUILD_BOOKWORM_STUB_DIR" + ARCH="$(dpkg --print-architecture)" +fi log() { echo " [build] $*"; } step() { echo; echo "==== $* ===="; } -# apt drops to the unprivileged `_apt` user for acquire, which cannot read the local -# file: repo under a 0700 mktemp dir — turn the sandbox off (standard container fix). -echo 'APT::Sandbox::User "root";' > /etc/apt/apt.conf.d/01-no-sandbox +if [ "${#BUILD_SOURCE_KEYS[@]}" -eq 0 ]; then + build_log "zero-build container role: no source selected; skipping all tooling and build setup" + assert_merged_runtime_closure "$OUT_DIR" + exit 0 +fi + +if [ -n "$STUB_BUILD_DIR" ]; then + echo "== stubbed in-container build (target=$ARCH) ==" + log "BUILD_BOOKWORM_STUB_DIR active: real apt and dpkg-buildpackage commands are disabled" +else + echo "== in-container build (arch=$(dpkg --print-architecture), target=$ARCH, $(uname -m)) ==" +fi + +export DEBIAN_FRONTEND=noninteractive +if [ -n "$STUB_BUILD_DIR" ]; then + NPROC=1 +else + NPROC="$(nproc)" +fi +export DEB_BUILD_OPTIONS="nocheck nodoc parallel=$NPROC" +export DEBEMAIL="ci@ceralive.tv" +export DEBFULLNAME="CeraLive CI" -step "install build tooling" -apt-get update -qq -apt-get install -y -qq --no-install-recommends \ - build-essential dpkg-dev devscripts equivs \ - meson ninja-build pkgconf ca-certificates curl xz-utils bzip2 >/dev/null -DPKGBP_VER="$(dpkg-buildpackage --version 2>/dev/null | sed -n '1p' || true)" -log "toolchain ready: $DPKGBP_VER" +if [ -z "$STUB_BUILD_DIR" ]; then + # apt drops to `_apt`, which cannot read a file: repo under mktemp's 0700 directory. + echo 'APT::Sandbox::User "root";' > /etc/apt/apt.conf.d/01-no-sandbox + step "install build tooling" + apt-get update -qq + apt-get install -y -qq --no-install-recommends \ + build-essential dpkg-dev devscripts equivs \ + meson ninja-build pkgconf ca-certificates curl xz-utils bzip2 >/dev/null + DPKGBP_VER="$(dpkg-buildpackage --version 2>/dev/null | sed -n '1p' || true)" + log "toolchain ready: $DPKGBP_VER" +else + step "install build tooling (stubbed)" + log "toolchain stub ready" +fi -# ---- writable copy of the packaging tree (source of truth is the ro mount) --------------- -WORK="$(mktemp -d /tmp/mmbuild.XXXXXX)" +WORK="$(mktemp -d "${TMPDIR:-/tmp}/mmbuild.XXXXXX")" +cleanup() { rm -rf "$WORK"; } +trap cleanup EXIT PKGW="$WORK/pkg" -cp -a /pkg "$PKGW" -REPO="$WORK/repo" # the temporary LOCAL apt repo +cp -a "$PKG_MOUNT" "$PKGW" +REPO="$WORK/repo" mkdir -p "$REPO" -: > "$REPO/Packages" # empty index so `apt-get update` is happy before build 1 -echo "deb [trusted=yes] file:$REPO ./" > /etc/apt/sources.list.d/local-mm.list -# Prefer the freshly built local packages over bookworm-main for the stack's own libs. -cat > /etc/apt/preferences.d/local-mm.pref <<'EOF' +: >"$REPO/Packages" + +if [ -z "$STUB_BUILD_DIR" ]; then + echo "deb [trusted=yes] file:$REPO ./" > /etc/apt/sources.list.d/local-mm.list + cat > /etc/apt/preferences.d/local-mm.pref <<'EOF' Package: * Pin: origin "" Pin-Priority: 1001 EOF -apt-get update -qq + apt-get update -qq +fi refresh_repo() { - ( cd "$REPO" && dpkg-scanpackages -m . /dev/null > Packages 2>/dev/null ) - apt-get update -qq + if [ -n "$STUB_BUILD_DIR" ]; then + find "$REPO" -maxdepth 1 -type f -name '*.deb' -printf '%f\n' | LC_ALL=C sort >"$REPO/Packages" + else + (cd "$REPO" && dpkg-scanpackages -m . /dev/null >Packages 2>/dev/null) + apt-get update -qq + fi } -# ---- version injection: A1.1's script, run against the writable packaging copy ----------- -# RELEASE_VERSION set (a vX.Y.Z tag) → inject ~ceraliveX.Y.Z; unset → the dev suffix. Either -# way this runs against the WRITABLE copy ($PKGW), never the committed source-of-truth tree. -INJECT_ARG="${RELEASE_VERSION:-}"; [ -n "$INJECT_ARG" ] || INJECT_ARG="--dev" -step "inject version ($INJECT_ARG) via ci/inject-deb-version.sh" -( cd "$PKGW" && bash ci/inject-deb-version.sh "$INJECT_ARG" ) +seed_carried_source() { # + local source_name="$1" package source_set count=0 + local matches=() + source_set="$(expected_source_set "$source_name" "$PKGW/ci/expected-packages.txt")" + [ -n "$source_set" ] || die "source '$source_name' has no [$source_name all-artifact] block in '$PKGW/ci/expected-packages.txt'" + + while IFS= read -r package; do + [ -n "$package" ] || continue + shopt -s nullglob + matches=("$OUT_DIR/${package}_"*.deb) + shopt -u nullglob + if [ "${#matches[@]}" -eq 0 ]; then + refuse "skipped source '$source_name' carried package '$package' is missing from '$OUT_DIR'; build-deps must never fall back to stock bookworm" + fi + if [ "${#matches[@]}" -gt 1 ]; then + refuse "skipped source '$source_name' carried package '$package' has ${#matches[@]} staged debs in '$OUT_DIR'; the local repo seed requires exactly one version" + fi + cp "${matches[0]}" "$REPO/" + count=$((count + 1)) + done <<<"$source_set" + log "seed $source_name: copied $count carried deb(s) into the local apt repo" +} + +step "seed local apt repo with carried debs for skipped sources" +seeded_sources=0 +for dir in "${BUILD_ORDER[@]}"; do + key="$(pin_key "$dir")" + if ! is_selected "$key"; then + seed_carried_source "$key" + seeded_sources=$((seeded_sources + 1)) + fi +done +if [ "$seeded_sources" -gt 0 ]; then + refresh_repo + log "carried-deb seed complete before first BUILD; Pin-Priority 1001 local repo contains every skipped source" +else + log "no skipped source; local apt repo starts empty and will be populated in bootstrap order" +fi + +INJECT_ARG="${RELEASE_VERSION:-}" +[ -n "$INJECT_ARG" ] || INJECT_ARG="--dev" +for dir in "${BUILD_DIRS[@]}"; do + key="$(pin_key "$dir")" + step "inject version for $key ($INJECT_ARG)" + (cd "$PKGW" && bash ci/inject-deb-version.sh --source "$key" "$INJECT_ARG") +done -# ---- tiny pin reader (same awk shape as verify-upstream-pins.sh) ------------------------- pin_scalar() { awk -v src="$1" -v key="$2" ' $0 ~ "^ " src ":[ \t]*$" { inblk=1; next } @@ -156,120 +423,94 @@ pin_scalar() { ' "$PKGW/upstream-pins.yaml" } -# ---- build one source ------------------------------------------------------------------- build_one() { - local dir="$1" key; key="$(pin_key "$dir")" + local dir="$1" key + key="$(pin_key "$dir")" step "BUILD $dir (pin key: $key)" - # Resolve the injected source + upstream version from the (now version-injected) changelog. + if [ -n "$STUB_BUILD_DIR" ]; then + local fixture="$STUB_BUILD_DIR/$key" file + local fixture_debs=() fixture_changes=() fixture_buildinfo=() + [ -d "$fixture" ] || die "stub build for source '$key' has no fixture directory '$fixture'" + shopt -s nullglob + fixture_debs=("$fixture"/*.deb) + fixture_changes=("$fixture"/*.changes) + fixture_buildinfo=("$fixture"/*.buildinfo) + shopt -u nullglob + [ "${#fixture_debs[@]}" -gt 0 ] || die "stub build for source '$key' contains no .deb" + [ "${#fixture_changes[@]}" -gt 0 ] || die "stub build for source '$key' contains no .changes" + for file in "${fixture_debs[@]}"; do cp "$file" "$OUT_DIR/"; cp "$file" "$REPO/"; done + for file in "${fixture_changes[@]}"; do cp "$file" "$OUT_DIR/"; done + for file in "${fixture_buildinfo[@]}"; do cp "$file" "$OUT_DIR/"; done + refresh_repo + log "$dir stub-built; fixture artifacts entered output + local repo" + return 0 + fi + local src ver upstream src="$(dpkg-parsechangelog -l "$PKGW/$dir/debian/changelog" -S Source)" ver="$(dpkg-parsechangelog -l "$PKGW/$dir/debian/changelog" -S Version)" - upstream="${ver%%-*}" # strip -~ceralive... + upstream="${ver%%-*}" log "source=$src version=$ver upstream=$upstream" - # Fetch + verify the provenance-pinned orig tarball. local url name sha url="$(pin_scalar "$key" orig_tar_url)" name="$(pin_scalar "$key" orig_tar_name)" sha="$(pin_scalar "$key" orig_tar_sha256)" - [ -n "$url" ] && [ -n "$name" ] && [ -n "$sha" ] || { echo "missing pin for $key" >&2; exit 2; } + if [ -z "$url" ] || [ -z "$name" ] || [ -z "$sha" ]; then + die "missing pin for source '$key'" + fi log "orig: $name" curl -fsSL --retry 3 --retry-delay 2 -o "$WORK/$name" "$url" - local got; got="$(sha256sum "$WORK/$name" | awk '{print $1}')" - [ "$got" = "$sha" ] || { echo "STOP: orig sha256 drift for $name (pin $sha, got $got)" >&2; exit 3; } + local got + got="$(sha256sum "$WORK/$name" | awk '{print $1}')" + [ "$got" = "$sha" ] || refuse "orig sha256 drift for '$name' (source '$key': pin $sha, got $got)" log "orig sha256 OK ($sha)" - # Assemble the build tree: -/ with debian/ overlaid; orig in the parent. local bdir="$WORK/build" mkdir -p "$bdir" local tree="$bdir/${src}-${upstream}" - rm -rf "$tree"; mkdir -p "$tree" + rm -rf "$tree" + mkdir -p "$tree" tar -xf "$WORK/$name" -C "$tree" --strip-components=1 cp -a "$PKGW/$dir/debian" "$tree/debian" - # Non-native 3.0 (quilt): dpkg-source wants ../_.orig.tar.. cp "$WORK/$name" "$bdir/${src}_${upstream}.orig.${name#*.orig.}" - # Resolve build-deps against bookworm-main + the local repo (freshly built deps). - log "apt-get build-dep (resolves against local repo for stack deps)" + log "apt-get build-dep (Pin-Priority 1001 local repo supplies built + carried stack deps)" apt-get build-dep -y --no-install-recommends "$tree" >/dev/null - # dh_girepository (bookworm GI 1.74) resolves cross-namespace typelib deps (Qmi-1.0 imports - # Qrtr-1.0; ModemManager-1.0 imports Qmi/Mbim/Qrtr) via the *installed* dependency .typelib, - # which ships only in the stack's own gir1.2-*-1.0 packages. On bookworm the rebuilt -dev - # build-deps do NOT pull them (GI 1.74 regenerates an empty ${gir:Depends} for -dev), so - # libqmi dies with "Could not find Qrtr-1.0.typelib dependency". Install every gir typelib - # already in the local repo first — mirrors an archive build (all gir1.2-* co-installable); - # affects only the build env, never a produced package's contents or the emitted set. local gir_names gir_names="$(find "$REPO" -maxdepth 1 -name 'gir1.2-*.deb' -printf '%f\n' 2>/dev/null \ | sed 's/_.*//' | sort -u | tr '\n' ' ')" if [ -n "${gir_names// /}" ]; then - log "install freshly-built gir typelibs so dh_girepository resolves them: $gir_names" + log "install local gir typelibs so dh_girepository resolves them: $gir_names" + # shellcheck disable=SC2086 # package names are intentionally split for apt-get argv apt-get install -y --no-install-recommends $gir_names >/dev/null fi - # Real binary build, arch-only (-B): all 9 runtime pkgs are arch-specific; -B skips the - # arch:all -doc pkgs and the -indep DEP-8 patch target. log "dpkg-buildpackage -B (DEB_BUILD_OPTIONS='$DEB_BUILD_OPTIONS')" - ( cd "$tree" && dpkg-buildpackage -B -us -uc ) + (cd "$tree" && dpkg-buildpackage -B -us -uc) - # dpkg-buildpackage writes artifacts to $bdir (the source tree's PARENT); prior sources' - # debs were already moved to $REPO, so $bdir holds only this source's fresh output. - find "$bdir" -maxdepth 1 -name '*.changes' -exec cp -t /out {} + 2>/dev/null || true - find "$bdir" -maxdepth 1 -name '*.buildinfo' -exec cp -t /out {} + 2>/dev/null || true - find "$bdir" -maxdepth 1 -name '*.deb' -exec cp -t /out {} + 2>/dev/null || true - find "$bdir" -maxdepth 1 -name '*.deb' -exec mv -t "$REPO" {} + 2>/dev/null || true + find "$bdir" -maxdepth 1 -name '*.changes' -exec cp -t "$OUT_DIR" {} + 2>/dev/null || true + find "$bdir" -maxdepth 1 -name '*.buildinfo' -exec cp -t "$OUT_DIR" {} + 2>/dev/null || true + find "$bdir" -maxdepth 1 -name '*.deb' -exec cp -t "$OUT_DIR" {} + 2>/dev/null || true + find "$bdir" -maxdepth 1 -name '*.deb' -exec mv -t "$REPO" {} + 2>/dev/null || true refresh_repo - log "$dir built; local repo now has $(ls "$REPO"/*.deb 2>/dev/null | wc -l) .deb(s)" + local repo_debs=() + shopt -s nullglob + repo_debs=("$REPO"/*.deb) + shopt -u nullglob + log "$dir built; local repo now has ${#repo_debs[@]} .deb(s)" } -for d in "${BUILD_ORDER[@]}"; do build_one "$d"; done - -# ---- runtime-closure verification from the four *.changes -------------------------------- -step "runtime closure verification (from *.changes)" -# All binary package names across the 4 binary .changes (RFC822 Binary: field, fold-safe). -mapfile -t ALL_BINS < <( - for ch in /out/*.changes; do - awk ' - /^[A-Za-z][A-Za-z0-9-]*:/ { inb=0 } - /^Binary:/ { inb=1; l=$0; sub(/^Binary:[ \t]*/, "", l); print l; next } - inb && /^[ \t]/ { l=$0; sub(/^[ \t]+/, "", l); print l } - ' "$ch" - done | tr ' ' '\n' | sed '/^$/d' | sort -u -) -# Runtime = not -dev, not -doc, not an auto-generated -dbgsym, not a gir typelib. -RUNTIME=() -for b in "${ALL_BINS[@]}"; do - case "$b" in - *-dev|*-doc|*-dbgsym|gir1.2-*) : ;; - *) RUNTIME+=("$b") ;; - esac -done -mapfile -t RUNTIME_SORTED < <(printf '%s\n' "${RUNTIME[@]}" | sort -u) - -echo "all binary packages produced:"; printf ' %s\n' "${ALL_BINS[@]}" -echo "runtime closure (dev/doc/gir excluded):"; printf ' %s\n' "${RUNTIME_SORTED[@]}" +for dir in "${BUILD_DIRS[@]}"; do build_one "$dir"; done -expected="$(printf '%s\n' "${EXPECTED_RUNTIME[@]}" | sort -u)" -got="$(printf '%s\n' "${RUNTIME_SORTED[@]}")" -if [ "$expected" = "$got" ]; then - echo "CLOSURE OK: exactly the 9 expected runtime packages." -else - echo "STOP: runtime closure drift." >&2 - echo "--- expected ---" >&2; printf '%s\n' "$expected" >&2 - echo "--- got ---" >&2; printf '%s\n' "$got" >&2 - echo "--- diff (want<->got) ---" >&2; diff <(printf '%s\n' "$expected") <(printf '%s\n' "$got") >&2 || true - exit 3 -fi +step "fresh-source package-set equality (selected sources only, from *.changes)" +bash "$PKGW/ci/check-package-sets.sh" "$OUT_DIR" "$PKGW/ci/expected-packages.txt" -# ---- exact per-source package-set EQUALITY (finalized two-set model) ---------------------- -# Stronger than the runtime closure above: asserts EVERY source's *.changes binary set equals -# its frozen [ all-artifact] set in expected-packages.txt, exactly (add/remove/rename -# all fail). check-package-sets.sh fails closed and names the discrepancy. -step "package-set equality verification (per source+arch, from *.changes)" -bash "$PKGW/ci/check-package-sets.sh" /out "$PKGW/ci/expected-packages.txt" +step "merged runtime closure verification (freshly built + carried debs)" +assert_merged_runtime_closure "$OUT_DIR" echo -echo "PASS [$ARCH]: 4 sources built in bootstrap order; runtime closure == the 9; per-source .changes sets EQUAL the finalized all-artifact expectation." +echo "PASS [$ARCH]: ${#BUILD_SOURCE_KEYS[@]} selected source(s) built in bootstrap order; skipped-source debs seeded before BUILD; merged runtime closure complete." diff --git a/packaging/ci/inject-deb-version.sh b/packaging/ci/inject-deb-version.sh index 0c83f76..fa55a59 100755 --- a/packaging/ci/inject-deb-version.sh +++ b/packaging/ci/inject-deb-version.sh @@ -1,69 +1,157 @@ #!/usr/bin/env bash -# inject-deb-version.sh — write the CeraLive-encoded version into each source's -# debian/changelog top entry. +# inject-deb-version.sh — inject one selected source's CeraLive rebuild version. # -# Encoded version: -~ceralive (see docs/VERSIONING.md). -# The tilde makes it sort BELOW the pinned - changelog top, so plain -# `dch --newversion` REFUSES it (dch(1) rejects a lower-than-current version). Hence -# `--force-bad-version` is REQUIRED, not optional. All four sources take the same suffix. +# Release suffixes are per-source counters: -~ceralive.N. N is derived from +# EVERY previous-manifest row for that source (both arches, runtime and aux), and is accepted +# only when those rows are coherent. Entirely legacy rows (~ceraliveX.Y.Z), or an absent previous +# manifest during the force-all bootstrap, initialize at .1. Non-tag builds keep the fixed +# ~ceralive0.0.0~dev suffix. # # Usage: -# inject-deb-version.sh vX.Y.Z # release: suffix ~ceraliveX.Y.Z -# inject-deb-version.sh X.Y.Z # same (leading v optional) -# inject-deb-version.sh --dev # non-tag CI: fixed suffix ~ceralive0.0.0~dev +# PREV_MANIFEST_FILE=release-manifest.txt \ +# inject-deb-version.sh --source libqmi vX.Y.Z +# inject-deb-version.sh --source modemmanager --dev # -# The 4 repackaged sources are ModemManager, libmbim, libqmi, libqrtr-glib. Their upstream -# versions are NOT hardcoded here — each is read from that source's own debian/changelog -# top entry (added in the packaging wave). Until those recipes exist, this script documents -# the exact `dch` invocation per source and exits 0. +# Source names are upstream-pins.yaml pin keys. The ModemManager recipe-directory mapping is +# deliberately non-identity. The previous manifest is an input seam supplied by the caller; this +# script never resolves or downloads a release independently. set -euo pipefail +LOG_PREFIX="inject-deb-version" +log() { printf '%s: %s\n' "$LOG_PREFIX" "$*" >&2; } +die() { + printf '%s: %s\n' "$LOG_PREFIX" "$*" >&2 + exit 2 +} +refuse() { + printf '%s: FAIL CLOSED — %s\n' "$LOG_PREFIX" "$*" >&2 + exit 3 +} + HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PKG_ROOT="$(cd "$HERE/.." && pwd)" -# Names only — versions/pins are provenance-verified in a later task, never hardcoded here. -SOURCES=(ModemManager libmbim libqmi libqrtr-glib) +SOURCE_KEYS=(libqrtr-glib libmbim libqmi modemmanager) +recipe_dir() { case "$1" in modemmanager) echo ModemManager ;; *) echo "$1" ;; esac; } -arg="${1-}" -if [ -z "$arg" ]; then - echo "usage: inject-deb-version.sh " >&2 - exit 2 -fi +usage() { + echo "usage: inject-deb-version.sh --source " +} + +SOURCE="" +VERSION_ARG="" +while [ $# -gt 0 ]; do + case "$1" in + --source) + shift + [ $# -gt 0 ] || die "--source requires a pin-key name" + [ -z "$SOURCE" ] || die "--source was supplied more than once" + SOURCE="$1" + ;; + --source=*) + [ -z "$SOURCE" ] || die "--source was supplied more than once" + SOURCE="${1#--source=}" + ;; + -h | --help) + usage + exit 0 + ;; + --dev | v* | [0-9]*) + [ -z "$VERSION_ARG" ] || die "version argument was supplied more than once ('$VERSION_ARG' then '$1')" + VERSION_ARG="$1" + ;; + *) die "unknown argument '$1'" ;; + esac + shift +done + +[ -n "$SOURCE" ] || die "--source is required (version injection is per-source and may only touch a source being built)" +[ -n "$VERSION_ARG" ] || die "a release version or --dev is required" + +known=0 +for key in "${SOURCE_KEYS[@]}"; do + if [ "$key" = "$SOURCE" ]; then known=1; break; fi +done +[ "$known" -eq 1 ] || die "unknown source '$SOURCE' (expected one of: ${SOURCE_KEYS[*]})" + +derive_release_suffix() { # + local source_name="$1" manifest version kind="" counter="" next + local versions=() -if [ "$arg" = "--dev" ]; then + if [ -z "${PREV_MANIFEST_FILE:-}" ]; then + log "source '$source_name': no previous manifest supplied (force-all bootstrap); initializing counter at .1" + printf '%s\n' '~ceralive.1' + return 0 + fi + + manifest="$PREV_MANIFEST_FILE" + [ -r "$manifest" ] || + die "source '$source_name': PREV_MANIFEST_FILE='$manifest' is set but not readable" + + mapfile -t versions < <( + awk -v source_name="$source_name" \ + '$0 !~ /^#/ && NF==7 && $1 !~ /:$/ && $3==source_name { print $4 }' "$manifest" + ) + if [ "${#versions[@]}" -eq 0 ]; then + refuse "source '$source_name' is being rebuilt but previous manifest '$manifest' contains ZERO rows for it; no counter can be derived" + fi + + for version in "${versions[@]}"; do + if [[ "$version" =~ ~ceralive\.([1-9][0-9]*)$ ]]; then + if [ "$kind" = legacy ]; then + refuse "source '$source_name' mixes counter and legacy suffixes across its previous-manifest rows (encountered '$version')" + fi + kind=counter + if [ -z "$counter" ]; then + counter="${BASH_REMATCH[1]}" + elif [ "$counter" != "${BASH_REMATCH[1]}" ]; then + refuse "source '$source_name' has differing counters in its previous-manifest rows (.$counter vs .${BASH_REMATCH[1]}); refusing to risk a downgrade" + fi + elif [[ "$version" =~ ~ceralive[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + if [ "$kind" = counter ]; then + refuse "source '$source_name' mixes counter and legacy suffixes across its previous-manifest rows (encountered '$version')" + fi + kind=legacy + else + refuse "source '$source_name' has malformed previous-manifest version '$version'; expected terminal '~ceralive.[1-9][0-9]*' or legacy '~ceraliveX.Y.Z'" + fi + done + + if [ "$kind" = legacy ]; then + log "source '$source_name': all ${#versions[@]} previous-manifest row(s) use the legacy suffix; initializing counter at .1" + printf '%s\n' '~ceralive.1' + return 0 + fi + + next=$((10#$counter + 1)) + log "source '$source_name': all ${#versions[@]} previous-manifest row(s) agree on counter .$counter; next counter is .$next" + printf '~ceralive.%s\n' "$next" +} + +if [ "$VERSION_ARG" = --dev ]; then SUFFIX="~ceralive0.0.0~dev" else - # Reuse the tag guard so a bad version is rejected identically everywhere. + # Reuse the tag guard so release invocation syntax is rejected identically everywhere. The + # tag no longer supplies the suffix: release provenance lives in the manifest, while the + # package version records this source's rebuild count. # shellcheck source=./tag-guard.sh source "$HERE/tag-guard.sh" - case "$arg" in - v*) XYZ="$(validate_tag "$arg")" ;; - *) XYZ="$(validate_tag "v$arg")" ;; + case "$VERSION_ARG" in + v*) validate_tag "$VERSION_ARG" >/dev/null ;; + *) validate_tag "v$VERSION_ARG" >/dev/null ;; esac - SUFFIX="~ceralive${XYZ}" + SUFFIX="$(derive_release_suffix "$SOURCE")" fi -echo "inject-deb-version: suffix ${SUFFIX}" - -injected=0 -pending=0 -for src in "${SOURCES[@]}"; do - changelog="$PKG_ROOT/$src/debian/changelog" - if [ -f "$changelog" ]; then - # Derive - from the top entry, strip any prior ~ceralive suffix, - # then append ours. - top="$(dpkg-parsechangelog -l "$changelog" -S Version)" - base="${top%%~ceralive*}" - version="${base}${SUFFIX}" - echo " ${src}: ${top} -> ${version}" - (cd "$PKG_ROOT/$src" && dch --force-bad-version --newversion "$version" "CeraLive rebuild") - injected=$((injected + 1)) - else - # No recipe yet (packaging wave). Document the exact invocation this WILL run. - echo " ${src}: no debian/changelog yet — will run:" - echo " dch --force-bad-version --newversion \"-${SUFFIX}\" \"CeraLive rebuild\"" - pending=$((pending + 1)) - fi -done +DIR="$(recipe_dir "$SOURCE")" +CHANGELOG="$PKG_ROOT/$DIR/debian/changelog" +[ -r "$CHANGELOG" ] || die "source '$SOURCE' changelog '$CHANGELOG' is not readable" +command -v dpkg-parsechangelog >/dev/null 2>&1 || die "source '$SOURCE': dpkg-parsechangelog is not installed" +command -v dch >/dev/null 2>&1 || die "source '$SOURCE': dch is not installed" -echo "inject-deb-version: ${injected} injected, ${pending} pending recipes" +TOP="$(dpkg-parsechangelog -l "$CHANGELOG" -S Version)" +BASE="${TOP%%~ceralive*}" +VERSION="${BASE}${SUFFIX}" +log "source '$SOURCE': $TOP -> $VERSION" +(cd "$PKG_ROOT/$DIR" && dch --force-bad-version --newversion "$VERSION" "CeraLive rebuild") +log "source '$SOURCE': injected suffix '$SUFFIX'" diff --git a/packaging/ci/test-build-bookworm-differential.sh b/packaging/ci/test-build-bookworm-differential.sh new file mode 100755 index 0000000..a1f7500 --- /dev/null +++ b/packaging/ci/test-build-bookworm-differential.sh @@ -0,0 +1,325 @@ +#!/usr/bin/env bash +# test-build-bookworm-differential.sh — differential builder + rebuild-counter contract. +# +# HOST-RUNNABLE, OFFLINE, NO DOCKER. The builder's BUILD_BOOKWORM_STUB_DIR seam replaces only +# the expensive source-build body with fixture artifacts; build-set parsing, carried-deb repo +# seeding, bootstrap-order dispatch, the unchanged check-package-sets.sh call, and the merged +# runtime-closure assertion are the production paths. A poisoned docker stub proves the +# zero-build host path never starts a container. +# +# Counter fixtures drive the real inject-deb-version.sh with stubbed dpkg-parsechangelog/dch. +# Every previous-manifest row is parsed by production code; the dch transcript proves the suffix +# that would be injected without touching a checked-in changelog. +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BUILD_SCRIPT="$HERE/build-bookworm.sh" +INJECT_SCRIPT="$HERE/inject-deb-version.sh" +EXPECTED="$HERE/expected-packages.txt" + +for required in "$BUILD_SCRIPT" "$INJECT_SCRIPT" "$EXPECTED" "$HERE/check-package-sets.sh" "$HERE/tag-guard.sh"; do + [ -r "$required" ] || { echo "missing: $required" >&2; exit 1; } +done + +pass=0; fail=0 +ok() { printf ' ok %s\n' "$1"; pass=$((pass + 1)); } +bad() { printf ' FAIL %s\n' "$1"; fail=$((fail + 1)); } + +ROOT="$(mktemp -d "${TMPDIR:-/tmp}/ceralive-build-differential.XXXXXX")" +trap 'rm -rf "$ROOT"' EXIT + +FIXTURE_PKG="$ROOT/pkg" +STUB_BIN="$ROOT/bin" +STUB_BUILDS="$ROOT/stub-builds" +OUT="$ROOT/stdout.txt" +ERR="$ROOT/stderr.txt" +TRACE="$ROOT/build-trace.txt" +DCH_LOG="$ROOT/dch.log" +DOCKER_LOG="$ROOT/docker.log" +mkdir -p "$FIXTURE_PKG/ci" "$STUB_BIN" "$STUB_BUILDS" + +# ---- a minimal standalone packaging tree ------------------------------------------------------ +cp "$INJECT_SCRIPT" "$FIXTURE_PKG/ci/inject-deb-version.sh" +cp "$HERE/tag-guard.sh" "$FIXTURE_PKG/ci/tag-guard.sh" +cp "$HERE/check-package-sets.sh" "$FIXTURE_PKG/ci/check-package-sets.sh" +cp "$EXPECTED" "$FIXTURE_PKG/ci/expected-packages.txt" + +base_version() { + case "$1" in + libqrtr-glib) printf '1.4.0-1\n' ;; + libmbim) printf '1.34.0-1\n' ;; + libqmi) printf '1.38.0-1\n' ;; + modemmanager) printf '1.24.2-2\n' ;; + *) echo "fixture: unknown source '$1'" >&2; return 1 ;; + esac +} + +recipe_dir() { case "$1" in modemmanager) echo ModemManager ;; *) echo "$1" ;; esac; } + +for source_name in libqrtr-glib libmbim libqmi modemmanager; do + dir="$(recipe_dir "$source_name")" + mkdir -p "$FIXTURE_PKG/$dir/debian" + printf '%s (%s) unstable; urgency=medium\n\n * fixture\n\n -- CeraLive CI Fri, 21 Aug 2026 00:00:00 +0000\n' \ + "$source_name" "$(base_version "$source_name")" >"$FIXTURE_PKG/$dir/debian/changelog" +done + +# These stubs remove the host's devscripts/dpkg dependency without moving counter parsing out of +# inject-deb-version.sh. dpkg-parsechangelog reads only the fixture changelog's top version; dch +# records the exact --newversion and working source directory it was asked to mutate. +cat >"$STUB_BIN/dpkg-parsechangelog" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +changelog="" +while [ $# -gt 0 ]; do + case "$1" in + -l) shift; changelog="${1:-}" ;; + -S) shift; [ "${1:-}" = Version ] || { echo "stub: only -S Version is supported" >&2; exit 97; } ;; + esac + shift +done +[ -r "$changelog" ] || { echo "stub: unreadable changelog '$changelog'" >&2; exit 97; } +IFS= read -r first <"$changelog" +version="${first#*(}" +version="${version%%)*}" +printf '%s\n' "$version" +STUB +cat >"$STUB_BIN/dch" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +printf '%s|%s\n' "$PWD" "$*" >>"$DCH_LOG" +STUB +cat >"$STUB_BIN/docker" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >>"$DOCKER_LOG" +echo "docker(stub): zero-build path must not invoke docker" >&2 +exit 97 +STUB +chmod +x "$STUB_BIN/dpkg-parsechangelog" "$STUB_BIN/dch" "$STUB_BIN/docker" + +# ---- expected-set fixture helpers -------------------------------------------------------------- +expected_set() { # + awk -v want="[$1 all-artifact]" ' + /^\[/ { h=$0; sub(/[ \t]*#.*$/, "", h); insec=(h==want)?1:0; next } + insec { l=$0; sub(/#.*$/, "", l); gsub(/[ \t]+/, "", l); if (l!="") print l } + ' "$EXPECTED" | LC_ALL=C sort -u +} + +write_source_debs() { # + local source_name="$1" dest="$2" suffix="$3" package version + version="$(base_version "$source_name")${suffix}" + mkdir -p "$dest" + while IFS= read -r package; do + [ -n "$package" ] || continue + printf 'fixture deb: %s %s\n' "$package" "$version" >"$dest/${package}_${version}_amd64.deb" + done < <(expected_set "$source_name") +} + +write_changes() { # + local source_name="$1" dest="$2" binaries + binaries="$(expected_set "$source_name" | tr '\n' ' ')" + cat >"$dest/${source_name}_amd64.changes" < + local dest="$1" suffix="$2" source_name + for source_name in libqrtr-glib libmbim libqmi modemmanager; do + write_source_debs "$source_name" "$dest" "$suffix" + done +} + +assert_rc() { if [ "$1" -eq "$2" ]; then ok "$3"; else bad "$3 — exit $1, expected $2"; fi; } +assert_nz() { if [ "$1" -ne 0 ]; then ok "$2"; else bad "$2 — expected a non-zero exit, got 0"; fi; } +assert_err() { if grep -qF -- "$1" "$ERR"; then ok "$2"; else bad "$2 — stderr does not name '$1'"; fi; } +assert_log() { if grep -qF -- "$1" "$2"; then ok "$3"; else bad "$3 — '$1' absent from $2"; fi; } + +# ---- previous-manifest fixtures for one source ------------------------------------------------- +write_qmi_manifest() { # + local path="$1" v_ar="$2" v_aa="$3" v_rr="$4" v_ra="$5" + cat >"$path" <"$M_ZERO" <<'EOF' +# CeraLive modem-stack release manifest (source intentionally absent) +tag: v1.1.0 +version: 1.1.0 +closure_version: 2 +amd64 libmbim-glib4 libmbim 1.34.0-1~ceralive.2 runtime libmbim-glib4_1.34.0-1~ceralive.2_amd64.deb 0000 +EOF + +if [ "$(awk '$3=="libqmi" && $1=="amd64" {a=1} $3=="libqmi" && $1=="arm64" {b=1} $3=="libqmi" && $5=="runtime" {r=1} $3=="libqmi" && $5=="aux" {x=1} END {print a+b+r+x}' "$M_UNIFORM")" -eq 4 ]; then + ok "counter fixture covers both arches and runtime+aux rows" +else + bad "counter fixture does not cover both arches and runtime+aux rows" +fi + +run_inject() { # + : >"$DCH_LOG" + if [ "$1" = ABSENT ]; then + env -u PREV_MANIFEST_FILE PATH="$STUB_BIN:$PATH" DCH_LOG="$DCH_LOG" \ + bash "$FIXTURE_PKG/ci/inject-deb-version.sh" --source libqmi v1.2.0 >"$OUT" 2>"$ERR" + else + PREV_MANIFEST_FILE="$1" PATH="$STUB_BIN:$PATH" DCH_LOG="$DCH_LOG" \ + bash "$FIXTURE_PKG/ci/inject-deb-version.sh" --source libqmi v1.2.0 >"$OUT" 2>"$ERR" + fi +} + +assert_injected_suffix() { #