From 5278297167a720cdd45254041fad7f2fa4b454e1 Mon Sep 17 00:00:00 2001 From: Ivan Carvalho Date: Mon, 7 Sep 2026 17:49:10 -0400 Subject: [PATCH 01/11] Use cargo patch for host as well. Benchmark patched vs unpatched --- .github/workflows/manual_reproducible.yml | 70 ++++++++++++++++++++--- 1 file changed, 61 insertions(+), 9 deletions(-) diff --git a/.github/workflows/manual_reproducible.yml b/.github/workflows/manual_reproducible.yml index f4d363d..b4b5cc5 100644 --- a/.github/workflows/manual_reproducible.yml +++ b/.github/workflows/manual_reproducible.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - target: [aarch64-unknown-linux-gnu, x86_64-unknown-freebsd] + target: [x86_64-unknown-linux-gnu, aarch64-unknown-linux-gnu, x86_64-unknown-freebsd] env: TARGET: ${{ matrix.target }} steps: @@ -90,9 +90,9 @@ jobs: use_cache: "false" - name: Install cargo-clone - uses: taiki-e/install-action@1ed6d7be6168f6c9046541087ff549b6bc581fdf # v2.87.2 - with: - tool: cargo-clone@1.2.4 + run: | + /home/linuxbrew/.linuxbrew/bin/brew install --force-bottle cargo-clone + echo "/home/linuxbrew/.linuxbrew/bin" >> "$GITHUB_PATH" - name: Download tool sources run: cargo clone cargo-zigbuild@0.23.2 cargo-auditable@0.7.5 -- freebsd-tool-sources/ @@ -141,10 +141,9 @@ jobs: persist-credentials: false - name: Download patched Cargo - if: matrix.runner == 'ubuntu-24.04-arm' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - name: patched-cargo-aarch64-unknown-linux-gnu + name: patched-cargo-${{ matrix.runner == 'ubuntu-24.04-arm' && 'aarch64-unknown-linux-gnu' || 'x86_64-unknown-linux-gnu' }} path: patched-cargo - name: Setup Zigbuild @@ -156,14 +155,12 @@ jobs: use_cache: "false" - name: Select patched Cargo - if: matrix.runner == 'ubuntu-24.04-arm' run: | chmod +x "$GITHUB_WORKSPACE/patched-cargo/cargo" echo "$GITHUB_WORKSPACE/patched-cargo" >> "$GITHUB_PATH" echo "CARGO=$GITHUB_WORKSPACE/patched-cargo/cargo" >> "$GITHUB_ENV" - name: Verify patched Cargo selection - if: matrix.runner == 'ubuntu-24.04-arm' run: | test "$(command -v cargo)" = "$CARGO" echo "Using patched Cargo: $CARGO" @@ -199,6 +196,61 @@ jobs: if-no-files-found: error compression-level: 0 + zigbuild_unpatched: + name: Zigbuild unpatched (ubuntu-24.04, ${{ matrix.build.target }}${{ matrix.build.zigtargetsuffix }}) + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + build: *targets + env: + TARGET: ${{ matrix.build.target }} + ZIGTARGETSUFFIX: ${{ matrix.build.zigtargetsuffix }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Setup Zigbuild + uses: $/.github/actions/setup-zigbuild + with: + target: ${{ matrix.build.target }} + rust_components: rust-src + install_target: "true" + use_cache: "false" + + - name: Configure reproducible builds + run: | + echo "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" >> "$GITHUB_ENV" + echo "TZ=UTC" >> "$GITHUB_ENV" + echo "LC_ALL=C" >> "$GITHUB_ENV" + RUST_SYSROOT=$(rustc --print sysroot) + echo "RUSTFLAGS=--remap-path-prefix=${CARGO_HOME:-$HOME/.cargo}=/cargo --remap-path-prefix=${RUST_SYSROOT}=/rust-toolchain --remap-path-prefix=${GITHUB_WORKSPACE}=/src" >> "$GITHUB_ENV" + + - name: Build with Zig + run: | + rustc -vV + cargo -vV + cargo zigbuild -vv --locked --release --no-default-features --features from-toml,from-yaml,greppable,from-xml --target "${TARGET}${ZIGTARGETSUFFIX}" + + - name: Create release archive + run: | + mkdir -p artifacts + cp "target/${TARGET}/release/celq" artifacts/celq + tar czf artifacts/celq.tar.gz -C "target/${TARGET}/release" celq + + - name: Upload archive and binary + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: reproducible-${{ matrix.build.target }}${{ matrix.build.zigtargetsuffix }}-ubuntu-24.04-unpatched + retention-days: 1 + path: | + artifacts/celq + artifacts/celq.tar.gz + if-no-files-found: error + compression-level: 0 + freebsd: name: Zigbuild (FreeBSD x86-64, ${{ matrix.build.target }}${{ matrix.build.zigtargetsuffix }}) needs: [patched_cargo, freebsd_tools] @@ -293,7 +345,7 @@ jobs: hashes: name: Print SHA-256 hashes - needs: [zigbuild, freebsd] + needs: [zigbuild, zigbuild_unpatched, freebsd] if: ${{ !cancelled() }} runs-on: ubuntu-24.04 steps: From aa26be6a8d85600fe3a8988f96af1619e756b461 Mon Sep 17 00:00:00 2001 From: Ivan Carvalho Date: Mon, 7 Sep 2026 18:06:16 -0400 Subject: [PATCH 02/11] Add extra logs to cargo patch --- .../patches/cargo-linux-x86_64-metadata.patch | 140 +++++++++++++++++- 1 file changed, 138 insertions(+), 2 deletions(-) diff --git a/.github/reproducibility/patches/cargo-linux-x86_64-metadata.patch b/.github/reproducibility/patches/cargo-linux-x86_64-metadata.patch index 1cfd001..29f2d37 100644 --- a/.github/reproducibility/patches/cargo-linux-x86_64-metadata.patch +++ b/.github/reproducibility/patches/cargo-linux-x86_64-metadata.patch @@ -1,7 +1,138 @@ diff --git a/src/cargo/core/compiler/build_runner/compilation_files.rs b/src/cargo/core/compiler/build_runner/compilation_files.rs --- a/src/cargo/core/compiler/build_runner/compilation_files.rs +++ b/src/cargo/core/compiler/build_runner/compilation_files.rs -@@ -864,6 +864,12 @@ +@@ -688,6 +688,15 @@ fn metadata_of<'a>( + &metas[unit] + } + ++fn trace_metadata(unit: &Unit, stage: &str, hasher: &StableHasher, input: impl fmt::Debug) { ++ if unit.pkg.name().as_str() == "unicode-ident" { ++ eprintln!( ++ "cargo-metadata-trace: crate={} kind={:?} stage={} hash={:016x} input={:?}", ++ unit.pkg.name(), unit.kind, stage, Hasher::finish(hasher), input, ++ ); ++ } ++} ++ + /// Computes the metadata hash for the given [`Unit`]. + fn compute_metadata( + unit: &Unit, +@@ -706,6 +715,7 @@ fn compute_metadata( + let mut shared_hasher = StableHasher::new(); + + METADATA_VERSION.hash(&mut shared_hasher); ++ trace_metadata(unit, "metadata-version", &shared_hasher, &METADATA_VERSION); + + let ws_root = if unit.is_std { + // SourceId for stdlib crates is an absolute path inside the sysroot. +@@ -722,17 +732,22 @@ fn compute_metadata( + .package_id() + .stable_hash(ws_root) + .hash(&mut shared_hasher); ++ trace_metadata(unit, "package", &shared_hasher, (unit.pkg.package_id(), ws_root)); + + // Also mix in enabled features to our metadata. This'll ensure that + // when changing feature sets each lib is separately cached. + unit.features.hash(&mut shared_hasher); ++ trace_metadata(unit, "features", &shared_hasher, &unit.features); + + // Throw in the profile we're compiling with. This helps caching + // `panic=abort` and `panic=unwind` artifacts, additionally with various + // settings like debuginfo and whatnot. + unit.profile.hash(&mut shared_hasher); ++ trace_metadata(unit, "profile", &shared_hasher, &unit.profile); + unit.mode.hash(&mut shared_hasher); ++ trace_metadata(unit, "mode", &shared_hasher, &unit.mode); + build_runner.lto[unit].hash(&mut shared_hasher); ++ trace_metadata(unit, "lto", &shared_hasher, &build_runner.lto[unit]); + + // Artifacts compiled for the host should have a different + // metadata piece than those compiled for the target, so make sure +@@ -740,13 +755,17 @@ fn compute_metadata( + // so that the StableHash doesn't change based on the pathnames + // of the custom target JSON spec files. + unit.kind.fingerprint_hash().hash(&mut shared_hasher); ++ trace_metadata(unit, "compile-kind", &shared_hasher, &unit.kind.fingerprint_hash()); + + // Finally throw in the target name/kind. This ensures that concurrent + // compiles of targets in the same crate don't collide. + unit.target.name().hash(&mut shared_hasher); ++ trace_metadata(unit, "target-name", &shared_hasher, &unit.target.name()); + unit.target.kind().hash(&mut shared_hasher); ++ trace_metadata(unit, "target-kind", &shared_hasher, &unit.target.kind()); + + hash_rustc_version(bcx, &mut shared_hasher, unit); ++ trace_metadata(unit, "rustc-version-complete", &shared_hasher, &bcx.rustc().version); + + if build_runner.bcx.ws.is_member(&unit.pkg) { + // This is primarily here for clippy. This ensures that the clippy +@@ -756,6 +775,11 @@ fn compute_metadata( + } + } + ++ trace_metadata( ++ unit, "workspace-wrapper", &shared_hasher, ++ (bcx.ws.is_member(&unit.pkg), &bcx.rustc().workspace_wrapper), ++ ); ++ + // Seed the contents of `__CARGO_DEFAULT_LIB_METADATA` to the hasher if present. + // This should be the release channel, to get a different hash for each channel. + if let Ok(ref channel) = build_runner +@@ -764,8 +788,14 @@ fn compute_metadata( + .get_env("__CARGO_DEFAULT_LIB_METADATA") + { + channel.hash(&mut shared_hasher); ++ trace_metadata(unit, "channel-seed", &shared_hasher, channel); + } + ++ trace_metadata( ++ unit, "channel-seed-complete", &shared_hasher, ++ bcx.gctx.get_env("__CARGO_DEFAULT_LIB_METADATA").ok(), ++ ); ++ + // std units need to be kept separate from user dependencies. std crates + // are differentiated in the Unit with `is_std` (for things like + // `-Zforce-unstable-if-unmarked`), so they are always built separately. +@@ -774,6 +804,7 @@ fn compute_metadata( + // `is_std` to false for build dependencies so that they can be shared + // with user dependencies. + unit.is_std.hash(&mut shared_hasher); ++ trace_metadata(unit, "is-std", &shared_hasher, &unit.is_std); + + // While we don't hash RUSTFLAGS because it may contain absolute paths that + // hurts reproducibility, we track whether a unit's RUSTFLAGS is from host +@@ -793,8 +824,13 @@ fn compute_metadata( + .links_overrides + != unit.links_overrides; + target_configs_are_different.hash(&mut shared_hasher); ++ trace_metadata(unit, "host-config-discriminator", &shared_hasher, target_configs_are_different); + } + ++ trace_metadata( ++ unit, "shared-complete", &shared_hasher, ++ (bcx.gctx.target_applies_to_host().ok(), &unit.rustflags, &unit.rustdocflags), ++ ); + let mut c_metadata_hasher = shared_hasher.clone(); + // Mix in the target-metadata of all the dependencies of this target. + let mut dep_c_metadata_hashes = deps_metadata +@@ -803,6 +839,7 @@ fn compute_metadata( + .collect::>(); + dep_c_metadata_hashes.sort(); + dep_c_metadata_hashes.hash(&mut c_metadata_hasher); ++ trace_metadata(unit, "dependencies-final-metadata", &c_metadata_hasher, &dep_c_metadata_hashes); + + let mut unit_id_hasher = shared_hasher.clone(); + // Mix in the target-metadata of all the dependencies of this target. +@@ -854,6 +891,7 @@ fn has_remap_path_prefix(args: &[String]) -> bool { + + /// Hash the version of rustc being used during the build process. + fn hash_rustc_version(bcx: &BuildContext<'_, '_>, hasher: &mut StableHasher, unit: &Unit) { ++ trace_metadata(unit, "rustc-version-raw", hasher, &bcx.rustc().verbose_version); + let vers = &bcx.rustc().version; + if vers.pre.is_empty() || bcx.gctx.cli_unstable().separate_nightlies { + // For stable, keep the artifacts separate. This helps if someone is +@@ -863,7 +901,14 @@ fn hash_rustc_version(bcx: &BuildContext<'_, '_>, hasher: &mut StableHasher, uni + // regardless of the host. for line in bcx.rustc().verbose_version.lines() { if unit.kind.is_host() || !line.starts_with("host: ") { + // Normalize hash input only; keep the real host for compilation. @@ -11,11 +142,16 @@ diff --git a/src/cargo/core/compiler/build_runner/compilation_files.rs b/src/car + line + }; line.hash(hasher); ++ trace_metadata(unit, "rustc-version-line", hasher, line); } } return; -@@ -881,3 +887,3 @@ +@@ -879,7 +924,7 @@ fn hash_rustc_version(bcx: &BuildContext<'_, '_>, hasher: &mut StableHasher, uni + // targets, (like gnu vs musl or gnu vs msvc). In the future, we may want + // to consider hashing `unit.kind.short_name()` instead. if unit.kind.is_host() { - bcx.rustc().host.hash(hasher); + "x86_64-unknown-linux-gnu".hash(hasher); } + // None of the other lines are important. Currently they are: + // binary: rustc <-- or "rustdoc" From d86bad9364a5d92afcb2040f05aa6e9a19a38fa9 Mon Sep 17 00:00:00 2001 From: Ivan Carvalho Date: Mon, 7 Sep 2026 18:25:14 -0400 Subject: [PATCH 03/11] Fix for cargo-reproducibility --- .../patches/cargo-linux-x86_64-metadata.patch | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/reproducibility/patches/cargo-linux-x86_64-metadata.patch b/.github/reproducibility/patches/cargo-linux-x86_64-metadata.patch index 29f2d37..8e0e87b 100644 --- a/.github/reproducibility/patches/cargo-linux-x86_64-metadata.patch +++ b/.github/reproducibility/patches/cargo-linux-x86_64-metadata.patch @@ -101,11 +101,16 @@ diff --git a/src/cargo/core/compiler/build_runner/compilation_files.rs b/src/car // While we don't hash RUSTFLAGS because it may contain absolute paths that // hurts reproducibility, we track whether a unit's RUSTFLAGS is from host -@@ -793,8 +824,13 @@ fn compute_metadata( +@@ -793,8 +824,18 @@ fn compute_metadata( .links_overrides != unit.links_overrides; target_configs_are_different.hash(&mut shared_hasher); + trace_metadata(unit, "host-config-discriminator", &shared_hasher, target_configs_are_different); ++ } else if unit.kind.is_host() { ++ // Reproducible fix: match the Linux x86-64 zigbuild baseline's discriminator. ++ // Change hash input only, not the actual host configuration or compiler flags. ++ false.hash(&mut shared_hasher); ++ trace_metadata(unit, "host-config-discriminator-reproducible", &shared_hasher, false); } + trace_metadata( @@ -115,7 +120,7 @@ diff --git a/src/cargo/core/compiler/build_runner/compilation_files.rs b/src/car let mut c_metadata_hasher = shared_hasher.clone(); // Mix in the target-metadata of all the dependencies of this target. let mut dep_c_metadata_hashes = deps_metadata -@@ -803,6 +839,7 @@ fn compute_metadata( +@@ -803,6 +844,7 @@ fn compute_metadata( .collect::>(); dep_c_metadata_hashes.sort(); dep_c_metadata_hashes.hash(&mut c_metadata_hasher); @@ -123,7 +128,7 @@ diff --git a/src/cargo/core/compiler/build_runner/compilation_files.rs b/src/car let mut unit_id_hasher = shared_hasher.clone(); // Mix in the target-metadata of all the dependencies of this target. -@@ -854,6 +891,7 @@ fn has_remap_path_prefix(args: &[String]) -> bool { +@@ -854,6 +896,7 @@ fn has_remap_path_prefix(args: &[String]) -> bool { /// Hash the version of rustc being used during the build process. fn hash_rustc_version(bcx: &BuildContext<'_, '_>, hasher: &mut StableHasher, unit: &Unit) { @@ -131,7 +136,7 @@ diff --git a/src/cargo/core/compiler/build_runner/compilation_files.rs b/src/car let vers = &bcx.rustc().version; if vers.pre.is_empty() || bcx.gctx.cli_unstable().separate_nightlies { // For stable, keep the artifacts separate. This helps if someone is -@@ -863,7 +901,14 @@ fn hash_rustc_version(bcx: &BuildContext<'_, '_>, hasher: &mut StableHasher, uni +@@ -863,7 +906,14 @@ fn hash_rustc_version(bcx: &BuildContext<'_, '_>, hasher: &mut StableHasher, uni // regardless of the host. for line in bcx.rustc().verbose_version.lines() { if unit.kind.is_host() || !line.starts_with("host: ") { @@ -146,7 +151,7 @@ diff --git a/src/cargo/core/compiler/build_runner/compilation_files.rs b/src/car } } return; -@@ -879,7 +924,7 @@ fn hash_rustc_version(bcx: &BuildContext<'_, '_>, hasher: &mut StableHasher, uni +@@ -879,7 +929,7 @@ fn hash_rustc_version(bcx: &BuildContext<'_, '_>, hasher: &mut StableHasher, uni // targets, (like gnu vs musl or gnu vs msvc). In the future, we may want // to consider hashing `unit.kind.short_name()` instead. if unit.kind.is_host() { From 8e3e8b6f617d6a4aa575487e1e951928365d9d64 Mon Sep 17 00:00:00 2001 From: Ivan Carvalho Date: Mon, 7 Sep 2026 18:44:58 -0400 Subject: [PATCH 04/11] Towards a reproducible build --- .../patches/cargo-linux-x86_64-metadata.patch | 136 +----------------- .github/workflows/manual_reproducible.yml | 80 +++-------- 2 files changed, 22 insertions(+), 194 deletions(-) diff --git a/.github/reproducibility/patches/cargo-linux-x86_64-metadata.patch b/.github/reproducibility/patches/cargo-linux-x86_64-metadata.patch index 8e0e87b..7b6a7bd 100644 --- a/.github/reproducibility/patches/cargo-linux-x86_64-metadata.patch +++ b/.github/reproducibility/patches/cargo-linux-x86_64-metadata.patch @@ -1,143 +1,18 @@ diff --git a/src/cargo/core/compiler/build_runner/compilation_files.rs b/src/cargo/core/compiler/build_runner/compilation_files.rs --- a/src/cargo/core/compiler/build_runner/compilation_files.rs +++ b/src/cargo/core/compiler/build_runner/compilation_files.rs -@@ -688,6 +688,15 @@ fn metadata_of<'a>( - &metas[unit] - } - -+fn trace_metadata(unit: &Unit, stage: &str, hasher: &StableHasher, input: impl fmt::Debug) { -+ if unit.pkg.name().as_str() == "unicode-ident" { -+ eprintln!( -+ "cargo-metadata-trace: crate={} kind={:?} stage={} hash={:016x} input={:?}", -+ unit.pkg.name(), unit.kind, stage, Hasher::finish(hasher), input, -+ ); -+ } -+} -+ - /// Computes the metadata hash for the given [`Unit`]. - fn compute_metadata( - unit: &Unit, -@@ -706,6 +715,7 @@ fn compute_metadata( - let mut shared_hasher = StableHasher::new(); - - METADATA_VERSION.hash(&mut shared_hasher); -+ trace_metadata(unit, "metadata-version", &shared_hasher, &METADATA_VERSION); - - let ws_root = if unit.is_std { - // SourceId for stdlib crates is an absolute path inside the sysroot. -@@ -722,17 +732,22 @@ fn compute_metadata( - .package_id() - .stable_hash(ws_root) - .hash(&mut shared_hasher); -+ trace_metadata(unit, "package", &shared_hasher, (unit.pkg.package_id(), ws_root)); - - // Also mix in enabled features to our metadata. This'll ensure that - // when changing feature sets each lib is separately cached. - unit.features.hash(&mut shared_hasher); -+ trace_metadata(unit, "features", &shared_hasher, &unit.features); - - // Throw in the profile we're compiling with. This helps caching - // `panic=abort` and `panic=unwind` artifacts, additionally with various - // settings like debuginfo and whatnot. - unit.profile.hash(&mut shared_hasher); -+ trace_metadata(unit, "profile", &shared_hasher, &unit.profile); - unit.mode.hash(&mut shared_hasher); -+ trace_metadata(unit, "mode", &shared_hasher, &unit.mode); - build_runner.lto[unit].hash(&mut shared_hasher); -+ trace_metadata(unit, "lto", &shared_hasher, &build_runner.lto[unit]); - - // Artifacts compiled for the host should have a different - // metadata piece than those compiled for the target, so make sure -@@ -740,13 +755,17 @@ fn compute_metadata( - // so that the StableHash doesn't change based on the pathnames - // of the custom target JSON spec files. - unit.kind.fingerprint_hash().hash(&mut shared_hasher); -+ trace_metadata(unit, "compile-kind", &shared_hasher, &unit.kind.fingerprint_hash()); - - // Finally throw in the target name/kind. This ensures that concurrent - // compiles of targets in the same crate don't collide. - unit.target.name().hash(&mut shared_hasher); -+ trace_metadata(unit, "target-name", &shared_hasher, &unit.target.name()); - unit.target.kind().hash(&mut shared_hasher); -+ trace_metadata(unit, "target-kind", &shared_hasher, &unit.target.kind()); - - hash_rustc_version(bcx, &mut shared_hasher, unit); -+ trace_metadata(unit, "rustc-version-complete", &shared_hasher, &bcx.rustc().version); - - if build_runner.bcx.ws.is_member(&unit.pkg) { - // This is primarily here for clippy. This ensures that the clippy -@@ -756,6 +775,11 @@ fn compute_metadata( - } - } - -+ trace_metadata( -+ unit, "workspace-wrapper", &shared_hasher, -+ (bcx.ws.is_member(&unit.pkg), &bcx.rustc().workspace_wrapper), -+ ); -+ - // Seed the contents of `__CARGO_DEFAULT_LIB_METADATA` to the hasher if present. - // This should be the release channel, to get a different hash for each channel. - if let Ok(ref channel) = build_runner -@@ -764,8 +788,14 @@ fn compute_metadata( - .get_env("__CARGO_DEFAULT_LIB_METADATA") - { - channel.hash(&mut shared_hasher); -+ trace_metadata(unit, "channel-seed", &shared_hasher, channel); - } - -+ trace_metadata( -+ unit, "channel-seed-complete", &shared_hasher, -+ bcx.gctx.get_env("__CARGO_DEFAULT_LIB_METADATA").ok(), -+ ); -+ - // std units need to be kept separate from user dependencies. std crates - // are differentiated in the Unit with `is_std` (for things like - // `-Zforce-unstable-if-unmarked`), so they are always built separately. -@@ -774,6 +804,7 @@ fn compute_metadata( - // `is_std` to false for build dependencies so that they can be shared - // with user dependencies. - unit.is_std.hash(&mut shared_hasher); -+ trace_metadata(unit, "is-std", &shared_hasher, &unit.is_std); - - // While we don't hash RUSTFLAGS because it may contain absolute paths that - // hurts reproducibility, we track whether a unit's RUSTFLAGS is from host -@@ -793,8 +824,18 @@ fn compute_metadata( +@@ -793,6 +793,10 @@ .links_overrides != unit.links_overrides; target_configs_are_different.hash(&mut shared_hasher); -+ trace_metadata(unit, "host-config-discriminator", &shared_hasher, target_configs_are_different); + } else if unit.kind.is_host() { + // Reproducible fix: match the Linux x86-64 zigbuild baseline's discriminator. + // Change hash input only, not the actual host configuration or compiler flags. + false.hash(&mut shared_hasher); -+ trace_metadata(unit, "host-config-discriminator-reproducible", &shared_hasher, false); } -+ trace_metadata( -+ unit, "shared-complete", &shared_hasher, -+ (bcx.gctx.target_applies_to_host().ok(), &unit.rustflags, &unit.rustdocflags), -+ ); let mut c_metadata_hasher = shared_hasher.clone(); - // Mix in the target-metadata of all the dependencies of this target. - let mut dep_c_metadata_hashes = deps_metadata -@@ -803,6 +844,7 @@ fn compute_metadata( - .collect::>(); - dep_c_metadata_hashes.sort(); - dep_c_metadata_hashes.hash(&mut c_metadata_hasher); -+ trace_metadata(unit, "dependencies-final-metadata", &c_metadata_hasher, &dep_c_metadata_hashes); - - let mut unit_id_hasher = shared_hasher.clone(); - // Mix in the target-metadata of all the dependencies of this target. -@@ -854,6 +896,7 @@ fn has_remap_path_prefix(args: &[String]) -> bool { - - /// Hash the version of rustc being used during the build process. - fn hash_rustc_version(bcx: &BuildContext<'_, '_>, hasher: &mut StableHasher, unit: &Unit) { -+ trace_metadata(unit, "rustc-version-raw", hasher, &bcx.rustc().verbose_version); - let vers = &bcx.rustc().version; - if vers.pre.is_empty() || bcx.gctx.cli_unstable().separate_nightlies { - // For stable, keep the artifacts separate. This helps if someone is -@@ -863,7 +906,14 @@ fn hash_rustc_version(bcx: &BuildContext<'_, '_>, hasher: &mut StableHasher, uni - // regardless of the host. +@@ -864,6 +868,12 @@ for line in bcx.rustc().verbose_version.lines() { if unit.kind.is_host() || !line.starts_with("host: ") { + // Normalize hash input only; keep the real host for compilation. @@ -147,16 +22,11 @@ diff --git a/src/cargo/core/compiler/build_runner/compilation_files.rs b/src/car + line + }; line.hash(hasher); -+ trace_metadata(unit, "rustc-version-line", hasher, line); } } return; -@@ -879,7 +929,7 @@ fn hash_rustc_version(bcx: &BuildContext<'_, '_>, hasher: &mut StableHasher, uni - // targets, (like gnu vs musl or gnu vs msvc). In the future, we may want - // to consider hashing `unit.kind.short_name()` instead. +@@ -881,3 +891,3 @@ if unit.kind.is_host() { - bcx.rustc().host.hash(hasher); + "x86_64-unknown-linux-gnu".hash(hasher); } - // None of the other lines are important. Currently they are: - // binary: rustc <-- or "rustdoc" diff --git a/.github/workflows/manual_reproducible.yml b/.github/workflows/manual_reproducible.yml index b4b5cc5..be36d34 100644 --- a/.github/workflows/manual_reproducible.yml +++ b/.github/workflows/manual_reproducible.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - target: [x86_64-unknown-linux-gnu, aarch64-unknown-linux-gnu, x86_64-unknown-freebsd] + target: [aarch64-unknown-linux-gnu, x86_64-unknown-freebsd] env: TARGET: ${{ matrix.target }} steps: @@ -141,9 +141,10 @@ jobs: persist-credentials: false - name: Download patched Cargo + if: matrix.runner == 'ubuntu-24.04-arm' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - name: patched-cargo-${{ matrix.runner == 'ubuntu-24.04-arm' && 'aarch64-unknown-linux-gnu' || 'x86_64-unknown-linux-gnu' }} + name: patched-cargo-aarch64-unknown-linux-gnu path: patched-cargo - name: Setup Zigbuild @@ -155,12 +156,14 @@ jobs: use_cache: "false" - name: Select patched Cargo + if: matrix.runner == 'ubuntu-24.04-arm' run: | chmod +x "$GITHUB_WORKSPACE/patched-cargo/cargo" echo "$GITHUB_WORKSPACE/patched-cargo" >> "$GITHUB_PATH" echo "CARGO=$GITHUB_WORKSPACE/patched-cargo/cargo" >> "$GITHUB_ENV" - name: Verify patched Cargo selection + if: matrix.runner == 'ubuntu-24.04-arm' run: | test "$(command -v cargo)" = "$CARGO" echo "Using patched Cargo: $CARGO" @@ -183,7 +186,12 @@ jobs: run: | mkdir -p artifacts cp "target/${TARGET}/release/celq" artifacts/celq - tar czf artifacts/celq.tar.gz -C "target/${TARGET}/release" celq + tar --version + gzip --version + tar --format=gnu --sort=name --mtime="@${SOURCE_DATE_EPOCH}" \ + --owner=0 --group=0 --numeric-owner --mode=0755 \ + -cf artifacts/celq.tar -C "target/${TARGET}/release" celq + gzip -n -9 artifacts/celq.tar - name: Upload archive and binary uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -196,61 +204,6 @@ jobs: if-no-files-found: error compression-level: 0 - zigbuild_unpatched: - name: Zigbuild unpatched (ubuntu-24.04, ${{ matrix.build.target }}${{ matrix.build.zigtargetsuffix }}) - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - build: *targets - env: - TARGET: ${{ matrix.build.target }} - ZIGTARGETSUFFIX: ${{ matrix.build.zigtargetsuffix }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ github.sha }} - persist-credentials: false - - - name: Setup Zigbuild - uses: $/.github/actions/setup-zigbuild - with: - target: ${{ matrix.build.target }} - rust_components: rust-src - install_target: "true" - use_cache: "false" - - - name: Configure reproducible builds - run: | - echo "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" >> "$GITHUB_ENV" - echo "TZ=UTC" >> "$GITHUB_ENV" - echo "LC_ALL=C" >> "$GITHUB_ENV" - RUST_SYSROOT=$(rustc --print sysroot) - echo "RUSTFLAGS=--remap-path-prefix=${CARGO_HOME:-$HOME/.cargo}=/cargo --remap-path-prefix=${RUST_SYSROOT}=/rust-toolchain --remap-path-prefix=${GITHUB_WORKSPACE}=/src" >> "$GITHUB_ENV" - - - name: Build with Zig - run: | - rustc -vV - cargo -vV - cargo zigbuild -vv --locked --release --no-default-features --features from-toml,from-yaml,greppable,from-xml --target "${TARGET}${ZIGTARGETSUFFIX}" - - - name: Create release archive - run: | - mkdir -p artifacts - cp "target/${TARGET}/release/celq" artifacts/celq - tar czf artifacts/celq.tar.gz -C "target/${TARGET}/release" celq - - - name: Upload archive and binary - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: reproducible-${{ matrix.build.target }}${{ matrix.build.zigtargetsuffix }}-ubuntu-24.04-unpatched - retention-days: 1 - path: | - artifacts/celq - artifacts/celq.tar.gz - if-no-files-found: error - compression-level: 0 - freebsd: name: Zigbuild (FreeBSD x86-64, ${{ matrix.build.target }}${{ matrix.build.zigtargetsuffix }}) needs: [patched_cargo, freebsd_tools] @@ -293,7 +246,7 @@ jobs: envs: TARGET ZIGTARGETSUFFIX SOURCE_DATE_EPOCH prepare: | set -eu - pkg install -y ca_root_nss git devel/rustup-init + pkg install -y ca_root_nss git devel/rustup-init archivers/gtar archivers/gzip # Install the official FreeBSD Zig archive directly from ziglang.org. cd /tmp @@ -330,7 +283,12 @@ jobs: mkdir -p artifacts cp "target/${TARGET}/release/celq" artifacts/celq - tar czf artifacts/celq.tar.gz -C "target/${TARGET}/release" celq + gtar --version + /usr/local/bin/gzip --version + gtar --format=gnu --sort=name --mtime="@${SOURCE_DATE_EPOCH}" \ + --owner=0 --group=0 --numeric-owner --mode=0755 \ + -cf artifacts/celq.tar -C "target/${TARGET}/release" celq + /usr/local/bin/gzip -n -9 artifacts/celq.tar - name: Upload archive and binary uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -345,7 +303,7 @@ jobs: hashes: name: Print SHA-256 hashes - needs: [zigbuild, zigbuild_unpatched, freebsd] + needs: [zigbuild, freebsd] if: ${{ !cancelled() }} runs-on: ubuntu-24.04 steps: From e0466b20891edb658d1e67acd156f66f4645e65d Mon Sep 17 00:00:00 2001 From: Ivan Carvalho Date: Mon, 7 Sep 2026 19:00:53 -0400 Subject: [PATCH 05/11] Try to reproduce linux musl builds --- .github/workflows/manual_reproducible.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/manual_reproducible.yml b/.github/workflows/manual_reproducible.yml index be36d34..355836e 100644 --- a/.github/workflows/manual_reproducible.yml +++ b/.github/workflows/manual_reproducible.yml @@ -130,6 +130,8 @@ jobs: build: &targets - target: x86_64-unknown-linux-gnu zigtargetsuffix: .2.28 + - target: x86_64-unknown-linux-musl + zigtargetsuffix: "" env: TARGET: ${{ matrix.build.target }} ZIGTARGETSUFFIX: ${{ matrix.build.zigtargetsuffix }} From cb87d14ca526082628f2d14b6da01b37e66578c5 Mon Sep 17 00:00:00 2001 From: Ivan Carvalho Date: Mon, 7 Sep 2026 19:33:57 -0400 Subject: [PATCH 06/11] Update reproducibility patch --- .../patches/cargo-linux-x86_64-metadata.patch | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/reproducibility/patches/cargo-linux-x86_64-metadata.patch b/.github/reproducibility/patches/cargo-linux-x86_64-metadata.patch index 7b6a7bd..1822d0c 100644 --- a/.github/reproducibility/patches/cargo-linux-x86_64-metadata.patch +++ b/.github/reproducibility/patches/cargo-linux-x86_64-metadata.patch @@ -1,18 +1,22 @@ diff --git a/src/cargo/core/compiler/build_runner/compilation_files.rs b/src/cargo/core/compiler/build_runner/compilation_files.rs --- a/src/cargo/core/compiler/build_runner/compilation_files.rs +++ b/src/cargo/core/compiler/build_runner/compilation_files.rs -@@ -793,6 +793,10 @@ +@@ -793,6 +793,14 @@ .links_overrides != unit.links_overrides; target_configs_are_different.hash(&mut shared_hasher); -+ } else if unit.kind.is_host() { -+ // Reproducible fix: match the Linux x86-64 zigbuild baseline's discriminator. ++ } else if unit.kind.is_host() ++ && bcx.build_config.requested_kinds.iter().any(|kind| { ++ matches!(kind, CompileKind::Target(target) if target.short_name() == "x86_64-unknown-linux-gnu") ++ }) ++ { ++ // Reproducible fix: only GNU matches the Linux x86-64 baseline host, not musl. + // Change hash input only, not the actual host configuration or compiler flags. + false.hash(&mut shared_hasher); } let mut c_metadata_hasher = shared_hasher.clone(); -@@ -864,6 +868,12 @@ +@@ -864,6 +872,12 @@ for line in bcx.rustc().verbose_version.lines() { if unit.kind.is_host() || !line.starts_with("host: ") { + // Normalize hash input only; keep the real host for compilation. @@ -25,7 +29,7 @@ diff --git a/src/cargo/core/compiler/build_runner/compilation_files.rs b/src/car } } return; -@@ -881,3 +891,3 @@ +@@ -881,3 +895,3 @@ if unit.kind.is_host() { - bcx.rustc().host.hash(hasher); + "x86_64-unknown-linux-gnu".hash(hasher); From ed3e5f2b598b2c3b847777e5668114e760fe1b10 Mon Sep 17 00:00:00 2001 From: Ivan Carvalho Date: Mon, 7 Sep 2026 19:51:04 -0400 Subject: [PATCH 07/11] Add entries for aarch64 musl builds --- .github/workflows/manual_reproducible.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/manual_reproducible.yml b/.github/workflows/manual_reproducible.yml index 355836e..3c13098 100644 --- a/.github/workflows/manual_reproducible.yml +++ b/.github/workflows/manual_reproducible.yml @@ -132,6 +132,10 @@ jobs: zigtargetsuffix: .2.28 - target: x86_64-unknown-linux-musl zigtargetsuffix: "" + - target: aarch64-unknown-linux-gnu + zigtargetsuffix: .2.28 + - target: aarch64-unknown-linux-musl + zigtargetsuffix: "" env: TARGET: ${{ matrix.build.target }} ZIGTARGETSUFFIX: ${{ matrix.build.zigtargetsuffix }} From bb193f3aa8aa359baf26ae80e1bdd5c5f3119cb0 Mon Sep 17 00:00:00 2001 From: Ivan Carvalho Date: Mon, 7 Sep 2026 22:53:04 -0400 Subject: [PATCH 08/11] Use aarch64 to verify x86-64 and freebsd x86-64 to verify aarch64 --- .github/workflows/manual_reproducible.yml | 55 +++++++++++++++++++---- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/.github/workflows/manual_reproducible.yml b/.github/workflows/manual_reproducible.yml index 3c13098..c26bce1 100644 --- a/.github/workflows/manual_reproducible.yml +++ b/.github/workflows/manual_reproducible.yml @@ -127,7 +127,7 @@ jobs: fail-fast: false matrix: runner: [ubuntu-24.04, ubuntu-24.04-arm] - build: &targets + build: - target: x86_64-unknown-linux-gnu zigtargetsuffix: .2.28 - target: x86_64-unknown-linux-musl @@ -136,6 +136,15 @@ jobs: zigtargetsuffix: .2.28 - target: aarch64-unknown-linux-musl zigtargetsuffix: "" + exclude: + - runner: ubuntu-24.04-arm + build: + target: aarch64-unknown-linux-gnu + zigtargetsuffix: .2.28 + - runner: ubuntu-24.04-arm + build: + target: aarch64-unknown-linux-musl + zigtargetsuffix: "" env: TARGET: ${{ matrix.build.target }} ZIGTARGETSUFFIX: ${{ matrix.build.zigtargetsuffix }} @@ -217,7 +226,11 @@ jobs: strategy: fail-fast: false matrix: - build: *targets + build: + - target: aarch64-unknown-linux-gnu + zigtargetsuffix: .2.28 + - target: aarch64-unknown-linux-musl + zigtargetsuffix: "" env: TARGET: ${{ matrix.build.target }} ZIGTARGETSUFFIX: ${{ matrix.build.zigtargetsuffix }} @@ -308,16 +321,42 @@ jobs: compression-level: 0 hashes: - name: Print SHA-256 hashes + name: SHA-256 (${{ matrix.pair.target }}, ${{ matrix.original }} vs ${{ matrix.pair.verify }}) needs: [zigbuild, freebsd] if: ${{ !cancelled() }} runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + original: [ubuntu-24.04] + pair: + - target: x86_64-unknown-linux-gnu.2.28 + verify: ubuntu-24.04-arm + - target: x86_64-unknown-linux-musl + verify: ubuntu-24.04-arm + - target: aarch64-unknown-linux-gnu.2.28 + verify: freebsd-x86_64 + - target: aarch64-unknown-linux-musl + verify: freebsd-x86_64 steps: - - name: Download all host artifacts + - name: Download original artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - pattern: reproducible-* - path: artifacts + name: reproducible-${{ matrix.pair.target }}-${{ matrix.original }} + path: artifacts/${{ matrix.original }} - - name: Print SHA-256 hashes - run: find artifacts -type f -exec sha256sum {} + + - name: Download verification artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: reproducible-${{ matrix.pair.target }}-${{ matrix.pair.verify }} + path: artifacts/${{ matrix.pair.verify }} + + - name: Print and compare SHA-256 hashes + env: + ORIGINAL: ${{ matrix.original }} + VERIFY: ${{ matrix.pair.verify }} + run: | + find artifacts -type f -exec sha256sum {} + + (cd "artifacts/$ORIGINAL" && sha256sum celq celq.tar.gz) > original.sha256 + (cd "artifacts/$VERIFY" && sha256sum celq celq.tar.gz) > verification.sha256 + diff -u original.sha256 verification.sha256 From 1805b159f7f4cf66a277672326d45a350bcbf243 Mon Sep 17 00:00:00 2001 From: Ivan Carvalho Date: Mon, 7 Sep 2026 23:04:59 -0400 Subject: [PATCH 09/11] Use cargo-auditable for reproducible builds again --- .github/workflows/manual_reproducible.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_reproducible.yml b/.github/workflows/manual_reproducible.yml index c26bce1..e6047fd 100644 --- a/.github/workflows/manual_reproducible.yml +++ b/.github/workflows/manual_reproducible.yml @@ -281,8 +281,11 @@ jobs: run: | set -eu . "$HOME/.cargo/env" - install -m 755 freebsd-tools/cargo-zigbuild freebsd-tools/cargo-auditable "$HOME/.cargo/bin/" - export PATH="/tmp/zig-x86_64-freebsd-0.15.2:$PATH" + install -m 755 freebsd-tools/cargo-zigbuild "$HOME/.cargo/bin/" + mkdir -p /home/runner/.cargo/bin + install -m 755 freebsd-tools/cargo-auditable /home/runner/.cargo/bin/cargo-auditable + export PATH="/home/runner/.cargo/bin:/tmp/zig-x86_64-freebsd-0.15.2:$PATH" + test "$(command -v cargo-auditable)" = /home/runner/.cargo/bin/cargo-auditable rustup target add --toolchain 1.94.1 "$TARGET" # Host job environment variables are not inherited by the VM. From f4ff7a2e1b25df0d60f6b2f5f9d8b63c62e94996 Mon Sep 17 00:00:00 2001 From: Ivan Carvalho Date: Mon, 7 Sep 2026 23:08:14 -0400 Subject: [PATCH 10/11] Use cargo-auditable for reproducible builds again (pt 2) --- .github/workflows/manual_reproducible.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_reproducible.yml b/.github/workflows/manual_reproducible.yml index e6047fd..56dc110 100644 --- a/.github/workflows/manual_reproducible.yml +++ b/.github/workflows/manual_reproducible.yml @@ -195,7 +195,7 @@ jobs: run: | rustc -vV cargo -vV - cargo zigbuild -vv --locked --release --no-default-features --features from-toml,from-yaml,greppable,from-xml --target "${TARGET}${ZIGTARGETSUFFIX}" + cargo auditable zigbuild -vv --locked --release --no-default-features --features from-toml,from-yaml,greppable,from-xml --target "${TARGET}${ZIGTARGETSUFFIX}" - name: Create release archive run: | @@ -301,7 +301,7 @@ jobs: echo "Using patched Cargo: $CARGO" rustc -vV "$CARGO" -vV - "$CARGO" zigbuild -vv --locked --release --no-default-features --features from-toml,from-yaml,greppable,from-xml --target "${TARGET}${ZIGTARGETSUFFIX}" + "$CARGO" auditable zigbuild -vv --locked --release --no-default-features --features from-toml,from-yaml,greppable,from-xml --target "${TARGET}${ZIGTARGETSUFFIX}" mkdir -p artifacts cp "target/${TARGET}/release/celq" artifacts/celq From edecd64741f8ade08d5d3041b30029b53b9d1402 Mon Sep 17 00:00:00 2001 From: Ivan Carvalho Date: Mon, 7 Sep 2026 23:21:50 -0400 Subject: [PATCH 11/11] Re-enable mimalloc for reproducible build --- .github/workflows/manual_reproducible.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_reproducible.yml b/.github/workflows/manual_reproducible.yml index 56dc110..9913f60 100644 --- a/.github/workflows/manual_reproducible.yml +++ b/.github/workflows/manual_reproducible.yml @@ -195,7 +195,7 @@ jobs: run: | rustc -vV cargo -vV - cargo auditable zigbuild -vv --locked --release --no-default-features --features from-toml,from-yaml,greppable,from-xml --target "${TARGET}${ZIGTARGETSUFFIX}" + cargo auditable zigbuild -vv --locked --release --target "${TARGET}${ZIGTARGETSUFFIX}" - name: Create release archive run: | @@ -301,7 +301,7 @@ jobs: echo "Using patched Cargo: $CARGO" rustc -vV "$CARGO" -vV - "$CARGO" auditable zigbuild -vv --locked --release --no-default-features --features from-toml,from-yaml,greppable,from-xml --target "${TARGET}${ZIGTARGETSUFFIX}" + "$CARGO" auditable zigbuild -vv --locked --release --target "${TARGET}${ZIGTARGETSUFFIX}" mkdir -p artifacts cp "target/${TARGET}/release/celq" artifacts/celq