From d245b58a49a3968a07d9ddadbfe7ee779fd45dfc Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 15:28:25 -0800 Subject: [PATCH 1/4] ci(setup-nix): disable HTTP/2 for Nix downloads A failed HTTP/2 connection fails every NAR multiplexed on it. Nix resumes NARs that had fully arrived at EOF, gets HTTP 416, and fails the substitution (NixOS/nix#16722). Separate HTTP/1.1 connections confine a failure to one transfer. Signed-off-by: Matthew Grossman --- .github/actions/setup-nix/action.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/actions/setup-nix/action.yml b/.github/actions/setup-nix/action.yml index 5ad0a071fa..0ae11edfe9 100644 --- a/.github/actions/setup-nix/action.yml +++ b/.github/actions/setup-nix/action.yml @@ -26,6 +26,11 @@ runs: with: github-token: ${{ github.token }} summarize: false + # One failed HTTP/2 connection fails every NAR multiplexed on it, and + # Nix fails completed NARs it then resumes at EOF with HTTP 416 + # (NixOS/nix#16722). Separate HTTP/1.1 connections limit the damage. + extra-conf: | + http2 = false - uses: cachix/cachix-action@5f2d7c5294214f71b873db4b969586b980625e71 # v17 with: From 33a7d5086d75e53ff41a9720491cdb3c17930ee7 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 15:28:25 -0800 Subject: [PATCH 2/4] ci: add temporary Nix HTTP/2 download experiment Signed-off-by: Matthew Grossman --- .github/workflows/nix-http2-experiment.yml | 81 ++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 .github/workflows/nix-http2-experiment.yml diff --git a/.github/workflows/nix-http2-experiment.yml b/.github/workflows/nix-http2-experiment.yml new file mode 100644 index 0000000000..181a530366 --- /dev/null +++ b/.github/workflows/nix-http2-experiment.yml @@ -0,0 +1,81 @@ +# Temporary A/B experiment for the Nix HTTP 416 substitution flake. Do not merge. +name: Nix HTTP/2 Experiment + +on: + push: + branches: + - "ci/cachix-download-flake/**" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + fetch: + name: fetch (shard ${{ matrix.shard }}) + runs-on: ubuntu-24.04 + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] + env: + ITERATIONS: "3" + SHARD: ${{ matrix.shard }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/setup-nix + + - name: Fetch the test-runner closure into fresh stores, alternating http2 + run: | + set -uo pipefail + nix config show | grep -E '^(http2|http-connections|download-attempts|substituters|download-buffer-size) ' || true + out="$(nix eval --raw .#openshell-test-runner.outPath)" + echo "closure: $out" + sudo mkdir -p /mnt/exp && sudo chown "$(id -u):$(id -g)" /mnt/exp + mkdir -p results/logs + echo "shard,iteration,arm,http2,rc,seconds,resumes,eof416,range_err,terminal,started" > results/summary.csv + if [ $((SHARD % 2)) -eq 0 ]; then arms="h2 h1"; else arms="h1 h2"; fi + for i in $(seq 1 "$ITERATIONS"); do + for arm in $arms; do + if [ "$arm" = h2 ]; then h2=true; else h2=false; fi + root="/mnt/exp/store-$i-$arm" + log="results/logs/$i-$arm.log" + started="$(date -u +%FT%TZ)" + start=$(date +%s) + nix build --no-link --max-jobs 0 --store "local?root=$root" \ + --option http2 "$h2" --option fallback false \ + --option substituters "https://openshell.cachix.org https://cache.nixos.org" \ + --option trusted-public-keys "openshell.cachix.org-1:OAr5MunsfH5PZvUsfD08OtGx5RtcwdNZGJdU5FqLm5w= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" \ + "$out" > "$log" 2>&1 + rc=$? + secs=$(( $(date +%s) - start )) + resumes=$(grep -c 'retrying from offset' "$log") + e416=$(grep -c 'HTTP error 416' "$log") + rerr=$(grep -c 'Requested range was not delivered' "$log") + term=$(grep -c "^error: unable to download" "$log") + echo "$SHARD,$i,$arm,$h2,$rc,$secs,$resumes,$e416,$rerr,$term,$started" >> results/summary.csv + grep -E 'unable to download|no substituter|will be fetched|could not be realised' "$log" | sed "s/^/[$i $arm] /" || true + chmod -R u+w "$root" 2>/dev/null; rm -rf "$root" + # Keep only the interesting lines to bound artifact size. + grep -E 'unable to download|no substituter|will be fetched|error' "$log" > "$log.tmp"; mv "$log.tmp" "$log" + done + done + cat results/summary.csv + { + echo '```' + cat results/summary.csv + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: nix-http2-experiment-${{ matrix.shard }} + path: results From fb6aa726c46ad838e4515fab9f03e935657c4e72 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 15:30:40 -0800 Subject: [PATCH 3/4] ci: keep the Nix HTTP/2 experiment running on zero matches Signed-off-by: Matthew Grossman --- .github/workflows/nix-http2-experiment.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nix-http2-experiment.yml b/.github/workflows/nix-http2-experiment.yml index 181a530366..6fbb42c277 100644 --- a/.github/workflows/nix-http2-experiment.yml +++ b/.github/workflows/nix-http2-experiment.yml @@ -23,7 +23,7 @@ jobs: matrix: shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] env: - ITERATIONS: "3" + ITERATIONS: "5" SHARD: ${{ matrix.shard }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -33,8 +33,10 @@ jobs: - uses: ./.github/actions/setup-nix - name: Fetch the test-runner closure into fresh stores, alternating http2 + shell: bash run: | - set -uo pipefail + # grep -c exits 1 on zero matches; record every result instead of stopping. + set +e -u nix config show | grep -E '^(http2|http-connections|download-attempts|substituters|download-buffer-size) ' || true out="$(nix eval --raw .#openshell-test-runner.outPath)" echo "closure: $out" From 0413e8d18d17ab3f5b73a6570526ced8f6706b06 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 20:24:09 -0800 Subject: [PATCH 4/4] ci: remove temporary Nix HTTP/2 download experiment Signed-off-by: Matthew Grossman --- .github/workflows/nix-http2-experiment.yml | 83 ---------------------- 1 file changed, 83 deletions(-) delete mode 100644 .github/workflows/nix-http2-experiment.yml diff --git a/.github/workflows/nix-http2-experiment.yml b/.github/workflows/nix-http2-experiment.yml deleted file mode 100644 index 6fbb42c277..0000000000 --- a/.github/workflows/nix-http2-experiment.yml +++ /dev/null @@ -1,83 +0,0 @@ -# Temporary A/B experiment for the Nix HTTP 416 substitution flake. Do not merge. -name: Nix HTTP/2 Experiment - -on: - push: - branches: - - "ci/cachix-download-flake/**" - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - fetch: - name: fetch (shard ${{ matrix.shard }}) - runs-on: ubuntu-24.04 - timeout-minutes: 120 - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] - env: - ITERATIONS: "5" - SHARD: ${{ matrix.shard }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/setup-nix - - - name: Fetch the test-runner closure into fresh stores, alternating http2 - shell: bash - run: | - # grep -c exits 1 on zero matches; record every result instead of stopping. - set +e -u - nix config show | grep -E '^(http2|http-connections|download-attempts|substituters|download-buffer-size) ' || true - out="$(nix eval --raw .#openshell-test-runner.outPath)" - echo "closure: $out" - sudo mkdir -p /mnt/exp && sudo chown "$(id -u):$(id -g)" /mnt/exp - mkdir -p results/logs - echo "shard,iteration,arm,http2,rc,seconds,resumes,eof416,range_err,terminal,started" > results/summary.csv - if [ $((SHARD % 2)) -eq 0 ]; then arms="h2 h1"; else arms="h1 h2"; fi - for i in $(seq 1 "$ITERATIONS"); do - for arm in $arms; do - if [ "$arm" = h2 ]; then h2=true; else h2=false; fi - root="/mnt/exp/store-$i-$arm" - log="results/logs/$i-$arm.log" - started="$(date -u +%FT%TZ)" - start=$(date +%s) - nix build --no-link --max-jobs 0 --store "local?root=$root" \ - --option http2 "$h2" --option fallback false \ - --option substituters "https://openshell.cachix.org https://cache.nixos.org" \ - --option trusted-public-keys "openshell.cachix.org-1:OAr5MunsfH5PZvUsfD08OtGx5RtcwdNZGJdU5FqLm5w= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" \ - "$out" > "$log" 2>&1 - rc=$? - secs=$(( $(date +%s) - start )) - resumes=$(grep -c 'retrying from offset' "$log") - e416=$(grep -c 'HTTP error 416' "$log") - rerr=$(grep -c 'Requested range was not delivered' "$log") - term=$(grep -c "^error: unable to download" "$log") - echo "$SHARD,$i,$arm,$h2,$rc,$secs,$resumes,$e416,$rerr,$term,$started" >> results/summary.csv - grep -E 'unable to download|no substituter|will be fetched|could not be realised' "$log" | sed "s/^/[$i $arm] /" || true - chmod -R u+w "$root" 2>/dev/null; rm -rf "$root" - # Keep only the interesting lines to bound artifact size. - grep -E 'unable to download|no substituter|will be fetched|error' "$log" > "$log.tmp"; mv "$log.tmp" "$log" - done - done - cat results/summary.csv - { - echo '```' - cat results/summary.csv - echo '```' - } >> "$GITHUB_STEP_SUMMARY" - - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: always() - with: - name: nix-http2-experiment-${{ matrix.shard }} - path: results