From 15ba36d3e6cb399e892bf964a4dee9c9a23804b8 Mon Sep 17 00:00:00 2001
From: Ken VanDine <ken@vandine.org>
Date: Thu, 2 Jul 2026 09:23:26 -0400
Subject: [PATCH 1/7] ci: fail release if expected artifacts are missing, per
 backend family
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Port the artifact-completeness gate from lemonade-sdk/llama.cpp#16 to
this repo's release job, adapted to its build matrix:

- cuda: 8 sm_* variants on ubuntu x64 + windows x64, plus sm_121-only
  on ubuntu arm64 (GB10) — 17 files total.
- rocm-therock: TheRock nightly ROCm 7.13.0 on windows + ubuntu.
- rocm-hipsdk: official/legacy HIP SDK ROCm on windows (7.1.1) +
  ubuntu (7.2.1) — kept separate from rocm-therock since they're
  different SDKs, not just different platforms of the same backend.
- cpu: ubuntu + windows avx2 builds.
- metal: macos-14 + macos-latest builds.

Each family is all-or-nothing: if any artifact in a family is
missing, every artifact for that family is dropped from the release
so a release never contains a partial/inconsistent set (e.g. CUDA
split across sm_* or platforms). Other families are unaffected by a
gap in one. A release with zero artifacts overall is never created.

Also brings the release/upload mechanism up to parity with
llama.cpp's: an idempotent create-or-get-release step (safe to
re-run after a partial failure) and a retrying, resumable asset
upload, replacing the old anzz1/action-create-release +
github-script@v3 flow.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .github/workflows/build.yml | 221 +++++++++++++++++++++++++++++++-----
 1 file changed, 192 insertions(+), 29 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index c159f8a8a..7242ac630 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -971,48 +971,211 @@ jobs:
           TAG_NAME="${{ env.BRANCH_NAME }}-${{ steps.commit_count.outputs.count }}-${{ steps.commit.outputs.short }}"
           echo "name=${TAG_NAME}" >> $GITHUB_OUTPUT
 
-      - name: Check if tag exists
-        id: check_tag
+      - name: Check release artifacts
+        id: check_artifacts
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          BRANCH: ${{ env.BRANCH_NAME }}
+          SHORT: ${{ steps.commit.outputs.short }}
         run: |
-          TAG_NAME="${{ steps.tag_name.outputs.name }}"
-          if git ls-remote --tags https://github.com/${{ github.repository }}.git | grep -q "refs/tags/${TAG_NAME}$"; then
-            echo "Tag ${TAG_NAME} already exists in ${{ github.repository }}. Skipping release creation."
-            echo "exists=true" >> $GITHUB_OUTPUT
+          total_missing=0
+          summary=""
+
+          check_family() {
+            local family="$1"; shift
+            local patterns=("$@")
+            local missing=0
+            for pattern in "${patterns[@]}"; do
+              if ! ls $pattern >/dev/null 2>&1; then
+                echo "::error::[$family] Missing artifact: $pattern"
+                missing=$((missing + 1))
+              fi
+            done
+            if [ "$missing" -gt 0 ]; then
+              echo "::warning::$family family is incomplete ($missing artifact(s) missing) — dropping all $family artifacts from this release."
+              for pattern in "${patterns[@]}"; do
+                rm -fv $pattern 2>/dev/null || true
+              done
+              total_missing=$((total_missing + missing))
+              summary="${summary}${summary:+, }${family}: $missing missing, family dropped"
+            fi
+          }
+
+          # Each backend family must be published as a complete, internally
+          # consistent set across every platform/variant/SDK it covers — never
+          # a partial mix. A gap in one family drops that whole family from
+          # the release; other families are unaffected.
+
+          # CUDA: 8 sm_* variants on ubuntu x64 and windows x64; only sm_121
+          # (GB10) is built for ubuntu arm64.
+          cuda_patterns=()
+          for sm in sm_75 sm_80 sm_86 sm_89 sm_90 sm_100 sm_120 sm_121; do
+            cuda_patterns+=("artifact/sd-${BRANCH}-${SHORT}-ubuntu-cuda-${sm}-x64.tar.xz")
+            cuda_patterns+=("artifact/sd-${BRANCH}-${SHORT}-windows-cuda-${sm}-x64.zip")
+          done
+          cuda_patterns+=("artifact/sd-${BRANCH}-${SHORT}-ubuntu-cuda-sm_121-arm64.tar.xz")
+          check_family cuda "${cuda_patterns[@]}"
+
+          # TheRock nightly ROCm 7.13.0 (windows-latest-rocm + the 7.13.0 leg
+          # of ubuntu-latest-rocm). Kept separate from the HIP SDK line below
+          # since they're different SDKs, not just different platforms.
+          check_family rocm-therock \
+            "artifact/sd-${BRANCH}-${SHORT}-bin-win-rocm-7.13.0-x64.zip" \
+            "artifact/sd-${BRANCH}-${SHORT}-bin-Linux-Ubuntu-24.04-x86_64-rocm-7.13.0.zip"
+
+          # Official/legacy HIP SDK ROCm (windows-latest-cmake-hip 7.1.1 +
+          # the 7.2.1 leg of ubuntu-latest-rocm).
+          check_family rocm-hipsdk \
+            "artifact/sd-${BRANCH}-${SHORT}-bin-win-rocm-7.1.1-x64.zip" \
+            "artifact/sd-${BRANCH}-${SHORT}-bin-Linux-Ubuntu-24.04-x86_64-rocm-7.2.1.zip"
+
+          # CPU: ubuntu-latest-cmake's filename embeds the runner's OS name/
+          # version via lsb_release, so it's wildcarded rather than pinned.
+          check_family cpu \
+            "artifact/sd-${BRANCH}-${SHORT}-bin-Linux-*-x86_64.zip" \
+            "artifact/sd-${BRANCH}-${SHORT}-bin-win-avx2-x64.zip"
+
+          check_family metal \
+            "artifact/sd-${BRANCH}-${SHORT}-bin-Darwin-arm64-metal.zip" \
+            "artifact/sd-${BRANCH}-${SHORT}-bin-Darwin-arm64-metal-latest.zip"
+
+          echo "missing=$total_missing" >> "$GITHUB_OUTPUT"
+          echo "summary=$summary" >> "$GITHUB_OUTPUT"
+
+          files=$(find ./artifact -maxdepth 1 \( -name '*.zip' -o -name '*.tar.xz' \) 2>/dev/null | wc -l)
+
+          # Never create or update a release with nothing to publish.
+          if [ "$files" -eq 0 ]; then
+            echo "::error::No release artifacts found in ./artifact — aborting before creating a release."
+            exit 1
+          fi
+
+          if [ "$total_missing" -gt 0 ]; then
+            echo "$total_missing artifact(s) missing ($summary) — continuing to publish the $files artifact(s) from complete families. The job will still fail below so the gap gets noticed."
           else
-            echo "Tag ${TAG_NAME} does not exist in ${{ github.repository }}. Proceeding with release."
-            echo "exists=false" >> $GITHUB_OUTPUT
+            echo "All expected artifacts present. Found $files artifact(s) ready to upload."
           fi
 
-      - name: Create release
-        id: create_release
-        if: steps.check_tag.outputs.exists == 'false'
-        uses: anzz1/action-create-release@v1
+      - name: Determine release summary
+        id: release_summary
+        run: |
+          if [[ "${{ github.event_name }}" == "schedule" ]]; then
+            echo "value=Nightly release for ${{ github.sha }}" >> "$GITHUB_OUTPUT"
+          else
+            echo "value=$(git log -1 --pretty=%s)" >> "$GITHUB_OUTPUT"
+          fi
+
+      # Get the release for this tag, creating it if it does not exist yet.
+      # This is idempotent: if a previous run created the release but failed
+      # partway through uploading (e.g. a transient GitHub outage), re-running
+      # reuses the existing release so the upload step below can fill the gaps.
+      - name: Create or get release
+        id: release
+        uses: actions/github-script@v8
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TAG_NAME: ${{ steps.tag_name.outputs.name }}
+          RELEASE_SUMMARY: ${{ steps.release_summary.outputs.value }}
         with:
-          tag_name: ${{ steps.tag_name.outputs.name }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const tag = process.env.TAG_NAME;
+            const { owner, repo } = context.repo;
+            let release;
+            try {
+              release = (await github.rest.repos.getReleaseByTag({ owner, repo, tag })).data;
+              core.info(`Reusing existing release ${tag} (id ${release.id})`);
+            } catch (e) {
+              if (e.status !== 404) throw e;
+              try {
+                release = (await github.rest.repos.createRelease({ owner, repo, tag_name: tag, body: process.env.RELEASE_SUMMARY })).data;
+                core.info(`Created release ${tag} (id ${release.id})`);
+              } catch (createError) {
+                if (createError.status !== 422) throw createError;
+                release = (await github.rest.repos.getReleaseByTag({ owner, repo, tag })).data;
+                core.info(`Reusing concurrently created release ${tag} (id ${release.id})`);
+              }
+            }
+            core.setOutput('id', release.id);
 
       - name: Upload release
         id: upload_release
-        if: steps.check_tag.outputs.exists == 'false'
-        uses: actions/github-script@v3
+        uses: actions/github-script@v8
+        env:
+          RELEASE_ID: ${{ steps.release.outputs.id }}
         with:
           github-token: ${{secrets.GITHUB_TOKEN}}
           script: |
-            const path = require('path');
             const fs = require('fs');
-            const release_id = '${{ steps.create_release.outputs.id }}';
-            for (let file of await fs.readdirSync('./artifact')) {
-              if (path.extname(file) === '.zip' || file.endsWith('.tar.xz')) {
-                console.log('uploadReleaseAsset', file);
-                await github.repos.uploadReleaseAsset({
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  release_id: release_id,
-                  name: file,
-                  data: await fs.readFileSync(`./artifact/${file}`)
-                });
+            const { owner, repo } = context.repo;
+            const release_id = Number(process.env.RELEASE_ID);
+
+            const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+            // Retry transient failures (network errors, 5xx, rate limits) with
+            // exponential backoff so a momentary GitHub blip does not abort the
+            // whole upload and leave the release partially populated.
+            async function withRetry(label, fn, attempts = 5) {
+              for (let i = 1; ; i++) {
+                try {
+                  return await fn();
+                } catch (e) {
+                  const transient = !e.status || e.status >= 500 || e.status === 429;
+                  if (i >= attempts || !transient) throw e;
+                  const delay = Math.min(30000, 1000 * 2 ** (i - 1));
+                  core.warning(`${label} failed (attempt ${i}/${attempts}): ${e.message}. Retrying in ${delay}ms`);
+                  await sleep(delay);
+                }
+              }
+            }
+
+            // Assets already attached (from an earlier partial run). GitHub only
+            // creates an asset once its upload completes, so anything listed here
+            // is intact and can be skipped — re-runs only fill the gaps.
+            const existing = new Set();
+            for (const a of await github.paginate(github.rest.repos.listReleaseAssets, { owner, repo, release_id })) {
+              if (a.state === 'uploaded') {
+                existing.add(a.name);
+              } else {
+                core.warning(`deleting incomplete asset ${a.name} (${a.state})`);
+                await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id: a.id });
+              }
+            }
+
+            const files = fs.readdirSync('./artifact').filter((f) => f.endsWith('.zip') || f.endsWith('.tar.xz'));
+
+            let uploaded = 0;
+            let skipped = 0;
+            for (const file of files) {
+              if (existing.has(file)) {
+                core.info(`skipping ${file} (already uploaded)`);
+                skipped++;
+                continue;
               }
+              const data = fs.readFileSync(`./artifact/${file}`);
+              await withRetry(`upload ${file}`, () =>
+                github.rest.repos.uploadReleaseAsset({ owner, repo, release_id, name: file, data })
+                  .catch(async (e) => {
+                    if (e.status !== 422) throw e;
+                    const assets = await github.paginate(github.rest.repos.listReleaseAssets, { owner, repo, release_id });
+                    const asset = assets.find((a) => a.name === file);
+                    if (asset?.state === 'uploaded') {
+                      core.info(`accepting ${file} (already uploaded)`);
+                      return;
+                    }
+                    if (asset) {
+                      core.warning(`deleting conflicting asset ${file} (${asset.state})`);
+                      await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id: asset.id });
+                      throw new Error(`retrying upload for ${file} after deleting conflicting asset`);
+                    }
+                    throw e;
+                  }));
+              core.info(`uploaded ${file}`);
+              uploaded++;
             }
+
+            core.info(`Done: ${uploaded} uploaded, ${skipped} already present, ${files.length} total.`);
+
+      - name: Fail if any artifacts were missing
+        if: steps.check_artifacts.outputs.missing != '0'
+        run: |
+          echo "::error::${{ steps.check_artifacts.outputs.missing }} artifact(s) were missing (${{ steps.check_artifacts.outputs.summary }}) — the incomplete families were dropped from this release; other families were still published. See the 'Check release artifacts' step for the full per-file list."
+          exit 1

From 03e249db83d2e2785645488373b09d3e912f2a23 Mon Sep 17 00:00:00 2001
From: Ken VanDine <ken@vandine.org>
Date: Thu, 2 Jul 2026 09:39:12 -0400
Subject: [PATCH 2/7] ci: run release job even when a build job fails
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Without always(), GitHub Actions' default skip-if-a-needed-job-failed
behavior meant a single failed build job (e.g. one CUDA sm_* matrix leg,
several of which use fail-fast: false) could skip the release job
outright, before the per-family completeness check ever ran — silently
producing no release instead of a correct partial one.

The other issue found in the equivalent llama.cpp fix (a hard-fail step
that could block unrelated families) doesn't apply here: this workflow
has no cross-job merge step comparable to llama.cpp's Windows CPU-into-
other-zips step, so there's nothing else that needs the same treatment.

Follow-up to #14.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .github/workflows/build.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 7242ac630..8b5cb7b7a 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -921,7 +921,12 @@ jobs:
             sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-Darwin-arm64-metal${{ matrix.archive_suffix }}.zip
 
   release:
-    if: ${{ github.event_name == 'schedule' || github.event.inputs.create_release == 'true' }}
+    # always() overrides the default behavior of skipping this job when a
+    # needed build job fails (e.g. one CUDA sm_* matrix leg fails while
+    # sibling legs succeed and upload). The job must still run so the
+    # per-family checks below can decide what's actually complete and safe
+    # to publish.
+    if: ${{ always() && (github.event_name == 'schedule' || github.event.inputs.create_release == 'true') }}
 
     permissions:
       contents: write

From bd1c5f2d43fc4a73df98c765652f559d6a0df0d7 Mon Sep 17 00:00:00 2001
From: Ken VanDine <ken@vandine.org>
Date: Thu, 2 Jul 2026 09:42:22 -0400
Subject: [PATCH 3/7] ci(windows-cuda): pass /bigobj to fix C1128 build failure

stable-diffusion.cpp's Windows CUDA build fails with:

  fatal error C1128: number of sections exceeded object file format
  limit: compile with /bigobj

src/stable-diffusion.cpp combined with the CUDA backend's generated
code exceeds MSVC's default COFF section limit per object file. The
windows-latest-cmake job already works around the same class of issue
with -DCMAKE_CXX_FLAGS='/bigobj'; apply the same flag to the CUDA
build.
---
 .github/workflows/build.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index c159f8a8a..e4fb3bcbe 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -250,7 +250,8 @@ jobs:
             -DGGML_CUDA=ON ^
             -DCMAKE_CUDA_ARCHITECTURES=%cmake_arch% ^
             -DGGML_NATIVE=OFF ^
-            -DSD_BUILD_SHARED_LIBS=ON
+            -DSD_BUILD_SHARED_LIBS=ON ^
+            -DCMAKE_CXX_FLAGS=/bigobj
           cmake --build build --config Release
 
       - name: Get commit hash

From bd5e37fcb9fa81d590df837c2dae01d82310a019 Mon Sep 17 00:00:00 2001
From: Ken VanDine <ken@vandine.org>
Date: Thu, 2 Jul 2026 10:00:29 -0400
Subject: [PATCH 4/7] ci: prune stale release assets from families dropped in
 the current run
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The release is reused across runs against the same tag (idempotent
create-or-get). If a family (e.g. rocm-therock) was complete and
published in an earlier run but the current run finds it incomplete and
drops it locally, its assets from that earlier run were never removed —
the release would keep looking complete for a family this run couldn't
actually reproduce.

The upload step now reconciles existing release assets against the
current ./artifact file set: anything not fully uploaded (a broken
partial upload, as before) or not present in this run's file set (stale
leftovers from a dropped family) gets deleted before uploading.

Addresses review feedback on lemonade-sdk/stable-diffusion.cpp#14. The
always() fix requested in the same review was already applied in
03e249d.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .github/workflows/build.yml | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 8b5cb7b7a..30efca810 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -1132,21 +1132,28 @@ jobs:
               }
             }
 
-            // Assets already attached (from an earlier partial run). GitHub only
-            // creates an asset once its upload completes, so anything listed here
-            // is intact and can be skipped — re-runs only fill the gaps.
+            const files = fs.readdirSync('./artifact').filter((f) => f.endsWith('.zip') || f.endsWith('.tar.xz'));
+            const wanted = new Set(files);
+
+            // Reconcile existing release assets against what this run decided to
+            // publish: drop anything not fully uploaded (a broken partial upload)
+            // and anything not in the current file set. The latter case matters
+            // because the release is reused across runs against the same tag —
+            // if a family was complete and published in an earlier run but this
+            // run finds it incomplete and drops it locally, its assets from that
+            // earlier run would otherwise linger and make the family look
+            // complete even though this run couldn't reproduce it.
             const existing = new Set();
             for (const a of await github.paginate(github.rest.repos.listReleaseAssets, { owner, repo, release_id })) {
-              if (a.state === 'uploaded') {
+              if (a.state === 'uploaded' && wanted.has(a.name)) {
                 existing.add(a.name);
               } else {
-                core.warning(`deleting incomplete asset ${a.name} (${a.state})`);
+                const reason = a.state !== 'uploaded' ? `incomplete (${a.state})` : 'stale (not part of this run)';
+                core.warning(`deleting ${reason} asset ${a.name}`);
                 await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id: a.id });
               }
             }
 
-            const files = fs.readdirSync('./artifact').filter((f) => f.endsWith('.zip') || f.endsWith('.tar.xz'));
-
             let uploaded = 0;
             let skipped = 0;
             for (const file of files) {

From f41bf4f420764813c4c8a06950ebe384d5d5187f Mon Sep 17 00:00:00 2001
From: Ken VanDine <ken@vandine.org>
Date: Thu, 2 Jul 2026 09:23:26 -0400
Subject: [PATCH 5/7] ci: fail release if expected artifacts are missing, per
 backend family
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Port the artifact-completeness gate from lemonade-sdk/llama.cpp#16 to
this repo's release job, adapted to its build matrix:

- cuda: 8 sm_* variants on ubuntu x64 + windows x64, plus sm_121-only
  on ubuntu arm64 (GB10) — 17 files total.
- rocm-therock: TheRock nightly ROCm 7.13.0 on windows + ubuntu.
- rocm-hipsdk: official/legacy HIP SDK ROCm on windows (7.1.1) +
  ubuntu (7.2.1) — kept separate from rocm-therock since they're
  different SDKs, not just different platforms of the same backend.
- cpu: ubuntu + windows avx2 builds.
- metal: macos-14 + macos-latest builds.

Each family is all-or-nothing: if any artifact in a family is
missing, every artifact for that family is dropped from the release
so a release never contains a partial/inconsistent set (e.g. CUDA
split across sm_* or platforms). Other families are unaffected by a
gap in one. A release with zero artifacts overall is never created.

Also brings the release/upload mechanism up to parity with
llama.cpp's: an idempotent create-or-get-release step (safe to
re-run after a partial failure) and a retrying, resumable asset
upload, replacing the old anzz1/action-create-release +
github-script@v3 flow.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .github/workflows/build.yml | 221 +++++++++++++++++++++++++++++++-----
 1 file changed, 192 insertions(+), 29 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index e4fb3bcbe..8780351d6 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -972,48 +972,211 @@ jobs:
           TAG_NAME="${{ env.BRANCH_NAME }}-${{ steps.commit_count.outputs.count }}-${{ steps.commit.outputs.short }}"
           echo "name=${TAG_NAME}" >> $GITHUB_OUTPUT
 
-      - name: Check if tag exists
-        id: check_tag
+      - name: Check release artifacts
+        id: check_artifacts
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          BRANCH: ${{ env.BRANCH_NAME }}
+          SHORT: ${{ steps.commit.outputs.short }}
         run: |
-          TAG_NAME="${{ steps.tag_name.outputs.name }}"
-          if git ls-remote --tags https://github.com/${{ github.repository }}.git | grep -q "refs/tags/${TAG_NAME}$"; then
-            echo "Tag ${TAG_NAME} already exists in ${{ github.repository }}. Skipping release creation."
-            echo "exists=true" >> $GITHUB_OUTPUT
+          total_missing=0
+          summary=""
+
+          check_family() {
+            local family="$1"; shift
+            local patterns=("$@")
+            local missing=0
+            for pattern in "${patterns[@]}"; do
+              if ! ls $pattern >/dev/null 2>&1; then
+                echo "::error::[$family] Missing artifact: $pattern"
+                missing=$((missing + 1))
+              fi
+            done
+            if [ "$missing" -gt 0 ]; then
+              echo "::warning::$family family is incomplete ($missing artifact(s) missing) — dropping all $family artifacts from this release."
+              for pattern in "${patterns[@]}"; do
+                rm -fv $pattern 2>/dev/null || true
+              done
+              total_missing=$((total_missing + missing))
+              summary="${summary}${summary:+, }${family}: $missing missing, family dropped"
+            fi
+          }
+
+          # Each backend family must be published as a complete, internally
+          # consistent set across every platform/variant/SDK it covers — never
+          # a partial mix. A gap in one family drops that whole family from
+          # the release; other families are unaffected.
+
+          # CUDA: 8 sm_* variants on ubuntu x64 and windows x64; only sm_121
+          # (GB10) is built for ubuntu arm64.
+          cuda_patterns=()
+          for sm in sm_75 sm_80 sm_86 sm_89 sm_90 sm_100 sm_120 sm_121; do
+            cuda_patterns+=("artifact/sd-${BRANCH}-${SHORT}-ubuntu-cuda-${sm}-x64.tar.xz")
+            cuda_patterns+=("artifact/sd-${BRANCH}-${SHORT}-windows-cuda-${sm}-x64.zip")
+          done
+          cuda_patterns+=("artifact/sd-${BRANCH}-${SHORT}-ubuntu-cuda-sm_121-arm64.tar.xz")
+          check_family cuda "${cuda_patterns[@]}"
+
+          # TheRock nightly ROCm 7.13.0 (windows-latest-rocm + the 7.13.0 leg
+          # of ubuntu-latest-rocm). Kept separate from the HIP SDK line below
+          # since they're different SDKs, not just different platforms.
+          check_family rocm-therock \
+            "artifact/sd-${BRANCH}-${SHORT}-bin-win-rocm-7.13.0-x64.zip" \
+            "artifact/sd-${BRANCH}-${SHORT}-bin-Linux-Ubuntu-24.04-x86_64-rocm-7.13.0.zip"
+
+          # Official/legacy HIP SDK ROCm (windows-latest-cmake-hip 7.1.1 +
+          # the 7.2.1 leg of ubuntu-latest-rocm).
+          check_family rocm-hipsdk \
+            "artifact/sd-${BRANCH}-${SHORT}-bin-win-rocm-7.1.1-x64.zip" \
+            "artifact/sd-${BRANCH}-${SHORT}-bin-Linux-Ubuntu-24.04-x86_64-rocm-7.2.1.zip"
+
+          # CPU: ubuntu-latest-cmake's filename embeds the runner's OS name/
+          # version via lsb_release, so it's wildcarded rather than pinned.
+          check_family cpu \
+            "artifact/sd-${BRANCH}-${SHORT}-bin-Linux-*-x86_64.zip" \
+            "artifact/sd-${BRANCH}-${SHORT}-bin-win-avx2-x64.zip"
+
+          check_family metal \
+            "artifact/sd-${BRANCH}-${SHORT}-bin-Darwin-arm64-metal.zip" \
+            "artifact/sd-${BRANCH}-${SHORT}-bin-Darwin-arm64-metal-latest.zip"
+
+          echo "missing=$total_missing" >> "$GITHUB_OUTPUT"
+          echo "summary=$summary" >> "$GITHUB_OUTPUT"
+
+          files=$(find ./artifact -maxdepth 1 \( -name '*.zip' -o -name '*.tar.xz' \) 2>/dev/null | wc -l)
+
+          # Never create or update a release with nothing to publish.
+          if [ "$files" -eq 0 ]; then
+            echo "::error::No release artifacts found in ./artifact — aborting before creating a release."
+            exit 1
+          fi
+
+          if [ "$total_missing" -gt 0 ]; then
+            echo "$total_missing artifact(s) missing ($summary) — continuing to publish the $files artifact(s) from complete families. The job will still fail below so the gap gets noticed."
           else
-            echo "Tag ${TAG_NAME} does not exist in ${{ github.repository }}. Proceeding with release."
-            echo "exists=false" >> $GITHUB_OUTPUT
+            echo "All expected artifacts present. Found $files artifact(s) ready to upload."
           fi
 
-      - name: Create release
-        id: create_release
-        if: steps.check_tag.outputs.exists == 'false'
-        uses: anzz1/action-create-release@v1
+      - name: Determine release summary
+        id: release_summary
+        run: |
+          if [[ "${{ github.event_name }}" == "schedule" ]]; then
+            echo "value=Nightly release for ${{ github.sha }}" >> "$GITHUB_OUTPUT"
+          else
+            echo "value=$(git log -1 --pretty=%s)" >> "$GITHUB_OUTPUT"
+          fi
+
+      # Get the release for this tag, creating it if it does not exist yet.
+      # This is idempotent: if a previous run created the release but failed
+      # partway through uploading (e.g. a transient GitHub outage), re-running
+      # reuses the existing release so the upload step below can fill the gaps.
+      - name: Create or get release
+        id: release
+        uses: actions/github-script@v8
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TAG_NAME: ${{ steps.tag_name.outputs.name }}
+          RELEASE_SUMMARY: ${{ steps.release_summary.outputs.value }}
         with:
-          tag_name: ${{ steps.tag_name.outputs.name }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const tag = process.env.TAG_NAME;
+            const { owner, repo } = context.repo;
+            let release;
+            try {
+              release = (await github.rest.repos.getReleaseByTag({ owner, repo, tag })).data;
+              core.info(`Reusing existing release ${tag} (id ${release.id})`);
+            } catch (e) {
+              if (e.status !== 404) throw e;
+              try {
+                release = (await github.rest.repos.createRelease({ owner, repo, tag_name: tag, body: process.env.RELEASE_SUMMARY })).data;
+                core.info(`Created release ${tag} (id ${release.id})`);
+              } catch (createError) {
+                if (createError.status !== 422) throw createError;
+                release = (await github.rest.repos.getReleaseByTag({ owner, repo, tag })).data;
+                core.info(`Reusing concurrently created release ${tag} (id ${release.id})`);
+              }
+            }
+            core.setOutput('id', release.id);
 
       - name: Upload release
         id: upload_release
-        if: steps.check_tag.outputs.exists == 'false'
-        uses: actions/github-script@v3
+        uses: actions/github-script@v8
+        env:
+          RELEASE_ID: ${{ steps.release.outputs.id }}
         with:
           github-token: ${{secrets.GITHUB_TOKEN}}
           script: |
-            const path = require('path');
             const fs = require('fs');
-            const release_id = '${{ steps.create_release.outputs.id }}';
-            for (let file of await fs.readdirSync('./artifact')) {
-              if (path.extname(file) === '.zip' || file.endsWith('.tar.xz')) {
-                console.log('uploadReleaseAsset', file);
-                await github.repos.uploadReleaseAsset({
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  release_id: release_id,
-                  name: file,
-                  data: await fs.readFileSync(`./artifact/${file}`)
-                });
+            const { owner, repo } = context.repo;
+            const release_id = Number(process.env.RELEASE_ID);
+
+            const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+            // Retry transient failures (network errors, 5xx, rate limits) with
+            // exponential backoff so a momentary GitHub blip does not abort the
+            // whole upload and leave the release partially populated.
+            async function withRetry(label, fn, attempts = 5) {
+              for (let i = 1; ; i++) {
+                try {
+                  return await fn();
+                } catch (e) {
+                  const transient = !e.status || e.status >= 500 || e.status === 429;
+                  if (i >= attempts || !transient) throw e;
+                  const delay = Math.min(30000, 1000 * 2 ** (i - 1));
+                  core.warning(`${label} failed (attempt ${i}/${attempts}): ${e.message}. Retrying in ${delay}ms`);
+                  await sleep(delay);
+                }
+              }
+            }
+
+            // Assets already attached (from an earlier partial run). GitHub only
+            // creates an asset once its upload completes, so anything listed here
+            // is intact and can be skipped — re-runs only fill the gaps.
+            const existing = new Set();
+            for (const a of await github.paginate(github.rest.repos.listReleaseAssets, { owner, repo, release_id })) {
+              if (a.state === 'uploaded') {
+                existing.add(a.name);
+              } else {
+                core.warning(`deleting incomplete asset ${a.name} (${a.state})`);
+                await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id: a.id });
+              }
+            }
+
+            const files = fs.readdirSync('./artifact').filter((f) => f.endsWith('.zip') || f.endsWith('.tar.xz'));
+
+            let uploaded = 0;
+            let skipped = 0;
+            for (const file of files) {
+              if (existing.has(file)) {
+                core.info(`skipping ${file} (already uploaded)`);
+                skipped++;
+                continue;
               }
+              const data = fs.readFileSync(`./artifact/${file}`);
+              await withRetry(`upload ${file}`, () =>
+                github.rest.repos.uploadReleaseAsset({ owner, repo, release_id, name: file, data })
+                  .catch(async (e) => {
+                    if (e.status !== 422) throw e;
+                    const assets = await github.paginate(github.rest.repos.listReleaseAssets, { owner, repo, release_id });
+                    const asset = assets.find((a) => a.name === file);
+                    if (asset?.state === 'uploaded') {
+                      core.info(`accepting ${file} (already uploaded)`);
+                      return;
+                    }
+                    if (asset) {
+                      core.warning(`deleting conflicting asset ${file} (${asset.state})`);
+                      await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id: asset.id });
+                      throw new Error(`retrying upload for ${file} after deleting conflicting asset`);
+                    }
+                    throw e;
+                  }));
+              core.info(`uploaded ${file}`);
+              uploaded++;
             }
+
+            core.info(`Done: ${uploaded} uploaded, ${skipped} already present, ${files.length} total.`);
+
+      - name: Fail if any artifacts were missing
+        if: steps.check_artifacts.outputs.missing != '0'
+        run: |
+          echo "::error::${{ steps.check_artifacts.outputs.missing }} artifact(s) were missing (${{ steps.check_artifacts.outputs.summary }}) — the incomplete families were dropped from this release; other families were still published. See the 'Check release artifacts' step for the full per-file list."
+          exit 1

From 40969fbb322446f587210fa130ee247d1dc15238 Mon Sep 17 00:00:00 2001
From: Ken VanDine <ken@vandine.org>
Date: Thu, 2 Jul 2026 09:39:12 -0400
Subject: [PATCH 6/7] ci: run release job even when a build job fails
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Without always(), GitHub Actions' default skip-if-a-needed-job-failed
behavior meant a single failed build job (e.g. one CUDA sm_* matrix leg,
several of which use fail-fast: false) could skip the release job
outright, before the per-family completeness check ever ran — silently
producing no release instead of a correct partial one.

The other issue found in the equivalent llama.cpp fix (a hard-fail step
that could block unrelated families) doesn't apply here: this workflow
has no cross-job merge step comparable to llama.cpp's Windows CPU-into-
other-zips step, so there's nothing else that needs the same treatment.

Follow-up to #14.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .github/workflows/build.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 8780351d6..e40b70fbc 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -922,7 +922,12 @@ jobs:
             sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-Darwin-arm64-metal${{ matrix.archive_suffix }}.zip
 
   release:
-    if: ${{ github.event_name == 'schedule' || github.event.inputs.create_release == 'true' }}
+    # always() overrides the default behavior of skipping this job when a
+    # needed build job fails (e.g. one CUDA sm_* matrix leg fails while
+    # sibling legs succeed and upload). The job must still run so the
+    # per-family checks below can decide what's actually complete and safe
+    # to publish.
+    if: ${{ always() && (github.event_name == 'schedule' || github.event.inputs.create_release == 'true') }}
 
     permissions:
       contents: write

From 6296594e1b02f1ffd07ce1ef6df006c9b683727d Mon Sep 17 00:00:00 2001
From: Ken VanDine <ken@vandine.org>
Date: Thu, 2 Jul 2026 10:00:29 -0400
Subject: [PATCH 7/7] ci: prune stale release assets from families dropped in
 the current run
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The release is reused across runs against the same tag (idempotent
create-or-get). If a family (e.g. rocm-therock) was complete and
published in an earlier run but the current run finds it incomplete and
drops it locally, its assets from that earlier run were never removed —
the release would keep looking complete for a family this run couldn't
actually reproduce.

The upload step now reconciles existing release assets against the
current ./artifact file set: anything not fully uploaded (a broken
partial upload, as before) or not present in this run's file set (stale
leftovers from a dropped family) gets deleted before uploading.

Addresses review feedback on lemonade-sdk/stable-diffusion.cpp#14. The
always() fix requested in the same review was already applied in
03e249d.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .github/workflows/build.yml | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index e40b70fbc..fdd355b3f 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -1133,21 +1133,28 @@ jobs:
               }
             }
 
-            // Assets already attached (from an earlier partial run). GitHub only
-            // creates an asset once its upload completes, so anything listed here
-            // is intact and can be skipped — re-runs only fill the gaps.
+            const files = fs.readdirSync('./artifact').filter((f) => f.endsWith('.zip') || f.endsWith('.tar.xz'));
+            const wanted = new Set(files);
+
+            // Reconcile existing release assets against what this run decided to
+            // publish: drop anything not fully uploaded (a broken partial upload)
+            // and anything not in the current file set. The latter case matters
+            // because the release is reused across runs against the same tag —
+            // if a family was complete and published in an earlier run but this
+            // run finds it incomplete and drops it locally, its assets from that
+            // earlier run would otherwise linger and make the family look
+            // complete even though this run couldn't reproduce it.
             const existing = new Set();
             for (const a of await github.paginate(github.rest.repos.listReleaseAssets, { owner, repo, release_id })) {
-              if (a.state === 'uploaded') {
+              if (a.state === 'uploaded' && wanted.has(a.name)) {
                 existing.add(a.name);
               } else {
-                core.warning(`deleting incomplete asset ${a.name} (${a.state})`);
+                const reason = a.state !== 'uploaded' ? `incomplete (${a.state})` : 'stale (not part of this run)';
+                core.warning(`deleting ${reason} asset ${a.name}`);
                 await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id: a.id });
               }
             }
 
-            const files = fs.readdirSync('./artifact').filter((f) => f.endsWith('.zip') || f.endsWith('.tar.xz'));
-
             let uploaded = 0;
             let skipped = 0;
             for (const file of files) {