diff --git a/.github/justfile b/.github/justfile index c7c07caac1c..e1b2b59d4a9 100644 --- a/.github/justfile +++ b/.github/justfile @@ -1,6 +1,6 @@ set working-directory := '..' -# Build and publish the development environment to nix.moq.dev. +# Build and publish the development environment and @moq/libav to nix.moq.dev. cache: bash sh/gh/cache.sh diff --git a/.github/workflows/cache.yml b/.github/workflows/cache.yml index 6ffce7cb10d..8a63655fb9b 100644 --- a/.github/workflows/cache.yml +++ b/.github/workflows/cache.yml @@ -27,7 +27,7 @@ jobs: persist-credentials: false - uses: ./.github/actions/nix - uses: ./.github/actions/just - - name: Cache the dev shell + - name: Cache the dev shell and @moq/libav env: NIX_SIGNING_KEY: ${{ secrets.NIX_SIGNING_KEY }} AWS_ACCESS_KEY_ID: ${{ secrets.NIX_R2_ACCESS_KEY_ID }} diff --git a/infra/README.md b/infra/README.md index 62d7ade9aaf..ff4d6a36191 100644 --- a/infra/README.md +++ b/infra/README.md @@ -268,7 +268,9 @@ gh workflow run rpm-repo.yml --ref release -f tag=moq-relay-v1.2.3 released packages, pushed by `release-nix.yml` on a release tag, and never expires. `nix-ci-moq-dev` holds the dev shells `cache.yml` pushes after every merge to `main`, for x86 Linux, ARM Linux, and ARM macOS. It publishes the full -build closure from `nix develop --profile`, uploading only missing paths. +build closure from `nix develop --profile`, uploading only missing paths, plus +`moq-libav`, its wasm build, and its downloaded sources, which `bun install` +builds outside the dev shell. Its lifecycle rule (`infra/nix/lifecycle.json`) deletes each object 30 days after upload; the next warm re-uploads what it still needs. `flake.nix` sets `fallback = true` because that expiry can remove a path before one that diff --git a/js/libav/package.nix b/js/libav/package.nix index 2296397580d..f62af9400f7 100644 --- a/js/libav/package.nix +++ b/js/libav/package.nix @@ -18,6 +18,7 @@ fetchurl, fetchFromGitHub, importNpmLock, + linkFarm, emscripten, nodejs, python3, @@ -143,7 +144,18 @@ stdenvNoCC.mkDerivation { runHook postInstall ''; - passthru = { inherit wasm; }; + passthru = { + inherit wasm; + # Everything the wasm build downloads, which sh/gh/cache.sh publishes so a + # rebuild never waits on ffmpeg.org, xiph.org, GitHub or npm. + sources = linkFarm "moq-libav-sources" ( + sources + // { + "libav.js" = wasm.src; + "npm-deps" = wasm.npmDeps; + } + ); + }; meta = { description = "libav.js decoding Opus and AAC for the @moq/hang WebCodecs polyfill"; diff --git a/quest/m0/ci/README.md b/quest/m0/ci/README.md index 72e2fcaf392..d109c85b2d7 100644 --- a/quest/m0/ci/README.md +++ b/quest/m0/ci/README.md @@ -175,4 +175,3 @@ The historical samples locate candidates; they are not controlled benchmarks. - [A release merge opens no stale release PR](/quest/m0/ci/release-pr-after-release.md) - `just rs release` publishes before it proposes, so a merged release isn't re-proposed - [Renovate keeps the workspace-hack generated](/quest/m0/ci/dependabot-workspace-hack.md) - a protected scheduled Renovate runs `cargo hakari generate` in cargo update PRs, replacing Dependabot's cargo updates - [Build two queue entries at once if it helps](/quest/m0/ci/queue-concurrency.md) - a measured trial of `maximumEntriesToBuild: 2` against the serial lane's burst latency -- [CI never downloads libav's sources](/quest/m0/ci/libav-sources.md) - `moq-libav` substitutes from `nix.moq.dev`, so an upstream mirror can't fail Check diff --git a/quest/m0/ci/libav-sources.md b/quest/m0/ci/libav-sources.md deleted file mode 100644 index 961fd9536a1..00000000000 --- a/quest/m0/ci/libav-sources.md +++ /dev/null @@ -1,26 +0,0 @@ -# [XS] CI never downloads libav's sources from upstream mirrors - -## Goal - -A `just ci` run never fetches `@moq/libav`'s source tarballs from their -upstream sites: `moq-libav` (or at least its fixed-output sources) substitutes -from `nix.moq.dev`, so a flaky upstream mirror can't fail a check. Done when a -clean checkout's Check builds `moq-libav` with no request to `ffmpeg.org`, -`downloads.xiph.org` or GitHub archives. - -## Plan - -Planned 2026-11-04 with the maintainer. Check failed twice on 2026-11-03/04 -for this alone: [moq-dev/moq#7685](https://github.com/moq-dev/moq/pull/7685) -and [moq-dev/moq#7760](https://github.com/moq-dev/moq/pull/7760) (run -41777407147: "cannot download ffmpeg-9.0.tar.xz from any mirror", then -`moq-libav` and the `moq` postinstall failed). `js/libav/package.nix` -`fetchurl`s `ffmpeg-9.0.tar.xz`, `opus-1.6.1.tar.gz` and -`emfiberthreads-1.3.tar.gz` whenever `moq-libav` isn't substituted. - -Find why `moq-libav` was built at all: whether `sh/gh/cache.sh` pushes it (and -its sources) with the dev shell's build closure, or whether its derivation -changed without being pushed. Prefer pushing the built `moq-libav` and its -source fixed-output derivations to `nix-ci-moq-dev` over mirroring tarballs -by hand. Related: [A cache error is a miss](/quest/m0/ci/nix-cache-5xx.md) -covers `nix.moq.dev` errors, not upstream ones. diff --git a/sh/gh/cache.sh b/sh/gh/cache.sh index 1beaed69496..42467b20152 100644 --- a/sh/gh/cache.sh +++ b/sh/gh/cache.sh @@ -1,14 +1,24 @@ #!/usr/bin/env bash -# Publish the dev shell's build closure, including its build-time inputs. +# Publish what CI builds to run checks: the dev shell's build closure, +# including its build-time inputs, and @moq/libav with its sources. set -euo pipefail : "${NIX_SIGNING_KEY:?not set; the job needs the cache-write Environment}" : "${AWS_ACCESS_KEY_ID:?not set}" : "${AWS_SECRET_ACCESS_KEY:?not set}" -profile=$(mktemp -d) -trap 'rm -rf "$profile"' EXIT +roots=$(mktemp -d) +trap 'rm -rf "$roots"' EXIT # Building the devShell derivation alone omits inputs that nix develop needs. -nix develop --profile "$profile/devshell" --command true -just infra nix push nix-ci-moq-dev "$profile/devshell" +nix develop --profile "$roots/devshell" --command true +just infra nix push nix-ci-moq-dev "$roots/devshell" + +# `bun install` builds @moq/libav outside the dev shell (js/libav/build.ts). +# The package copies the wasm build's files rather than referencing them, so +# the wasm build goes up separately: a change to the package alone (a version +# bump, README or index.d.ts) still substitutes the ffmpeg compile. With the +# sources here too, a PR that changes the wasm build never downloads from the +# upstream mirrors. +nix build .#moq-libav .#moq-libav.wasm .#moq-libav.sources --out-link "$roots/libav" +just infra nix push nix-ci-moq-dev "$roots"/libav*