ci(studio): run the edit accuracy slice on PRs and the full grid in the merge queue

.github/workflows/ci.yml

@@ -79,7 +79,7 @@ jobs:
       codex_plugin: ${{ steps.filter.outputs.codex_plugin }}
       gcp_beginframe: ${{ steps.filter.outputs.gcp_beginframe }}
       studio: ${{ steps.filter.outputs.studio }}
-      edit_accuracy: ${{ steps.filter.outputs.edit_accuracy }}
+      edit_accuracy: ${{ steps.edit_accuracy_filter.outputs.edit_accuracy }}
     steps:
       # Force git-based change detection instead of the pull_request REST API.
       # The API path can fail the whole workflow on transient listFiles
@@ -189,45 +189,70 @@ jobs:
               - "packages/studio-server/**"
               - "bun.lock"
               - ".github/workflows/ci.yml"
+            gcp_beginframe:
+              - "packages/gcp-cloud-run/Dockerfile"
+              - "packages/aws-lambda/scripts/probe-beginframe.ts"
+              - "packages/engine/src/services/browserManager.ts"
+              - "package.json"
+              - "bun.lock"
+              - ".github/workflows/ci.yml"
+      # Only what the edit accuracy bench runs: the Studio it drives, the preview server the CLI hosts it from,
+      # and the engine and producer modules it captures frames with (the import closure of browserManager,
+      # frameCapture and both fileServers). Encoders, renders and other CLI commands are not on its path.
+      # Its own base: the merge queue's is the event's base_sha, since that run's HEAD is not a PR merge commit.
+      - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4
+        id: edit_accuracy_filter
+        if: steps.edit.outputs.passed != 'true' && (github.event_name == 'pull_request' || github.event_name == 'merge_group')
+        with:
+          token: ""
+          base: ${{ github.event_name == 'merge_group' && github.event.merge_group.base_sha || steps.base.outputs.sha }}
+          predicate-quantifier: some-with-excludes
+          filters: |
             edit_accuracy:
               - "packages/studio/src/**"
+              - "packages/studio/index.html"
+              - "packages/studio/vite.*.ts"
+              - "packages/studio/public/**"
+              - "packages/studio/tests/e2e/edit-accuracy/**"
+              - "packages/studio/tests/e2e/webmcp-host.mjs"
               - "packages/studio-server/src/**"
               - "packages/core/src/**"
               - "packages/core/scripts/**"
-              - "packages/cli/src/**"
-              - "packages/player/src/**"
+              - "packages/core/schemas/**"
               - "packages/parsers/src/**"
-              - "packages/producer/src/**"
-              - "packages/engine/src/**"
-              - "packages/lint/src/**"
+              - "packages/player/src/**"
               - "packages/sdk/src/**"
-              - "packages/studio/index.html"
-              - "packages/studio/vite.config.ts"
-              - "packages/studio/public/**"
+              - "packages/lint/src/**"
+              - "packages/cli/src/{cli,help,version}.ts"
+              - "packages/cli/src/renderSetupWorker*.ts"
+              - "packages/cli/src/commands/{preview*,add,browser}.ts"
+              - "packages/cli/src/server/**"
+              - "packages/cli/src/browser/**"
+              - "packages/cli/src/utils/**"
+              - "packages/cli/src/ui/**"
+              - "packages/cli/src/{background-removal,registry,telemetry}/**"
+              - "packages/cli/src/capture/{captureCompositionFrame,readBoundedResponse}.ts"
+              - "packages/cli/src/media-use/lib/config-lock.mjs"
               - "packages/cli/scripts/**"
-              - "packages/cli/tsup.config.ts"
-              - "packages/studio/package.json"
-              - "packages/cli/package.json"
-              - "packages/core/package.json"
-              - "packages/producer/package.json"
-              - "packages/engine/package.json"
-              - "packages/parsers/package.json"
-              - "packages/lint/package.json"
-              - "packages/sdk/package.json"
-              - "packages/studio-server/package.json"
-              - "packages/player/package.json"
+              - "packages/core/stubs/**"
+              - "packages/engine/src/**"
+              - "packages/producer/src/**"
+              - "packages/producer/stubs/**"
+              - "packages/producer/scripts/build-hf-early-stub.ts"
+              - "packages/producer/build*.mjs"
+              - "packages/player/scripts/verify-runtime-pin.mjs"
+              - "packages/{studio,studio-server,core,parsers,player,sdk,lint,cli,engine,producer}/{package.json,tsconfig.json,tsup.config.ts}"
               - "packages/*/package-subpaths.json"
               - "scripts/package-subpaths.mjs"
-              - "packages/studio/tests/e2e/edit-accuracy/**"
-              - "bun.lock"
-              - ".github/workflows/ci.yml"
-            gcp_beginframe:
-              - "packages/gcp-cloud-run/Dockerfile"
-              - "packages/aws-lambda/scripts/probe-beginframe.ts"
-              - "packages/engine/src/services/browserManager.ts"
+              - "scripts/ci/base-file.sh"
+              - "scripts/ci/install-workspace-dependencies.sh"
+              - ".github/actions/prepare-ffmpeg-bin/**"
               - "package.json"
               - "bun.lock"
               - ".github/workflows/ci.yml"
+              - "!**/*.{test,spec}.{ts,tsx,mjs,js}"
+              - "!**/{__tests__,__fixtures__}/**"
+              - "!**/*.md"
 
   gcp-beginframe-contract:
     name: GCP BeginFrame image contract
@@ -263,7 +288,7 @@ jobs:
     name: Build
     needs: changes
     # A skipped required check passes, so a broken change detection must fail it instead.
-    if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true' || needs.changes.outputs.cli_bundle == 'true') }}
+    if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true' || needs.changes.outputs.cli_bundle == 'true' || needs.changes.outputs.edit_accuracy == 'true') }}
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
@@ -286,7 +311,7 @@ jobs:
       - run: bun run build
       - run: bun run verify:packed-manifests
       - name: Share the built CLI with the edit accuracy shards
-        if: github.event_name == 'pull_request' && needs.changes.outputs.edit_accuracy == 'true'
+        if: needs.changes.outputs.edit_accuracy == 'true'
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: edit-accuracy-cli
@@ -1102,17 +1127,21 @@ jobs:
             /tmp/undo-socket-pool.json
           if-no-files-found: error
 
-  # Manual editing accuracy in the built Studio, gated against the base branch's baseline.json.
+  # Manual editing accuracy in the built Studio, gated against the base branch's baseline.json. A pull request
+  # runs the `pr` slice in 5 shards; the merge queue runs the full grid in 20, so nothing reaches main unmeasured.
+  # The gate job below repeats both shard counts.
   studio-edit-accuracy:
-    name: "Studio: edit accuracy (${{ matrix.shard }}/20)"
+    name: "Studio: edit accuracy (${{ matrix.shard }}/${{ strategy.job-total }})"
     needs: [changes, build]
-    if: github.event_name == 'pull_request' && needs.changes.outputs.edit_accuracy == 'true'
+    if: needs.changes.outputs.edit_accuracy == 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 45
     strategy:
       fail-fast: false
       matrix:
-        shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
+        shard: ${{ fromJSON(github.event_name == 'merge_group' && '[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]' || '[1, 2, 3, 4, 5]') }}
+    env:
+      EDIT_ACCURACY_GRID: ${{ github.event_name == 'merge_group' && 'full' || 'pr' }}
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
       - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
@@ -1136,8 +1165,8 @@ jobs:
       - name: Run this shard, then re-run twice each case whose verdict differs from the base branch
         run: |
           set -euo pipefail
-          bench() { bun run --cwd packages/studio test:edit-accuracy -- --grid full --jobs 2 "$@"; }
-          bench --shard "${{ matrix.shard }}/20" --out /tmp/edit-accuracy/run1
+          bench() { bun run --cwd packages/studio test:edit-accuracy -- --grid "$EDIT_ACCURACY_GRID" --jobs 2 "$@"; }
+          bench --shard "${{ matrix.shard }}/${{ strategy.job-total }}" --out /tmp/edit-accuracy/run1
           mapfile -t FLIPPED < <(node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs flipped \
             /tmp/base-edit-accuracy.json /tmp/edit-accuracy/run1/results.json)
           if (( ${#FLIPPED[@]} > 0 )); then
@@ -1155,7 +1184,7 @@ jobs:
   studio-edit-accuracy-gate:
     name: "Studio: edit accuracy gate"
     needs: [changes, studio-edit-accuracy]
-    if: ${{ !cancelled() && github.event_name == 'pull_request' && needs.changes.outputs.edit_accuracy == 'true' }}
+    if: ${{ !cancelled() && needs.changes.outputs.edit_accuracy == 'true' }}
     runs-on: ubuntu-latest
     timeout-minutes: 5
     permissions:
@@ -1176,16 +1205,19 @@ jobs:
           path: /tmp/edit-accuracy-shards
       - name: Compare every run with the base branch
         id: gate
-        # Smoothness is reported, not gated; a case fails only if it fails 2 of its 3 runs.
+        # Smoothness and newly passing cases are reported, not gated; a case fails only if it fails 2 of its 3 runs.
+        env:
+          SHARDS: ${{ github.event_name == 'merge_group' && 20 || 5 }}
         run: |
           set +e
           mapfile -t RESULTS < <(find /tmp/edit-accuracy-shards -name results.json | sort)
-          node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs gate /tmp/base-edit-accuracy.json \
+          node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs gate --shards "$SHARDS" /tmp/base-edit-accuracy.json \
             packages/studio/tests/e2e/edit-accuracy/baseline.json /tmp/edit-accuracy-gate "${RESULTS[@]}"
           echo "exit_code=$?" >> "$GITHUB_OUTPUT"
           cat /tmp/edit-accuracy-gate/comment.md >> "$GITHUB_STEP_SUMMARY" || true
       - name: Post sticky comment
-        # Fork PRs get a read-only token; the gate below still fires.
+        # Fork PRs get a read-only token; the gate below still fires. The merge queue has no PR to comment on.
+        if: github.event_name == 'pull_request'
         continue-on-error: true
         uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4
         with:

package.json

@@ -53,7 +53,7 @@
     "player:perf": "bun run --filter @hyperframes/player perf",
     "format:check": "oxfmt --check .",
     "knip": "knip",
-    "test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-studio-copy-dashes.test.mjs scripts/check-large-files.test.mjs scripts/prune-fallow-audit-caches.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/clawhub-sync.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/backfill-block-previews.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/ci/base-file.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs registry/components/filmed-screen/filmed-screen.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts",
+    "test:scripts": "node --import tsx --test scripts/animejs-v4-guidance.test.mjs scripts/check-tracked-artifacts.test.mjs scripts/check-registry-set-delta.test.mjs scripts/check-no-main-deletions.test.mjs scripts/check-pr-captures.test.mjs scripts/check-comment-citations.test.mjs scripts/comments-workflow.test.mjs scripts/comment-ratchet.test.mjs scripts/check-docs-snippet-motion.test.mjs scripts/registry-target-paths.test.mjs scripts/check-workspace-contracts.test.mjs scripts/check-media-use-copy-parity.test.mjs scripts/check-svg-sanitize-parity.test.mjs scripts/check-media-use-svg-sanitize-generated.test.mjs scripts/check-package-cycles.test.mjs scripts/check-cli-process-ownership.test.mjs scripts/check-studio-copy-dashes.test.mjs scripts/check-large-files.test.mjs scripts/prune-fallow-audit-caches.test.mjs scripts/package-subpaths.test.mjs scripts/validate-release-channel.test.mjs scripts/clawhub-sync.test.mjs scripts/publish-workflow.test.mjs scripts/pr-edit-concurrency.test.mjs scripts/merge-queue-workflows.test.mjs scripts/install-workspace-dependencies.test.mjs scripts/check-edit-accuracy-filter.test.mjs scripts/draft-changelog.test.ts scripts/set-version.test.ts scripts/release-prepare.test.ts scripts/cli-options.test.ts scripts/changelog-weekly.test.ts scripts/claude-plugin-compression.test.ts scripts/catalog-payload-assets.test.ts scripts/host-registry-assets.test.ts scripts/catalog-preview-temp.test.ts scripts/backfill-block-previews.test.ts scripts/catalog-hosted-files.test.ts scripts/player-cdn-pin.test.ts scripts/studio-runtime-smoke.test.mjs scripts/verify-packed-manifests.test.mjs scripts/lint-skills.test.mjs scripts/creator-editing-recipes.test.mjs packages/gcp-cloud-run/check-dockerfile-workspaces.test.mjs packages/core/scripts/writeGeneratedFile.test.ts scripts/catalog-publication.test.mjs scripts/ci/resolve-workflow-pr.test.mjs scripts/ci/base-file.test.mjs scripts/check-catalog-source-pr.test.mjs scripts/generate-registry-items.test.ts scripts/catalog-drift.test.ts scripts/catalog-fetch-mirror.test.ts scripts/catalog-script-inlining.test.ts scripts/catalog-detail.test.ts scripts/generate-catalog-pages.test.ts scripts/verify-catalog-payloads.test.ts scripts/registry-skill-files.test.ts scripts/creator-editing-capabilities.test.mjs scripts/generate-catalog-previews.test.ts scripts/registry-primitive-payloads.test.ts registry/components/pan-stations/pan-stations.test.mjs registry/components/filmed-screen/filmed-screen.test.mjs && vitest run scripts/catalog/ scripts/contrastRatchet.test.ts scripts/generate-catalog-payloads.test.ts",
     "typecheck:scripts": "tsc --noEmit -p scripts/tsconfig.json",
     "test:skills": "HYPERFRAMES_MEDIA_HOME_REQUIRED=1 node --import ./packages/cli/scripts/test-home.mjs --test 'skills/**/*.test.mjs' 'packages/cli/src/media-use/**/*.test.mjs'",
     "generate:previews": "tsx scripts/generate-template-previews.ts",
@@ -78,6 +78,7 @@
     "lefthook": "^2.1.4",
     "oxfmt": "^0.41.0",
     "oxlint": "^1.56.0",
+    "picomatch": "^4.0.4",
     "tsx": "^4.21.0",
     "typescript": "^5.0.0",
     "vitest": "^4.1.11",

packages/studio/tests/e2e/edit-accuracy/grid.mjs

@@ -130,7 +130,11 @@ const product = (axes) =>
 const caseId = (c) =>
   [c.gesture, c.gsap, c.placement, `r${c.rotation}`, c.nesting, `z${c.zoom}`].join("-");
 
-/** `pr` is a smaller slice for CI; `keyframes` is the GSAP-animated set alone, which `full` also runs. */
+/**
+ * `pr` is the pull request slice, zoom 100 and no rotation in both nestings: it held a regressed case for 10 of the 11
+ * pull requests the full gate caught in its first two weeks (the miss was a zoom 50 pick). The merge queue runs `full`.
+ * `keyframes` is the GSAP-animated set alone, which `full` also runs.
+ */
 export function buildGrid(kind = "full") {
   if (kind === "keyframes") return keyframedCases();
   return (
@@ -139,7 +143,7 @@ export function buildGrid(kind = "full") {
       .filter((c) => c.placement !== "xpercent" || !["none", "idle"].includes(c.gsap))
       .map((c) => ({ id: caseId(c), ...c, other: c.gsap === "idle" }))
       .concat(dragCases(), keyframedCases())
-      .filter((c) => kind !== "pr" || (c.zoom === 100 && c.nesting === "root"))
+      .filter((c) => kind !== "pr" || (c.zoom === 100 && c.rotation === 0))
   );
 }