diff --git a/.changeset/happy-lands-sink.md b/.changeset/happy-lands-sink.md new file mode 100644 index 0000000000..853d812bb3 --- /dev/null +++ b/.changeset/happy-lands-sink.md @@ -0,0 +1,3 @@ +--- + +--- diff --git a/.github/workflows/pr-bundle-size-comment.yml b/.github/workflows/pr-bundle-size-comment.yml index 349e1559d7..3449b96caf 100644 --- a/.github/workflows/pr-bundle-size-comment.yml +++ b/.github/workflows/pr-bundle-size-comment.yml @@ -31,7 +31,22 @@ jobs: with: node-version: '22' + - name: Find bundle-size artifact + id: artifact + env: + GH_TOKEN: ${{ github.token }} + SOURCE_RUN_ID: ${{ github.event.workflow_run.id }} + shell: bash + run: | + set -euo pipefail + found=$(gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/actions/runs/$SOURCE_RUN_ID/artifacts?per_page=100" --jq '[.[].artifacts[]] | any(.name == "bundle-size-analysis" and .expired == false)') + echo "found=$found" >> "$GITHUB_OUTPUT" + if [[ "$found" != "true" ]]; then + echo "No bundle-size artifact for this attempt; report-only verification runs do not rebuild bundles." >> "$GITHUB_STEP_SUMMARY" + fi + - name: Download bundle-size report + if: steps.artifact.outputs.found == 'true' uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: bundle-size-analysis @@ -41,9 +56,11 @@ jobs: run-id: ${{ github.event.workflow_run.id }} - name: Validate downloaded bundle size report + if: steps.artifact.outputs.found == 'true' run: node apps/bundle-size/scripts/validate-report.mts bundle-size-analysis/report.md - name: Publish bundle size PR comment + if: steps.artifact.outputs.found == 'true' env: EXPECTED_HEAD_REPOSITORY: ${{ github.event.workflow_run.head_repository.full_name }} EXPECTED_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} diff --git a/.github/workflows/pr-control-assessment.yml b/.github/workflows/pr-control-assessment.yml new file mode 100644 index 0000000000..968da26dac --- /dev/null +++ b/.github/workflows/pr-control-assessment.yml @@ -0,0 +1,75 @@ +name: Control Assessment PR Summary + +on: + workflow_run: + workflows: [PR] + types: [completed] + +permissions: + actions: read + contents: read + pull-requests: read + +concurrency: + group: control-assessment-${{ github.event.workflow_run.head_repository.id }}-${{ github.event.workflow_run.head_branch }} + cancel-in-progress: false + +jobs: + publish: + name: Publish trusted Checkbox assessment + if: ${{ github.event.workflow_run.event == 'pull_request' && vars.CONTROL_ASSESSMENT_PUBLISH == 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout trusted publisher + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 0 + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + + - name: Find source assessment + id: artifact + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/src/tasks/controlAssessmentCiTask.ts find-artifact + + - name: Download source assessment data + if: steps.artifact.outputs.found == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ steps.artifact.outputs.artifact-name }} + path: ${{ runner.temp }}/checkbox-source-evidence + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ github.event.workflow_run.id }} + + - name: Validate publication without write credentials + id: prepare + if: steps.artifact.outputs.found == 'true' + env: + GITHUB_TOKEN: ${{ github.token }} + BUNDLE_PATH: ${{ runner.temp }}/checkbox-source-evidence/bundle.json + run: node scripts/src/tasks/controlAssessmentCiTask.ts prepare-publication + + - name: Create narrowly scoped publisher token + id: app-token + if: steps.prepare.outputs.eligible == 'true' + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3 + with: + client-id: ${{ vars.CONTROL_ASSESSMENT_APP_CLIENT_ID }} + private-key: ${{ secrets.CONTROL_ASSESSMENT_APP_PRIVATE_KEY }} + permission-contents: write + + - name: Commit the generated summary and SPEC assessment fields to the PR branch + if: steps.prepare.outputs.eligible == 'true' + env: + GITHUB_TOKEN: ${{ github.token }} + ASSESSMENT_WRITE_TOKEN: ${{ steps.app-token.outputs.token }} + BUNDLE_PATH: ${{ runner.temp }}/checkbox-source-evidence/bundle.json + run: node scripts/src/tasks/controlAssessmentCiTask.ts publish diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 3117b5b6eb..3d3c4d70e9 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -13,10 +13,70 @@ concurrency: cancel-in-progress: true jobs: + assessment-inputs: + name: Validate assessment inputs + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: read + contents: read + outputs: + report-only: ${{ steps.verify.outputs.report-only || steps.inspect.outputs.report-only || 'false' }} + source-result: ${{ steps.verify.outputs.source-result }} + steps: + - name: Checkout trusted input validator + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.base.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + + - name: Inspect report-only candidate + id: inspect + env: + GITHUB_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + if [[ -f scripts/src/tasks/controlAssessmentCiTask.ts ]]; then + node scripts/src/tasks/controlAssessmentCiTask.ts inspect + else + echo "candidate=false" >> "$GITHUB_OUTPUT" + echo "report-only=false" >> "$GITHUB_OUTPUT" + echo "Trusted assessment validator is not on the base branch yet. Running all normal source checks." >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Download original assessment evidence + if: steps.inspect.outputs.candidate == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ steps.inspect.outputs.artifact-name }} + path: ${{ runner.temp }}/checkbox-source-evidence + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ steps.inspect.outputs.run-id }} + + - name: Verify unchanged inputs and generated report plus SPEC assessment fields + id: verify + if: steps.inspect.outputs.candidate == 'true' + env: + CANDIDATE_PATH: ${{ steps.inspect.outputs.candidate-path }} + BUNDLE_PATH: ${{ runner.temp }}/checkbox-source-evidence/bundle.json + run: node scripts/src/tasks/controlAssessmentCiTask.ts verify + js: name: JS PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: ubuntu-latest timeout-minutes: 60 + outputs: + unit-artifact: ${{ steps.unit-evidence.outputs.artifact-id }} steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -34,11 +94,28 @@ jobs: - name: Test agent permissions run: yarn test:agent-permissions + - name: Capture existing component test inputs + run: node scripts/src/tasks/controlAssessmentCiTask.ts begin-unit + - name: Build CI + env: + FURN_ASSESSMENT_UNIT_CAPTURE: ${{ github.workspace }}/apps/storybook/artifacts/assessment-unit-ci run: yarn lage buildci + - name: Upload existing component unit evidence + id: unit-evidence + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: checkbox-unit-evidence-${{ github.run_attempt }} + path: apps/storybook/artifacts/assessment-unit-ci + if-no-files-found: warn + retention-days: 14 + bundle-size: name: Bundle Size PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: windows-latest timeout-minutes: 30 permissions: @@ -108,6 +185,8 @@ jobs: android: name: Android PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -132,6 +211,8 @@ jobs: macos: name: macOS PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: macos-26 timeout-minutes: 60 steps: @@ -180,6 +261,8 @@ jobs: macos-storybook: name: macOS Storybook PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: macos-26 timeout-minutes: 40 steps: @@ -233,6 +316,8 @@ jobs: macos-paper-storybook: name: macOS Paper Storybook PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: macos-26 timeout-minutes: 40 steps: @@ -284,6 +369,8 @@ jobs: ios: name: iOS PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: macos-26 timeout-minutes: 60 steps: @@ -321,6 +408,8 @@ jobs: windows: name: Windows PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: windows-latest timeout-minutes: 60 steps: @@ -397,6 +486,8 @@ jobs: windows-storybook: name: Windows Storybook PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: windows-latest timeout-minutes: 60 steps: @@ -463,6 +554,8 @@ jobs: win32: name: Win32 PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: windows-latest timeout-minutes: 60 steps: @@ -511,17 +604,23 @@ jobs: win32-storybook: name: Win32 Storybook PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: windows-latest timeout-minutes: 60 + outputs: + native-artifact: ${{ steps.native-evidence.outputs.artifact-id }} steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 - name: Set up toolchain uses: microsoft/react-native-test-app/.github/actions/setup-toolchain@6046bdd4914c7ceb35ec83b92ae3211a03ddccef # 5.4.14 with: platform: windows - node-version: 22 + node-version: 24 - name: Install dependencies run: yarn @@ -536,10 +635,32 @@ jobs: platform: win32 disabled-input-features: keyboard,physicalClick,wheel + - name: Capture Checkbox execution inputs + id: checkbox-inputs + run: node scripts/src/tasks/controlAssessmentCiTask.ts begin + - name: Smoke test Win32 Storybook + id: smoke + if: ${{ !cancelled() && steps.checkbox-inputs.outcome == 'success' }} run: yarn storybook smoke --win32 --mode stories-and-tests working-directory: apps/storybook + - name: Collect Checkbox native evidence + if: ${{ always() && !cancelled() }} + env: + SMOKE_OUTCOME: ${{ steps.smoke.outcome }} + run: node scripts/src/tasks/controlAssessmentCiTask.ts finish + + - name: Upload Checkbox native evidence + id: native-evidence + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: checkbox-native-evidence-${{ github.run_attempt }} + path: apps/storybook/artifacts/win32/assessment-ci + if-no-files-found: error + retention-days: 14 + - name: Upload Win32 Storybook artifacts if: ${{ always() }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -553,11 +674,59 @@ jobs: ${{ steps.desktop-driver.outputs.cache-root }}/v1/diagnostics/** if-no-files-found: warn + control-assessment: + name: Assess Checkbox evidence + needs: [assessment-inputs, js, win32-storybook] + if: ${{ always() && !cancelled() && needs.assessment-inputs.result == 'success' && needs.assessment-inputs.outputs.report-only != 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout assessed source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + + - name: Download native evidence + if: needs.win32-storybook.outputs.native-artifact != '' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: checkbox-native-evidence-${{ github.run_attempt }} + path: apps/storybook/artifacts/win32/assessment-ci + + - name: Download existing unit evidence + if: needs.js.outputs.unit-artifact != '' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: checkbox-unit-evidence-${{ github.run_attempt }} + path: apps/storybook/artifacts/assessment-unit-ci + + - name: Assess existing results without rerunning tests + env: + UNIT_OUTCOME: ${{ needs.js.result }} + NATIVE_OUTCOME: ${{ needs.win32-storybook.result }} + run: node scripts/src/tasks/controlAssessmentCiTask.ts combine + + - name: Upload Checkbox assessment + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: checkbox-win32-assessment-${{ github.run_attempt }} + path: apps/storybook/artifacts/win32/assessment-ci + if-no-files-found: error + retention-days: 14 + check-changesets: name: Check for Changesets + needs: assessment-inputs runs-on: ubuntu-latest # Skip this check for the version bump PR created by changesets - if: ${{ github.head_ref != 'changeset-release/main' }} + if: ${{ needs.assessment-inputs.outputs.report-only != 'true' && github.head_ref != 'changeset-release/main' }} steps: - name: Checkout @@ -583,6 +752,8 @@ jobs: publish-dry-run-pack: name: NPM Publish Dry Run — Pack + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: ubuntu-latest timeout-minutes: 60 outputs: @@ -624,6 +795,8 @@ jobs: test-links: name: Test repo links + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -641,11 +814,12 @@ jobs: - name: Test markdown links run: yarn lage test-links --verbose --grouped - pr: - name: PR + pr-source: + name: PR source validation if: always() runs-on: ubuntu-latest needs: + - assessment-inputs - js - android - macos @@ -656,6 +830,11 @@ jobs: - publish-dry-run-pack - test-links steps: + - name: Preserve reused source result + if: ${{ needs.assessment-inputs.outputs.report-only == 'true' && needs.assessment-inputs.outputs.source-result != 'success' }} + run: | + echo "The original source validation did not pass. A report-only commit cannot turn it green." + exit 1 - name: Check for failures or cancellations if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} run: | @@ -663,3 +842,23 @@ jobs: exit 1 - name: All required jobs passed run: echo "All required jobs completed successfully." + + pr: + name: PR + if: always() + runs-on: ubuntu-latest + needs: [assessment-inputs, pr-source] + steps: + - name: Require successful source validation + if: ${{ needs.assessment-inputs.result != 'success' || needs.pr-source.result != 'success' }} + run: | + echo "Source validation or assessment-input validation failed." + exit 1 + - name: Require same-PR assessment when publishing is enabled + if: ${{ vars.CONTROL_ASSESSMENT_PUBLISH == 'true' && github.event.pull_request.head.repo.full_name == github.repository && github.base_ref == github.event.repository.default_branch && needs.assessment-inputs.outputs.report-only != 'true' }} + run: | + echo "Source CI completed. Waiting for the trusted publisher to commit the Checkbox assessment to this PR." + echo "The report-only follow-up run must verify that report before this required check can pass." + exit 1 + - name: Required PR validation passed + run: echo "Source checks and the applicable assessment-publication requirements passed." diff --git a/.oxfmtrc.json b/.oxfmtrc.json index 4750ea7dac..831c5e11bb 100644 --- a/.oxfmtrc.json +++ b/.oxfmtrc.json @@ -10,6 +10,7 @@ "**/*.generated.*", "**/lib-commonjs/**", "**/dist/**", + "packages/agentic/components/src/components/checkbox/assessment/ASSESSMENT.md", "packages/agentic/native-core/windows/NativeCore/codegen/**", "**/CHANGELOG.*", "**/CODE_OF_CONDUCT.md", diff --git a/package.json b/package.json index 2c4c791c32..064a6dea28 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "build:clean": "yarn root-build --clean && yarn build", "bundle-size": "yarn workspace @fluentui-react-native/bundle-size measure", "bundle-size:update": "yarn workspace @fluentui-react-native/bundle-size update-baseline", + "assess-control": "node ./scripts/src/cli.ts assess-control", "docs": "yarn workspace fluent-rn-website start", "bundle:repo": "lage bundle", "clean": "tsc -b --clean && lage clean", diff --git a/packages/agentic/components/package.json b/packages/agentic/components/package.json index cf84c88241..8cfc317870 100644 --- a/packages/agentic/components/package.json +++ b/packages/agentic/components/package.json @@ -21,6 +21,7 @@ "spec-source-lock.json", "spec-source-report.json", "src/", + "!src/components/checkbox/assessment/**", "tsconfig.json", "tsconfig.stories.json" ], diff --git a/packages/agentic/components/references/contracts.md b/packages/agentic/components/references/contracts.md index 10da0530d0..efb719a72a 100644 --- a/packages/agentic/components/references/contracts.md +++ b/packages/agentic/components/references/contracts.md @@ -28,6 +28,19 @@ discard the main benefit of YAML comments. Keep this provenance in JSON. `spec/tokens.yaml` remains YAML because it is concise, human-authored token mapping data rather than a generated identity record. +The optional Checkbox assessor snapshot is separate from contract provenance. +It uses a strictly managed `assessment-*` block in the agentic Checkbox +`SPEC.md` front matter; full evidence, digests, and workflow identity remain +in the assessment JSON/report. It never changes `status`, `spec/source.json`, +requirements, or review approval. Only a matched clean execution may populate +its tested commit; local unverified snapshots leave that field null. The +repository-only `@fluentui-react-native/scripts/CONTROL-ASSESSMENT.md` guide +defines generation, fingerprinting, and trusted two-file publication. These +assessment fields do not apply to legacy CheckboxV1. +Generated Checkbox assessment reports are source-repository snapshots, not +package contents; they are excluded from the npm inventory. Their directory +is created only when an assessment is explicitly published. + ## Schema version 2 Every source document has the following document-level shape: diff --git a/plan.md b/plan.md new file mode 100644 index 0000000000..07ff7a5077 --- /dev/null +++ b/plan.md @@ -0,0 +1,249 @@ +# Control readiness assessor: Checkbox pilot + +## Purpose and scope + +Make the evidence behind a control's readiness review visible and repeatable. +Start with the **agentic Checkbox on Win32/Paper**, using tests FURN already +runs. This is an advisory assessor, not a new test framework, release gate, +or automatic maturity promotion. + +The longer-term goal is an agent that evaluates affected controls against their +reviewed contracts, reports gaps, and supports human decisions about lifecycle +stages such as Backlog, Experimental, Beta, and Production. The pilot deliberately +does not assign those stages: the evidence requirements and promotion policy +still need agreement. + +## Design + +```text +Reviewed Checkbox SPEC + requirement metadata + reviewed assertion mappings + | +Existing JS PR Jest results --+-- Existing Win32 Storybook native results + | + Validate execution provenance + | + Assess each mapped requirement + | + JSON + Markdown + proposed SPEC snapshot + | + Optional trusted publisher -> same PR -> human review +``` + +1. **Use the contract as the inventory.** Read CBX-001 through CBX-006 from + `packages/agentic/components/src/components/checkbox/SPEC.md` and its + `spec/source.json`. A filename listed as evidence is not an executed test. +2. **Use reviewed mappings, not guessed coverage.** Map exact test identities to + the particular assertions they establish. Fingerprints detect changes to the + contract, test sources, and relevant harness inputs. Drift requires review; + the pilot does not automatically rewrite its mappings. +3. **Reuse existing execution.** Capture JSON and context from the component + package's existing Jest invocation in **JS PR**, without changing its test + selection or running Checkbox again. Independently collect the canonical + `desktop-test` report and Storybook companion from **Win32 Storybook PR**. + The subsequent **Assess Checkbox evidence** job combines results; it runs no + tests and does not serialize the JS and native workloads. +4. **Bind evidence to what actually ran.** Validate source/build identity, + report hashes, target, execution intervals, dirty state, and mapping + fingerprints. Preserve failures, skips, missing execution, retries, flaky + results, and stale or mismatched evidence rather than turning them into a + pass. Other controls' results are not Checkbox coverage. +5. **Publish an advisory explanation.** Emit machine-readable JSON and readable + Markdown with requirement-level observations, gaps, and provenance. Upload + attempt-specific artifacts and write a job summary. An opt-in publisher can + later commit the generated Markdown and managed SPEC assessment fields + atomically to the same PR; it does not change the human-authored contract. + +Implementation details, commands, schemas, and setup: +[scripts/CONTROL-ASSESSMENT.md](scripts/CONTROL-ASSESSMENT.md). + +## What the initial mappings establish + +| Requirement | Appropriate evidence | Initial pilot scope | +| ---------------------------- | ---------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | +| CBX-001: defaults and API | Runtime unit assertions and compiler/type assertions | Selected existing unit assertions; compiler-result ingestion is not implemented. | +| CBX-002: state and callbacks | Unit assertions for ownership/transitions; native execution for integration claims | Selected existing units plus partial binary-toggle native coverage when physical input is available. | +| CBX-003: anatomy and text | Unit assertions for structure/props; native layout for rendered geometry | Selected existing units; no blanket claim about rendered wrapping. | +| CBX-004: accessibility | Unit assertions for resolved semantics; native assertions for actual platform exposure/actions | Selected existing units plus the existing native callback's checkbox-role assertion. | +| CBX-005: styling and tokens | Unit assertions for resolved styles; visual evidence for visual claims | Selected existing units; no visual qualification. | +| CBX-006: Win32 focus | Native keyboard/focus and geometry evidence | No mapped executed evidence in the initial slice. | + +A requirement does **not** inherently need both unit and native tests. Choose +the evidence layer that proves the claim. Jest's macOS mock and simulated +`Platform.OS` values do not prove native Win32 behavior. The normal compiler +already includes Checkbox type assertions, but the pilot does not yet ingest +that build result. + +The initial mapping intentionally remains bounded to 42 previously reviewed +runtime cases and the existing default-story callback, +`toggles native checked state in both directions`. Subsequent Checkbox tests +already on `main` remain existing tests; unmapped results are not silently +credited as requirement coverage. Expanding those mappings is a follow-up. +This PR adds tooling tests, not new Checkbox behavior tests. + +## Pilot rollout and acceptance + +1. **Review this draft.** Agree on the architecture, partial mappings, report + vocabulary, and CI impact before enabling any automated branch writes. +2. **Validate collection in CI.** Confirm the ordinary Jest invocation executes + once, reports retain package-wide failures, and the assessor combines only + matching source/run/attempt evidence. Exercise missing and skipped evidence + as well as successful execution. +3. **Review the Checkbox report.** Confirm each credited assertion is supported + and missing claims remain explicit. Hosted Win32 Storybook CI disables + `keyboard`, `physicalClick`, and `wheel`; the mapped physical-toggle test + therefore remains skipped there. A green CI job is not physical-input + qualification or full Checkbox readiness. +4. **Enable optional same-PR publication separately.** After trusted tooling is + on the default branch, a maintainer may configure the narrowly scoped GitHub + App and opt-in variable described in the usage guide. The publisher executes + trusted default-branch code, treats PR artifacts as data, and permits only + the generated Checkbox summary and managed SPEC block. Verified report-only + follow-ups may reuse unchanged source validation; ordinary changes still run + normal checks. Fork PRs do not receive automated branch writes. +5. **Expand only after the pilot is useful.** Review mappings for the remaining + existing Checkbox tests, ingest existing compiler results, define readiness + policy with owners, then introduce another control/platform through explicit + adapters rather than generalizing from Checkbox automatically. + +Acceptance means honest, reproducible evidence reporting with preserved source +validation, not a Production rating. No hosted publisher activation, physical +input enablement, repository-rule changes, or additional control rollout is +part of submitting this draft. No historical local assessment is checked in as +current CI evidence. + +## Follow-up: mapping maintenance within one orchestrating agent + +This is **future work, not implemented in the pilot**. Extend the existing +design with one orchestrating agent and separate mapping-maintenance and +deterministic assessment modules; a separately deployed mapping agent is not +required. + +The intended sequence is: + +```text +Control / contract / test / harness change + -> select affected controls and supported targets + -> validate mappings + -> propose updates if needed -> owner review + -> assess actual results using approved mappings + -> publish the advisory report +``` + +The mapping module should identify affected requirements, inspect actual test +assertions rather than infer coverage from names, and propose explicit mapping +deltas with their evidence layer and platform scope. It must distinguish an +existing test that needs mapping from a claim that needs additional assertions. +Not every implementation change requires a mapping change. + +Unchanged, validated mappings can proceed directly. Changed mappings require +owner review and validation before they become authoritative; fingerprints must +not be refreshed merely to suppress a drift warning. The assessor must not +invent a mapping and immediately certify its own interpretation. If approved +mappings are unavailable, it should still report stale mappings and evidence +gaps without claiming readiness. + +Existing test jobs can remain parallel; acceptance of their evidence waits for +valid mappings. Mapping changes are ordinary source changes requiring normal +CI, not generated report-only updates, and are outside the publisher's two-file +allowlist. + +This adds a preparatory phase rather than replacing evidence collection, +provenance checks, deterministic evaluation, or report publication. The pilot +continues to use reviewed static mappings and detect drift without repairing it. + +## Follow-up: affected-control selection, Fabric and multi-control CI + +These are **future work, not implemented in the pilot**. The current +**Assess Checkbox evidence** job assesses Checkbox on ordinary PR runs even +when Checkbox is unchanged. Verified generated-report-only updates remain a +separate exception to avoid publication loops. + +### Assess only affected controls + +- [ ] Select affected controls and supported platform/renderer targets from + the PR changes and explicit dependency metadata, not just component filenames. +- [ ] Include changes to each control's implementation, contract, stories, + tests and mappings. Shared dependencies, test harnesses, assessment tooling + and relevant workflow changes must select their affected dependents, + potentially all supported controls. +- [ ] Skip assessment for reliably unrelated changes. If impact cannot be + determined, assess a conservative superset and explain the uncertainty + rather than silently skipping potentially affected controls. +- [ ] Record the selected controls/targets and selection reasons. Missing + current execution evidence stays missing; an unaffected control's older + report remains a dated snapshot, not validation of the new revision. + +This selection scopes the assessor, not necessarily every existing CI test +job. It must not silently narrow existing test selection or required checks. + +### Add Windows/Fabric assessment + +- [ ] Reuse results from the existing **Windows Storybook PR** Fabric lane; + add evidence collection and provenance validation rather than another test + runner. Fabric tests already run in CI, but the pilot does not assess them. +- [ ] Review requirement-to-assertion mappings for that target. Reuse shared + unit evidence only for claims it establishes; identify native integration + gaps without assuming Paper mappings prove Fabric behavior. +- [ ] Report each control's Office Win32/Paper and Windows/Fabric outcomes + separately, bound to their actual host, renderer and execution. Passing on + one target must not qualify another. +- [ ] Preserve the Fabric lane's disabled `keyboard`, `physicalClick` and + `wheel` capabilities as explicit coverage gaps. Evidence ingestion does not + enable or qualify physical input. + +### Use one generic assessment job + +- [ ] Replace the pilot-specific job with a generic **Assess affected controls** + job. One orchestrator selects controls/targets, validates mappings and + dispatches deterministic assessment through explicit adapters. +- [ ] Consume the existing parallel CI jobs' reports without rerunning tests. + Produce separate JSON/Markdown reports per control and target, plus a + combined summary that preserves each result's failures, skips and gaps. +- [ ] Initially batch assessment in one job to avoid per-control runner startup + and artifact-download overhead. Do not hand-author a CI job for every control. +- [ ] If measured assessment cost later warrants parallelism, use a reusable + job with a dynamically selected, concurrency-limited matrix. This is a + deferred scaling option, not a requirement for the initial expansion. + +## Automatically update the SPEC and report in the same PR + +This is **optional maintainer-enabled publication**, separate from generating +CI artifacts and a job summary. The setup enables later eligible CI runs to +publish; configuring it does not itself write an assessment. + +A maintainer must: + +1. Merge the reviewed tooling/workflows into `main`, initially with publishing disabled. +2. Install a dedicated GitHub App on FURN with **Contents: write**. +3. Configure variable `CONTROL_ASSESSMENT_APP_CLIENT_ID` and secret `CONTROL_ASSESSMENT_APP_PRIVATE_KEY`. +4. Verify organization policies permit the App's branch writes. +5. Set `CONTROL_ASSESSMENT_PUBLISH=true` and validate the complete flow on a pilot PR. + +Scope the App installation to this repository only. No workflow-write, +administration, review-approval, or protection-bypass permission is needed. +Store the private key only as a repository Actions secret, never in source, +reports, or logs. Trial publication on a same-repository PR targeting `main`. + +After an eligible source PR workflow completes, trusted code reconstructs the +assessment from the matching evidence and atomically commits both files to the +same PR branch: + +- The managed `assessment-*` **YAML front-matter block** in Checkbox's `SPEC.md`. + It records the source/package identity, tested commit when supported by + matched clean native evidence, target, outcome, health, provenance, + generation time, and report link. The stage remains `not-determined`. +- The linked `checkbox/assessment/ASSESSMENT.md` generated report. + +The publisher does not edit the human-authored requirements, contract approval, +or lifecycle status. The report commit is not itself labelled as the tested +source. Fork PRs retain artifacts but receive no automated branch write. +The existing required **PR** check waits for verified publication when this +opt-in applies; failed source validation cannot become green through a report +commit. A misconfigured publisher therefore blocks eligible PRs rather than +silently claiming success. + +For rollback, disable `CONTROL_ASSESSMENT_PUBLISH` and rerun the PR workflow if +needed to restore the ordinary artifact-only path. No physical-input enablement +or branch-protection bypass is required. The exact field schema, trust checks, +and setup details remain in +[the implementation guide](scripts/CONTROL-ASSESSMENT.md#maintainer-activation-and-first-hosted-trial). diff --git a/scripts/CONTROL-ASSESSMENT.md b/scripts/CONTROL-ASSESSMENT.md new file mode 100644 index 0000000000..99af2180c2 --- /dev/null +++ b/scripts/CONTROL-ASSESSMENT.md @@ -0,0 +1,657 @@ +# Advisory control assessment: Checkbox pilot + +This repository-only tool reads existing native and unit test reports and the reviewed +Checkbox contract. By default it does **not** run tests; `--run-unit-tests` explicitly +runs the existing Checkbox runtime tests. It does not change controls, edit lifecycle +metadata, or approve a release. It currently supports only the agentic Checkbox +in Win32/Paper Storybook. CI collection and an opt-in same-PR publisher are +implemented alongside the local command; they require deployment and the +maintainer setup below before hosted operation can be verified. + +For the design, pilot acceptance criteria, and rollout boundaries, start with +[`plan.md`](../plan.md). This draft does not enable the publisher or check in +historical local output as current CI evidence. + +From the repository root: + +```powershell +yarn assess-control --component checkbox --evidence "apps\storybook\artifacts\desktop\\run.json" +``` + +Reports default to the ignored +`apps\storybook\artifacts\win32\assessment\assessment.json` and `assessment.md`. +Use `--output "path\to\report-directory"` to retain separate assessments. +Omit both evidence inputs to produce an inventory with explicitly missing execution evidence. +At most one native and one unit execution are accepted per assessment. Their +source/build/freshness bindings are checked independently; do not combine different +local/CI revisions into apparent current coverage. + +## Unit evidence and choosing the right test + +**A requirement does not need both unit and native tests by default.** Choose +evidence that establishes the particular claim: + +| Claim | Suitable evidence | +| ------------------------------------------------------------------------------- | --------------------------------------------------------------------------------- | +| Defaults, state transitions, callbacks, resolved tokens/styles | Complete unit assertions can suffice; no native duplicate is inherently required. | +| Public TypeScript constraints, including `children: never` | TypeScript build/type-check evidence, not Jest's stripped types. | +| Rendered layout or focus-ring geometry | Layout/visual evidence where those properties are claimed. | +| Actual native accessibility dispatch, keyboard focus/navigation, physical input | Native execution for that integration claim; mocked events do not establish it. | + +The initial reviewed unit mapping selects **42 existing runtime cases** in +`checkbox.test.tsx`. It maps applicable JS assertions to parts of CBX-001 through +CBX-005; its remaining gaps identify missing assertions rather than requiring +native duplication of all logic/style tests. It authors no new Checkbox tests. +The mappings remain deliberately partial; full per-requirement completion and +maturity gates have not yet been defined by this pilot. + +Additional existing regression cases merged after the initial mapping remain +visible as `unmappedCases`, not credited requirement coverage. The October 29 +mapping review confirmed the original 42 assertions and default native callback +are unchanged. Newly added defaults, forwarding, state, anatomy, styling, +native naming, and wrapping tests can be mapped in a follow-up; a reported +mapping gap does not necessarily mean a test is absent. + +FURN's existing **JS PR** job already runs package unit tests through +`yarn lage buildci`. The pilot adds assessment and evidence collection, not a new +unit-testing framework. It **does not rerun Checkbox in CI**. An opt-in capture +decorates the existing component package's Jest invocation, preserving its full +file selection, platform, arguments, and exit status while adding JSON output. +The runner is invoked exactly once. JS and native jobs remain parallel; a later +**Assess Checkbox evidence** job validates and combines their artifacts without +running tests or installing dependencies. + +The original full-package JSON is retained. The parser selects only +`checkbox.test.tsx` for requirement mapping; it does not count +`checkbox.types.test.ts` as runtime or compiler qualification. Package-wide +counters and failures remain visible. Missing/cached execution without an +exported report remains missing, never a pass. A failure elsewhere in the Lage +aggregate does not erase a separately captured successful component command; +original source-validation failure still stays failed. + +The component package's `furn.jestPlatform` is `macos`. CI capture records that +**React Native macOS Jest mock** and rejects conflicting inherited platform +overrides. The explicit local runner selects that same profile. +Eight cases simulate Win32 accessibility action naming/handlers via +`Platform.OS`; this is not native UIA dispatch. The 16 Windows/macOS action cases +and the macOS-only system-focus case remain recorded but are not Win32 requirement +coverage. In particular, that focus case does not establish Win32's custom dual-ring +path. An evidence label describes what ran, not a demand for a second test. + +For optional explicit local execution (not used by CI): + +```powershell +yarn assess-control --component checkbox --run-unit-tests --max-age-hours 24 +``` + +This invokes the workspace's declared `test` script with the exact runtime-file +selector, `--runInBand --json --outputFile`. Its contract/story prechecks still +run. It does not select `checkbox.types.test.ts` or turn a Jest pass into type +qualification. A fresh UUID-named directory under +`apps\storybook\artifacts\win32\assessment-units\` retains `jest.json`, +`context.json`, and `execution.json`; the assessment JSON links those inputs. +The native input remains optional and is never synthesized. + +To replay a report without running anything: + +```powershell +yarn assess-control --component checkbox --unit-evidence "path\to\jest.json" +``` + +Replay without its original context remains unverified. To evaluate a genuine +execution binding, also supply `--unit-context "path\to\context.json"` +`--unit-build-id "" --max-age-hours 24`. The sidecar binds the +exact JSON bytes, source SHA, dirty flag, contract and unit mapping fingerprints, +mock platform, start/finish times, build ID, and command exit code. Do not create +one retroactively. A dirty checkout remains unverified even after a fresh run; +recorded results remain visible. Native and unit provenance are separate. + +The reviewed unit fingerprint covers the runtime test file, package/shared Jest +configuration, Babel configuration, platform constant, and package test script/mock +profile, plus the Jest/capture harness source. Changed assertions or mock inputs invalidate the reviewed mapping even +when test names are unchanged. CI additionally binds the complete tracked tree. +The parser accepts a complete package report but requires exactly one Checkbox +runtime suite. It rejects type-only substitution, duplicate suites or Checkbox +assertion identities, inconsistent names, invalid statuses, contradictory counts, +and oversized JSON. Repeated titles in unrelated parameterized suites are counted +without treating them as Checkbox assertion identities; their failures remain visible. +Jest's `focused` status for nonfailed partially skipped suites is accepted and +counted as an executed suite while individual pending assertions remain skipped. +Skipped/todo/missing cases never become passes. Optional assertion `invocations` +and `retryReasons` retain observed retry counts, retry-success flags and prior +failure messages. Repeated execution disqualifies the enclosing package's passes +even without logged retry reasons; its unit outcome is `retried`, not `passed`. +Enclosing unit-command failure +prevents its recorded passes from qualifying; mapped failures remain visible. + +## Assessment snapshot in the agentic SPEC + +The pilot targets +`packages/agentic/components/src/components/checkbox/SPEC.md`, **not** +the legacy `packages/components/Checkbox/SPEC.md`. Its existing +`status: implemented` is contract lifecycle, not a maturity rating. + +The default local command does not modify tracked inputs. Explicitly add +`--update-spec` to save a generated front-matter snapshot and its linked local +`assessment/ASSESSMENT.md`. The block has flat, canonical `assessment-*` fields +between `# furn-assessment:begin` and `# furn-assessment:end`. It records the +source manifest version, assessed source commit, verified tested commit or +`null`, dirty flag, Win32/Paper Storybook target, outcome, assessment health, +undetermined stage, evidence binding, generation time, and report pointer. + +`assessment-source-commit` identifies the assessed checkout/merge, **not proof +that its native tests ran**. `assessment-tested-commit` is populated only for +matched, clean **native** execution evidence. Unit-only evidence does not populate +this native-tested identity or change `assessment-provenance`, which also remains +the native binding. The full report contains separate unit binding and outcomes. +The existing SPEC schema stays v1; its overall outcome/health incorporate unit +failures or stale/mismatched mappings. A dirty or unbound local run keeps the tested commit `null`; +its recorded pass is still visible in the full report. Package version is +descriptive source identity, not qualification of a published npm artifact. +No Backlog/Experimental/Beta/Production promotion is performed. + +The publisher is the separate, opt-in task after the **whole source PR +workflow completes**, not a second assessor or a new test runner. It reconstructs +the report and SPEC fields from original evidence using trusted tooling, +then commits both to that PR atomically. It preserves the PR head's contract +bytes, takes version and assessment identity from the tested merge, and never +labels the generated commit as tested. CI collection writes a proposed +`SPEC.md` only in its artifact; it does not edit the tracked SPEC during tests. +Local snapshots explicitly say they are not bound CI publications. +Generated `checkbox/assessment/` report files are excluded from the component's +npm pack inventory; the linked report is browsed in the source repository. + +After CI deployment, the **Assess Checkbox evidence** job combines the existing +**JS PR** unit report and **Win32 Storybook PR** native bundle and produces files in +`apps/storybook/artifacts/win32/assessment-ci/`. Open the PR's associated +Actions run, select **Summary**, and download +`checkbox-win32-assessment-` under **Artifacts**. It contains +`ASSESSMENT.md`, `assessment.json`, `bundle.json`, proposed `SPEC.md`, and +execution capture data and original package Jest JSON/unit context. The source +artifacts are `checkbox-unit-evidence-` and +`checkbox-native-evidence-`, downloaded to separate directories to +avoid overwrites. All three are retained for 14 days. The readable +report is also written to the job summary. These are workflow artifacts, +not PR file changes; only subsequent publisher commits appear in **Files +changed**. No hosted artifact is implied while this pilot remains undeployed. + +### Historical local validation, October 26, 2026 + +These dated results document the earlier local checkpoint, not validation of +the current draft PR. See the PR description for its current checks and limits. + +The full tooling suite passed **228 tests across 18 suites**, with no failures +or skips. Scripts build/lint, root TypeScript build/repository lint, +instruction/publishing checks, component contracts, and all three workflow +checks passed. The existing normal component command passed **561 tests across +66 suites and 8 snapshots in one Jest invocation**. Replaying its original JSON +selected **42 Checkbox runtime passes** without launching tests or treating +other-component/type suites as Checkbox evidence. Local replay has no original +execution context, so it remains unverified and assigns no maturity stage. + +The required clean-cache plain aggregate reported 95 successful and 1 failed +Lage task before stopping. The continuation completed with **82 successful, +89 skipped, and 4 failed tasks**. The aggregate is **not green**: the unchanged +Windows package-content launcher fails with `spawnSync yarn ENOENT`, two +unchanged codemod newline fixtures fail, and unchanged Desktop Driver and +Storybook Desktop tests fail under parallel load. All timeout-affected files +passed isolated serial retry with original assertions/deadlines: Storybook +Desktop **10 tests/4 suites**, Desktop Driver **7 tests/1 suite**. These retries +do not erase aggregate failures or establish native qualification. + +At that checkpoint, the base pilot was committed locally and the unit extension +was uncommitted and unpushed. Hosted collection/publication had not been activated. The following +SPEC-only checkpoint is historical, not the current unit-extension result. + +### Prior SPEC snapshot validation, October 26, 2026 + +The final tooling suite passed **189 tests across 16 suites**, with no failures +or skips. Scripts build/lint, root TypeScript build/repository lint, all 24 +component contract checks, instruction/publishing checks, and formatting passed. +All 561 component tests across 66 suites passed in the aggregate. Actual +`yarn pack --dry-run --json` confirmed that SPEC is included and generated +assessment reports are excluded. The real local SPEC snapshot preserves every +original contract byte and the reviewed mapping digest. All 20 retained native +run files and eight older evidence/report hashes remain unchanged. + +The clean-cache aggregate and continuation were run; the final continuation +reported 83 successful, 88 skipped, and 4 failed Lage tasks. It is **not a +green aggregate**. The Windows package-check launcher still fails with +`spawnSync yarn ENOENT`, and two unchanged codemod newline fixtures still fail. +The other failures were in unchanged desktop packages under aggregate load; +all their affected suites passed isolated serial retry with original assertions +and deadlines: Storybook Desktop 11 tests/5 suites, Desktop Driver 7 tests/1 +suite. These retries do not erase the original aggregate failures or establish +new native qualification. Hosted activation remains pending. + +The generated block is a snapshot, not a live claim about a subsequently +changed contract. Do not edit it manually. Only its strictly validated, +bounded schema is normalized out of fingerprints. Unknown/duplicate fields, +invalid identities, malformed markers, and content injected into the block +are errors. All surrounding front matter and the full human-authored body +remain fingerprinted. Trusted follow-up verification compares **both** files +byte-for-byte with the original evidence rendering, so plausible but altered +metadata cannot authorize reuse. + +## What the pilot evaluates + +All six `CBX-*` requirements are loaded from Checkbox's `SPEC.md` and +`spec/source.json`. Metadata evidence filenames remain **declared references**, +not passing results. + +The single existing named native case, +`components-checkbox--default` / `toggles native checked state in both directions`, +provides partial mappings for CBX-002 (binary uncontrolled transitions) and CBX-004 +(native checkbox role). Neither complete requirement is covered by that case. +A multi-assertion case failure blocks both mappings; it does not identify exactly +which individual requirement is defective. + +The remaining gaps are explicit. The unit adapter above supplements the native +case; type/build/visual/manual adapters remain unimplemented. The pilot authors +no new Checkbox tests. A passing native +case therefore **cannot establish complete readiness**. + +The current default is the schema-v1 **desktop-test canonical report**, introduced +by PR #4389 (merged October 23, 2026), plus its declared Storybook smoke companion: + +```text +apps\storybook\artifacts\desktop\\run.json +apps\storybook\artifacts\desktop\\evidence\win32\static\run.json +``` + +Pass the canonical `run.json`; keep the complete run directory together. The +assessor loads its companion automatically. It checks supervisor failures, +unfinished work, attempt history, counts, timestamps, and callback identity/status +against the smoke results. Navigation and portable/static checks are not +Checkbox requirement coverage. Retried/flaky runs and cleanup/evidence failures +cannot qualify a passing case in this conservative pilot. + +Historical direct Desktop Driver `tests` reports and schema-v1 Storybook smoke +`wdio` reports remain supported for replay. Smoke `tests` entries are portable/legacy evidence and are not +mistaken for native WDIO results. Enclosing `"status": "passed"` never converts a +named skipped case into a pass. Invalid JSON/schema, duplicate identities, +contradictory statuses, and unreadable files are errors. + +## Provenance and policy + +Existing direct reports have no source commit, build ID, or timestamp. Smoke +reports add timestamps but still lack full provenance. The assessor records their +outcomes as observations, not current qualification. File modification times, +current checkout identity, and manifest hashes alone do not establish execution +identity. + +For local runner integration, an optional **execution-generated** sidecar can +bind provenance claims to the exact UTF-8 evidence bytes. For canonical input, +these are a JSON envelope containing `schemaVersion: 1`, `format: "desktop-test"`, +and the exact original canonical and smoke JSON strings as `desktopRun` and +`storyReport`. `reportSha256` hashes that envelope as returned by +`readNativeEvidence()`, not either file alone. Historical direct/smoke input still +hashes the original report bytes. Do not construct a sidecar +retroactively from the current checkout to make old results appear current. +This is correlation metadata, not a cryptographic execution attestation. + +```json +{ + "schemaVersion": 1, + "reportSha256": "<64 lowercase hex characters: SHA-256 of the accepted evidence bytes>", + "sourceCommit": "<40 lowercase hex characters: execution source commit>", + "buildId": "", + "contractDigest": "", + "platformManifestDigest": "<64 lowercase hex characters from the report>", + "endpoint": "win32", + "renderer": "paper", + "targetId": "", + "finishedAt": "2026-10-21T20:00:00.000Z", + "physicalClick": true +} +``` + +Use `--context "path\to\context.json" --build-id "" +--max-age-hours ""` when real runner context is +available. For the current harness, also supply +`--target-id "win32-<16-hex-instance-id>"` from the runner's pre-execution identity; +the context and report must both match it. New IDs derive from the real app root, +endpoint, and renderer, so they differ between checkouts. The old fixed +`agenticstorybook-win32` identity remains accepted for historical report formats, +not as a replacement for the new instance binding. CI captures the expected +Win32/Paper Storybook ID before smoke and rejects a different instance. +The hash protocol has a regression check against the actual upstream resolver. +No default age or stage threshold is invented. Context must match the +report, current source revision, contract inputs, expected build, native target, +and manifest; its time cannot be in the future or contradict a smoke report. +A dirty target working tree remains unverified because its commit does not +identify the changes. Supplied context is still only a claim, and full readiness +remains unsupported in this pilot. + +The reviewed mapping fingerprint covers, in order, `SPEC.md`, `spec/source.json`, +`spec/interaction.md`, `spec/accessibility.md`, `spec/tokens.yaml`, `spec/usage.md`, +and `checkbox.stories.tsx`, followed by the assertion-bearing helper +`packages/agentic/components/src/common/desktopFocus.wdio.ts`. +Hash input is each listed filename (component-relative for the first seven, +repository-relative for the shared helper), a NUL, its +UTF-8 content normalized to LF, and a NUL, after removing only the validated +generated SPEC assessment block. Any other changed input invalidates the +mapping. Review the contract and exact assertions before deliberately updating +the pinned digest in `src/tasks/controlAssessment.ts`; do not merely refresh it +to silence a stale result. + +## Reading the output + +`assessmentHealth` describes whether the assessment is complete, stale, or blocked. +`assessmentOutcome` distinguishes insufficient evidence, a qualifying native/unit +case failure, and runner failure. Native `check` and unit `unitChecks` remain +separate under each requirement. Each mapped check separates `recordedOutcome` +from `evidenceOutcome`; only matched current evidence can qualify. + +`declaredStatus`, `evidenceSupportedStatus`, and `approvedStatus` are null. +The contract's `implemented` / `reviewed` metadata is preserved separately and +is **not** converted to Backlog, Experimental, Beta, or Production. Maturity gates +and approvals require a separate agreed policy. + +Exit code **0 means the advisory reports were generated**, not that the control +is ready. With `--run-unit-tests`, a failed executed unit command also returns +nonzero after retaining available assessment evidence. Invalid inputs or failed +report writes return nonzero. This is not a +release gate. The CI integration checks report integrity and publication, not +Experimental/Beta/Production qualification. + +## Same-PR CI pilot (Option 2) + +The hand-authored integration is in `.github/workflows/pr.yml`, +`.github/workflows/pr-control-assessment.yml`, and +`scripts/src/tasks/controlAssessmentCi*.ts` plus `controlAssessmentGithub.ts`. +No additional native framework or input-capable runner is provisioned. + +1. Ordinary source PRs run the existing jobs. The JS job captures source before + its unchanged `yarn lage buildci`; the existing component Jest invocation + exports a full-package JSON report and execution context exactly once. + The independent Win32 Storybook job uses Node 24, retains hosted + `keyboard,physicalClick,wheel` disabling and captures source before native + execution. Its capture inventories existing + `artifacts/desktop` run directories; collection accepts exactly one new run + and checks its ID and execution time. It never deletes/reuses an old run or + selects an arbitrary newest report. Missing/unfinished output is explicitly + unavailable, and multiple fresh runs are an ambiguity error. Unit/native jobs + do not depend on one another. No `continue-on-error`, test rerun, or + permission/input-capability bypass is used. +2. The **Assess Checkbox evidence** job waits for JS and Win32 results and runs + even after an ordinary source-job failure. It downloads separate same-attempt + artifacts, validates matching repository/PR/head/base/tested SHA/tree/run/attempt, + report hashes and each source's own execution interval, and combines them. + Unit execution may precede native capture; cross-job evidence is not falsely + required to fall inside the native execution interval. + It never selects JSON by modification time or executes tests. Missing source + artifacts are reported as unavailable. Failed original source checks remain failed. + Collection writes `bundle.json`, `assessment.json`, `ASSESSMENT.md`, and a proposed `SPEC.md` in + ignored `apps/storybook/artifacts/win32/assessment-ci`. The readable report + also appears in the job summary. Artifact name: + `checkbox-win32-assessment-`, retained for 14 days. + New assessment JSON and CI bundles use schema v2. The trusted reader explicitly + accepts native-only v1 bundles, but rejects unit fields smuggled into v1. + Trusted reconstruction reads reviewed unit inputs from the tested Git tree, + not PR-supplied assessment output, and executes no PR unit code with write credentials. +3. An opt-in, default-branch `workflow_run` publisher validates that data and + recomputes the advisory assessment using trusted tooling. It does not check + out, install dependencies from, or execute PR code with a write credential. +4. For an eligible same-repository PR targeting the default branch, a narrowly + scoped GitHub App atomically commits **only** + `packages/agentic/components/src/components/checkbox/assessment/ASSESSMENT.md` + and the generated assessment fields in the adjacent `SPEC.md` + to the existing PR source branch. It preserves every other SPEC byte and + never writes to main or merges a PR. + The API's `expectedHeadOid` rejects a branch race; newer source results must + be used instead of force-pushing or appending stale evidence. +5. The report commit triggers a follow-up PR run. Trusted input validation + downloads the original attempt's bundle, verifies the exact rendered report and SPEC, + checks all tracked inputs, and retrieves the original source-validation + result from GitHub. Only then may expensive jobs be skipped. A failed or + unrun source-validation job cannot become successful through a report commit. +6. Report-only verification runs do not create native assessment artifacts, so + the publisher makes no further report commit. The bundle-size commenter also + handles the absence of its artifact on those runs. + +The existing required check is still named **PR**. Its source-check aggregation +is now separately visible as **PR source validation**. With publishing enabled +for an eligible PR, the initial **PR** check deliberately fails with a +"waiting for the trusted publisher" message after source validation. This +prevents merging before publication. The report-only follow-up replaces it with +a successful **PR** check only if both the source result and report verification +pass. This temporary publication failure is not a control-readiness rating. + +Without opt-in, CI still collects reports and the required check preserves normal +source-validation behavior. Fork PRs and non-default-target PRs retain CI +reports, but the pilot does not write their branches or require automatic +publication. No fork token, maintainer-edit assumption, approval bypass, or +branch-protection exception is used. + +### Tested source, report commits, and freshness + +`sourceCommit` remains the actual tested PR **merge revision**, not the later +report commit. The summary separately records PR head, tested base, source +workflow run and attempt, generation time, and the input digest. + +The CI digest hashes Git tree entries, including blob IDs and file modes, for +**every tracked file except the exact generated summary path**, with only the +validated SPEC assessment block removed before recomputing that SPEC's blob ID. +The original mode and path remain inputs. A SPEC without the generated block +retains its previous digest unchanged. Tests, +specifications, code, all dependency files, workflows, formatting policy, and +other configuration are inputs. This deliberately conservative pilot does not +attempt dependency-graph impact analysis. Another control change can therefore +invalidate reuse. + +Reuse requires a single two-file report/SPEC commit directly after the tested PR head, +the same complete input digest in the new merge tree, the matching original +workflow attempt and PR, and byte-for-byte equality of both files with trusted renderings. +It does not use the cumulative PR diff or merely inspect the latest commit. +The explicit CI reuse limit is **168 hours**; this is an operational evidence-age +limit, not a maturity-stage policy. Older references cause a full source run. +Deleted or expired original artifacts cannot authorize a report-only fast path. +The original local CLI still requires an explicit `--max-age-hours` value. + +Legacy summary-only publications fall back to full source CI rather than +receiving an unchecked exemption. Malformed SPEC assessment metadata fails +explicitly; valid but altered generated fields fail exact reconstruction. + +The summary says **AUTO-GENERATED. DO NOT EDIT**. Its sole formatter exclusion +prevents a source formatter from changing evidence-bound output. Do not exclude +the entire control, its specifications, or its tests. The SPEC renderer uses +the repository's single-quoted YAML style and does not require a SPEC-wide +formatting exemption. + +### Maintainer activation and first hosted trial + +Deployment is not implied by local validation: + +1. Review and merge the collector, trusted validator, publisher workflow, and + tooling to main **with publication disabled**. `workflow_run` needs its + workflow on the default branch. The bootstrap PR runs full CI if the trusted + validator is not yet on its base branch. +2. Install a dedicated GitHub App on **only this repository**, granting + `Contents: write`. It needs no Actions-write, Workflows-write, + Administration, review-approval, or protection-bypass permission. +3. Configure repository variable `CONTROL_ASSESSMENT_APP_CLIENT_ID` and secret + `CONTROL_ASSESSMENT_APP_PRIVATE_KEY`. Keep the private key out of source and + logs. The action requests only `contents:write`, scopes the token to the + current repository, and revokes it when its job finishes. +4. Have a maintainer verify organizational Actions execution policies, the App's + permitted branch writes, and fork-run approval behavior. No policy is + weakened by this pilot. +5. Set repository variable `CONTROL_ASSESSMENT_PUBLISH=true` only after setup. + Open a same-repository pilot PR, then observe source CI, the single report + commit, trusted report-only validation, and the final required **PR** check. + Verify one failed source check stays failed, a newer source push rejects stale + publication, and concurrent Checkbox PRs regenerate reports after conflicts. + +If the publisher is misconfigured, the eligible PR remains blocked rather than +merging without its report. Disabling the opt-in restores the ordinary artifact +flow; changing a variable may require rerunning the PR workflow. This does not +change the repository's branch rules. + +### Merge conflicts and concurrent changes + +When two Checkbox PRs conflict, resolve actual source conflicts first. Keep +main's generated report and assessment block temporarily rather than merging assessment prose, +update the PR branch, and rerun source CI. The publisher regenerates the summary and SPEC snapshot +for the updated inputs. GitHub's ordinary `pull_request` workflow will not start +while conflicts remain. + +After a newer source commit, closed PR, unavailable merge revision, different +merged inputs, or unsupported fork, publication explicitly reports why no commit +was made. Permission and API failures are errors, not successful empty reports. +There is no force push, automatic source-conflict resolver, auto-approval, or +automatic merge. + +### FURN rules verified October 21, 2026 (Pacific) + +Read-only API inspection found active main ruleset **12778298**: + +- One approving review; stale-review dismissal **off**, latest-push approval + requirement **off**, code-owner review requirement **off**. +- Review-thread resolution required; only squash merges allowed. +- Required status context **PR**, GitHub Actions integration **15368**. +- Strict up-to-date checking **off**. The report identifies the tested + integration tree; it does not promise compatibility with every later main + revision. +- Extra approval for unattributed Copilot PRs is enabled. This is not a blanket + exemption for generated reports; applicable repository rules still govern. + +The inspecting account had read-only repository access. Actions settings +endpoints returned **403**, so organization execution policy, default workflow +permissions, and fork-contributor approval configuration were **not verified**. +The legacy branch-protection endpoint returned **404**; that does not negate +the active ruleset returned by the effective branch-rules API. + +The published rules permit the Option 2 design; they do not establish that an +App is installed, that secrets exist, or that hosted publication has run. + +## Historical October 26, 2026 migration checkpoint + +Local main was fast-forwarded to +`01ab9fee6dbf40de9cec40926f8aa8d68193d446`, including Jason's PR #4389. +The reviewed Checkbox mapping digest changed only after checking the upstream +focus-document link relocations and unchanged mapped assertions. No new control +test, maturity policy, publisher permission, or native input capability was added. +The legacy desktop Appium and mobile lanes were not removed by #4389. + +A replay of PR #4389's actual Win32 artifact (Actions run `39868561037`, attempt +1, desktop run `bfcbec99-2712-4449-9722-81dac06a02cd`) successfully parsed the +canonical result and companion: supervisor passed; Checkbox interaction skipped +because physical pointer input was disabled. This is historical artifact replay, +not a new native test or current qualification. + +Scripts build/lint and all **174 tests passed** (15 suites, no skips). Tests +include canonical reporting, incomplete/ambiguous/missing evidence, retries, +target-ID conformance against the real upstream resolver, target mismatch, +actual CLI entrypoints, publication, and report-only verification. +Actionlint 1.7.12 passed for all three pilot workflows. Formatting checks and +`git diff --check` passed. These are local tooling checks, not native qualification. + +**Immutable dependency restoration now passed.** The corporate registry supplied +`ip-address@10.7.4` and `zod@4.6.7`. The final missing package, +`storybook@10.6.3`, was recovered from public CDN files over verified HTTPS. +Its reconstructed Yarn cache archive matched the **entire original lockfile +SHA-512 checksum**, before any recovered package code was executed. Offline +`yarn install --immutable`, with checksum behavior `throw`, then succeeded with +peer-dependency/linker warnings. The lockfile and package versions are unchanged; +no certificate validation, permission, or integrity check was bypassed. +Registry/auth overrides were process-local. This repairs the local cache, not +the normal registry TLS route; a future cache miss may still need that route fixed. + +The Storybook update from 10.6.0 to 10.6.3 came from Dependabot PR #4387 on +October 23, not Jason's #4389. The app already declared `storybook: ^10.4.0`; +the lockfile selects the exact version. This is an existing npm development +dependency, not another desktop application or a binary to commit into Git. + +The required clean-cache aggregate now reaches actual validation. Root +TypeScript build, repository lint, agent-instruction checks, and publishing +checks passed. It is **not entirely green**: the unchanged upstream +`checkPackageContentsTask.ts` invokes `execFileSync('yarn', ...)`, which fails +with `ENOENT` on this Windows setup for all eight guided packages. Directly +spawning the available `.cmd` shim also reproduces `EINVAL`. That task came +from upstream PR #4393 and was not changed by this pilot. This is not the old +stale-pnpm-map failure or proof that the entire pre-sync baseline persists. + +The continuation completed with **82 successful, 88 skipped, and 5 failed Lage +tasks**; those are task statuses, not native test-case counts. The scripts +regression suite passed again after restoration, as did the neutral +`desktop-test` suite. Besides the package-content check, both unchanged codemod +fixture tests failed with newline-only output differences. Three other +workspace tasks failed on timeouts during the parallel aggregate. All their +affected suites passed on isolated serial retry with unchanged assertions and +time limits: Storybook Desktop 11 tests, Desktop Driver 7, and agentic components +73 (**91 tests across 11 suites**). This is evidence of load-sensitive failures, +not a clean aggregate pass or native qualification. The upstream packages and +their test sources are unchanged by this pilot. + +Post-restore CLI replay of the real canonical artifact also passed, preserving +insufficient evidence and null maturity statuses. +**Fresh local native verification now passed.** After generating the current +manifest, bundling Win32, explicitly building the helper, and passing its +prebuilt-only `doctor`, the current harness completed desktop run +`740c6879-528f-46f2-818f-3eb36b5fbd34` on October 26, 20:19:33-20:21:33 UTC. +It recorded 166 story-navigation passes and the one Checkbox callback pass: +unchecked -> checked -> unchecked through physical clicks. All 167 selected +checks passed, with no skips, failures, retries, flaky results, incomplete work, +or harness failures. The story filter selected the Checkbox callback; it did +not reduce the harness's full story traversal. + +Run app-owned commands from `apps/storybook`, or select its workspace explicitly +from the repository root, for example: + +```powershell +yarn workspace @fluentui-react-native/agentic-components-storybook storybook bundle --win32 +yarn workspace @fluentui-react-native/agentic-components-storybook storybook build-driver --win32 +yarn workspace @fluentui-react-native/agentic-components-storybook desktop-driver doctor --platform win32 +$env:STORYBOOK_SMOKE_STORY = 'components-checkbox--default' +$env:FURN_DESKTOP_DRIVER_BUILD_POLICY = 'never' +yarn workspace @fluentui-react-native/agentic-components-storybook storybook smoke --win32 --mode stories-and-tests +``` + +The environment overrides above are process-local; clear the story filter +before selecting other controls. Win32 uses the prebuilt REX Paper host: the +bundle and native helper are built, not a new native Win32 app host. + +The assessor consumed that actual canonical run and its companion. It preserved +one passed case, zero skipped cases, and the current reviewed mapping, while +correctly reporting `insufficient-evidence` and null maturity statuses. Local +execution has no bound CI provenance and the checkout is dirty; no context or +attestation was fabricated. The native case still covers only parts of CBX-002 +and CBX-004. Navigation passes do not become requirement coverage. +The full run's 20 files were retained byte-for-byte with a SHA-256 inventory in +the user's FURN evidence folder, with a separate assessment and local provenance +note; historical evidence and CI skips were preserved. + +No hosted collector, GitHub App publication, or required-check transition was +run. A fully green repository aggregate is not claimed. + +At this checkpoint, the original pre-sync pilot was retained in a named Git stash +and migration changes remained local, uncommitted, and unpushed. + +## Historical pre-sync validation (October 21-22, 2026) + +The owning scripts workspace build, lint, and all **112 tests passed**. +Tests cover actual CLI entrypoints and disposable Git histories, including +report-only reuse, changed inputs, failed source checks, stale publication, +branch races, fork rejection, and concurrent generated-report conflicts followed +by regeneration. Write API calls use test doubles; these are not hosted trials. +Actionlint 1.7.12 passed for all three changed workflows. + +The required clean-cache `yarn lage buildci` run failed on the same 11 Knip +unused dependencies previously reproduced on the unchanged baseline. A +continuation run completed with 84 successful tasks, 85 cached/skipped tasks, +no pending or aborted tasks, and two failed tasks: repository lint and the +legacy codemod test task (two failing tests). The root TypeScript build, +agent-instruction checks, repository formatting, and scripts tests passed. +These baseline failures have not been hidden or changed. + +No new native execution was performed for this CI implementation. Hosted +collection, GitHub App publication, and required-check transitions were +**not run** at that checkpoint, pending maintainer deployment and configuration. +None of these historical results constitute hosted activation. + +Run the owning workspace tests with: + +```powershell +yarn workspace @fluentui-react-native/scripts test +``` diff --git a/scripts/package.json b/scripts/package.json index 2ed46c544d..4f5bc2b86d 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -30,7 +30,7 @@ "bundlesize": "bundlesize --debug", "format": "node ./src/cli.ts format", "lint": "node ./src/cli.ts lint", - "test": "node --test src/tasks/agentInstructions.test.ts src/tasks/clean.test.mjs src/utils/jestSelection.test.ts src/utils/runScript.test.ts" + "test": "node --test \"$@\" src/tasks/agentInstructions.test.ts src/tasks/clean.test.mjs src/utils/jestSelection.test.ts src/utils/runScript.test.ts src/tasks/controlAssessment.test.ts src/tasks/controlAssessmentCi.test.ts src/tasks/controlAssessmentUnit.test.ts" }, "dependencies": { "@babel/core": "catalog:", diff --git a/scripts/src/cli.ts b/scripts/src/cli.ts index 2d0b17c244..e78fcf5f38 100644 --- a/scripts/src/cli.ts +++ b/scripts/src/cli.ts @@ -8,6 +8,7 @@ import { JestCommand } from './tasks/jest.ts'; import { CheckPublishingCommand } from './tasks/checkPublishingTask.ts'; import { CheckAgentInstructionsCommand } from './tasks/checkAgentInstructionsTask.ts'; import { CheckPackageContentsCommand } from './tasks/checkPackageContentsTask.ts'; +import { AssessControlCommand } from './tasks/assessControlTask.ts'; const cli = new Cli({ binaryLabel: 'fluentui-scripts CLI', @@ -22,6 +23,7 @@ cli.register(JestCommand); cli.register(CheckPublishingCommand); cli.register(CheckAgentInstructionsCommand); cli.register(CheckPackageContentsCommand); +cli.register(AssessControlCommand); cli.register(Builtins.DefinitionsCommand); cli.register(Builtins.HelpCommand); diff --git a/scripts/src/tasks/assessControlTask.ts b/scripts/src/tasks/assessControlTask.ts new file mode 100644 index 0000000000..78088ed263 --- /dev/null +++ b/scripts/src/tasks/assessControlTask.ts @@ -0,0 +1,184 @@ +import { execFileSync } from 'node:child_process'; +import { existsSync, lstatSync, mkdirSync, readFileSync, realpathSync, statSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { Command, Option } from 'clipanion'; +import { findGitRoot } from 'workspace-tools'; + +import { + assessCheckbox, + formatAssessmentMarkdown, + object, + parseEvidenceContext, + parseNativeReport, + readCheckboxContract, + readNativeEvidence, + reportDigest, + text, +} from './controlAssessment.ts'; +import { assessmentPackagePath, assessmentSpecPath, specAssessmentFor, updateSpecAssessment } from './controlAssessmentSpec.ts'; +import { parseUnitContext, readCheckboxUnitMappingFrom } from './controlAssessmentUnit.ts'; +import { runCheckboxUnitTests } from './controlAssessmentUnitExecution.ts'; + +function sameFile(first: string, second: string): boolean { + if (path.relative(first, second) === '') return true; + if (!existsSync(first) || !existsSync(second)) return false; + const firstInfo = statSync(first, { bigint: true }); + const secondInfo = statSync(second, { bigint: true }); + return firstInfo.dev === secondInfo.dev && firstInfo.ino === secondInfo.ino; +} + +export class AssessControlCommand extends Command { + static override paths = [['assess-control']]; + + static override usage = Command.Usage({ + description: 'Produce an advisory Checkbox/Win32 assessment from native and JavaScript unit evidence', + details: + 'Reads existing native/unit reports, or explicitly runs Checkbox runtime tests with --run-unit-tests. Writes assessment.json and assessment.md, preserving skips, provenance, and evidence gaps. --update-spec records a generated SPEC snapshot and linked report. No maturity stage is assigned. See scripts/CONTROL-ASSESSMENT.md.', + examples: [ + [ + 'Assess an existing native report', + '$0 assess-control --component checkbox --evidence apps/storybook/artifacts/desktop//run.json', + ], + ], + }); + + component = Option.String('--component', { required: true, description: 'Pilot component: checkbox' }); + evidence = Option.String('--evidence', { + description: 'Canonical desktop-test run.json with its Storybook companion, or a historical WDIO/smoke report', + }); + contextFile = Option.String('--context', { + description: 'Optional execution-generated provenance sidecar, never synthesized from the current checkout', + }); + unitEvidence = Option.String('--unit-evidence', { description: 'Jest JSON selecting only checkbox.test.tsx, not type tests' }); + unitContextFile = Option.String('--unit-context', { description: 'Execution-generated unit provenance sidecar' }); + unitBuildId = Option.String('--unit-build-id', { description: 'Expected unit build ID for replayed evidence' }); + runUnitTests = Option.Boolean('--run-unit-tests', false, { + description: 'Run the declared Checkbox runtime test script and capture fresh unit evidence', + }); + buildId = Option.String('--build-id', { description: 'Expected test build ID; required to accept bound evidence' }); + targetId = Option.String('--target-id', { description: 'Expected native target instance; required for bound desktop-test evidence' }); + maxAgeHours = Option.String('--max-age-hours', { description: 'Explicit positive evidence-age limit; no default readiness policy' }); + output = Option.String('--output', { description: 'Report directory; defaults to apps/storybook/artifacts/win32/assessment' }); + updateSpec = Option.Boolean('--update-spec', false, { + description: 'Also write the generated SPEC assessment snapshot and local report', + }); + + async execute() { + if (this.component !== 'checkbox') throw new Error('This pilot supports only --component checkbox on Win32/Paper.'); + if (this.runUnitTests && (this.unitEvidence !== undefined || this.unitContextFile !== undefined || this.unitBuildId !== undefined)) { + throw new Error('--run-unit-tests cannot be combined with replayed unit evidence, context, or build ID.'); + } + if (this.unitContextFile !== undefined && this.unitEvidence === undefined) throw new Error('--unit-context requires --unit-evidence.'); + if (this.unitBuildId !== undefined && this.unitEvidence === undefined) throw new Error('--unit-build-id requires --unit-evidence.'); + const root = findGitRoot(process.cwd()); + const unitRun = this.runUnitTests + ? runCheckboxUnitTests(root, path.join(root, 'apps', 'storybook', 'artifacts', 'win32', 'assessment-units')) + : undefined; + const sourceCommit = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).trim(); + const workingTreeDirty = + execFileSync('git', ['status', '--porcelain', '--untracked-files=all'], { cwd: root, encoding: 'utf8' }).length > 0; + const input = this.evidence === undefined ? undefined : path.resolve(this.evidence); + const contextFile = this.contextFile === undefined ? undefined : path.resolve(this.contextFile); + const unitInput = unitRun?.report + ? path.join(unitRun.directory, 'jest.json') + : this.unitEvidence === undefined + ? undefined + : path.resolve(this.unitEvidence); + const unitContextFile = unitRun?.context + ? path.join(unitRun.directory, 'context.json') + : this.unitContextFile === undefined + ? undefined + : path.resolve(this.unitContextFile); + const content = input === undefined ? undefined : readNativeEvidence(input); + const specPath = path.join(root, assessmentSpecPath); + const originalSpec = readFileSync(specPath, 'utf8'); + const report = assessCheckbox({ + contract: readCheckboxContract(root), + sourceCommit, + workingTreeDirty, + now: new Date(), + report: content === undefined ? undefined : parseNativeReport(JSON.parse(content)), + reportSha256: content === undefined ? undefined : reportDigest(content), + context: contextFile === undefined ? undefined : parseEvidenceContext(JSON.parse(readFileSync(contextFile, 'utf8'))), + expectedBuildId: this.buildId, + expectedTargetId: this.targetId, + maxAgeHours: this.maxAgeHours === undefined ? undefined : Number(this.maxAgeHours), + unit: + unitRun || unitInput !== undefined + ? { + mapping: readCheckboxUnitMappingFrom((file) => readFileSync(path.join(root, file), 'utf8')), + report: unitRun?.report ?? (unitInput === undefined ? undefined : readFileSync(unitInput, 'utf8')), + context: + unitRun?.context ?? + (unitContextFile === undefined ? undefined : parseUnitContext(JSON.parse(readFileSync(unitContextFile, 'utf8')))), + executionFailure: unitRun?.executionFailure ?? undefined, + expectedBuildId: unitRun?.context?.buildId ?? this.unitBuildId, + } + : undefined, + }); + const directory = + this.output === undefined ? path.join(root, 'apps', 'storybook', 'artifacts', 'win32', 'assessment') : path.resolve(this.output); + const jsonPath = path.join(directory, 'assessment.json'); + const markdownPath = path.join(directory, 'assessment.md'); + const localSummary = path.join(path.dirname(specPath), 'assessment', 'ASSESSMENT.md'); + const outputs = this.updateSpec ? [jsonPath, markdownPath, specPath, localSummary] : [jsonPath, markdownPath]; + for (const output of outputs) { + if ([input, contextFile, unitInput, unitContextFile].some((file) => file !== undefined && sameFile(file, output))) { + throw new Error('Assessment output must not overwrite an input report or context.'); + } + } + if (sameFile(jsonPath, markdownPath)) throw new Error('Assessment JSON and Markdown must be separate files.'); + for (const output of [jsonPath, markdownPath, ...(this.updateSpec ? [localSummary] : [])]) { + if (sameFile(output, specPath)) throw new Error('Assessment output must not overwrite the specification.'); + } + if (this.updateSpec && [jsonPath, markdownPath].some((output) => sameFile(output, localSummary))) { + throw new Error('Local summary must be separate from the artifact reports.'); + } + let updatedSpec: string | undefined; + if (this.updateSpec) { + for (const file of [specPath, localSummary]) { + if (existsSync(file) && (!lstatSync(file).isFile() || lstatSync(file).nlink !== 1)) { + throw new Error('SPEC snapshot destinations must be regular, unlinked files.'); + } + } + const manifest = object(JSON.parse(readFileSync(path.join(root, assessmentPackagePath), 'utf8')), 'Component package manifest'); + updatedSpec = updateSpecAssessment(originalSpec, specAssessmentFor(report, text(manifest.version, 'Component package version'))); + mkdirSync(path.dirname(localSummary), { recursive: true }); + if (path.relative(realpathSync(path.dirname(specPath)), realpathSync(path.dirname(localSummary))) !== 'assessment') { + throw new Error('Local assessment directory must not redirect outside its component.'); + } + } + mkdirSync(directory, { recursive: true }); + writeFileSync( + jsonPath, + `${JSON.stringify( + { + ...report, + inputs: { + report: input ?? null, + context: contextFile ?? null, + unitReport: unitInput ?? null, + unitContext: unitContextFile ?? null, + unitExecuted: this.runUnitTests, + }, + }, + null, + 2, + )}\n`, + ); + writeFileSync(markdownPath, formatAssessmentMarkdown(report)); + if (updatedSpec !== undefined) { + if (readFileSync(specPath, 'utf8') !== originalSpec) + throw new Error('SPEC changed during assessment; refusing concurrent overwrite.'); + writeFileSync( + localSummary, + `\n\n${formatAssessmentMarkdown(report)}`, + ); + writeFileSync(specPath, updatedSpec); + } + this.context.stdout.write( + `Assessment report generated: ${report.assessmentOutcome}; readiness not established (advisory).\n${jsonPath}\n${markdownPath}\n`, + ); + return unitRun?.exitCode ?? 0; + } +} diff --git a/scripts/src/tasks/controlAssessment.test.ts b/scripts/src/tasks/controlAssessment.test.ts new file mode 100644 index 0000000000..6759c0b7ab --- /dev/null +++ b/scripts/src/tasks/controlAssessment.test.ts @@ -0,0 +1,808 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { copyFileSync, linkSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, it } from 'node:test'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +import { + assessCheckbox, + checkboxNativeAssertionPath, + formatAssessmentMarkdown, + parseEvidenceContext, + parseNativeReport, + readCheckboxContract, + readCheckboxContractFrom, + readNativeEvidence, + reportDigest, + win32StorybookTargetId, +} from './controlAssessment.ts'; +import type { AssessmentOptions, EvidenceContext } from './controlAssessment.ts'; +import { checkboxJestFixture, desktopEvidenceFixture } from './controlAssessmentFixtures.ts'; +import { + assessmentPackagePath, + assessmentSpecPath, + specAssessmentFor, + splitSpecAssessment, + updateSpecAssessment, +} from './controlAssessmentSpec.ts'; + +const root = fileURLToPath(new URL('../../../', import.meta.url)); +const contract = readCheckboxContract(root); +const sourceCommit = 'a'.repeat(40); +const platformManifestDigest = 'b'.repeat(64); +const now = new Date('2026-10-21T21:00:00.000Z'); +const nativeCase = { + storyId: 'components-checkbox--default', + testName: 'toggles native checked state in both directions', + status: 'passed', + durationMs: 3687, +}; +const direct = { endpoint: 'win32', platformManifestDigest, status: 'passed', tests: [nativeCase] }; +const aggregate = { + schemaVersion: 1, + endpoint: 'win32', + platformName: 'windows', + targetId: 'agenticstorybook-win32', + runId: 'synthetic-run', + startedAt: '2026-10-21T19:59:00.000Z', + finishedAt: '2026-10-21T20:00:00.000Z', + manifest: { platform: platformManifestDigest, portable: 'c'.repeat(64) }, + status: 'passed', + tests: [{ storyId: 'ignored-portable-test', status: 'passed' }], + wdio: [{ ...nativeCase, status: 'skipped', skipReason: 'This test requires physical pointer input.' }], +}; +const raw = JSON.stringify(direct); +const context: EvidenceContext = { + schemaVersion: 1, + reportSha256: reportDigest(raw), + sourceCommit, + buildId: 'synthetic-build', + contractDigest: contract.digest, + platformManifestDigest, + endpoint: 'win32', + renderer: 'paper', + targetId: 'agenticstorybook-win32', + finishedAt: '2026-10-21T20:00:00.000Z', + physicalClick: true, +}; + +function options(overrides: Partial = {}): AssessmentOptions { + return { + contract, + sourceCommit, + workingTreeDirty: false, + now, + report: parseNativeReport(direct), + reportSha256: reportDigest(raw), + context, + expectedBuildId: context.buildId, + maxAgeHours: 24, + ...overrides, + }; +} + +const fixtures: string[] = []; +function fixture(): string { + const directory = mkdtempSync(path.join(tmpdir(), 'furn-assessment-')); + fixtures.push(directory); + return directory; +} +afterEach(() => { + for (const directory of fixtures.splice(0)) rmSync(directory, { recursive: true, force: true }); +}); + +describe('Checkbox advisory assessment', () => { + it('loads six reviewed requirements without converting implemented/reviewed into a maturity stage', () => { + assert.equal(contract.mappingCurrent, true); + assert.deepEqual( + contract.requirements.map(({ id }) => id), + ['CBX-001', 'CBX-002', 'CBX-003', 'CBX-004', 'CBX-005', 'CBX-006'], + ); + const report = assessCheckbox(options()); + assert.equal(report.contract.lifecycle, 'implemented'); + assert.equal(report.contract.conformance, 'reviewed'); + assert.equal(report.declaredStatus, null); + assert.equal(report.approvedStatus, null); + assert.equal(report.evidenceSupportedStatus, null); + assert.equal(report.policyStatus, 'pending'); + assert.equal(report.assessmentOutcome, 'insufficient-evidence'); + assert.deepEqual( + report.requirements.map(({ outcome }) => outcome), + ['insufficient-evidence', 'partial', 'insufficient-evidence', 'partial', 'insufficient-evidence', 'insufficient-evidence'], + ); + assert.equal(report.evidence.caseCounts.passed, 1); + assert.equal(report.requirements.filter(({ check }) => check).length, 2); + assert.ok(report.requirements.every(({ remainingCoverage }) => remainingCoverage.length > 0)); + }); + + describe('desktop-test canonical evidence', () => { + const smoke = JSON.stringify(aggregate); + it('computes the same Win32/Paper target identity as the actual upstream configuration resolver', () => { + const directory = fixture(); + const appRoot = path.join(directory, 'apps', 'storybook'); + mkdirSync(appRoot, { recursive: true }); + const configPath = path.join(appRoot, 'desktop.config.mts'); + writeFileSync(configPath, 'export default { schemaVersion: 1, targets: { win32: { host: { kind: "rex", renderer: "paper" } } } };'); + const resolver = pathToFileURL(path.join(root, 'packages', 'agentic', 'desktop-test', 'src', 'config.ts')).href; + const result = spawnSync( + process.execPath, + [ + '--input-type=module', + '-e', + 'const { loadDesktopConfig } = await import(process.argv[1]); const { targets } = await loadDesktopConfig(process.argv[2]); console.log(targets.win32.id);', + resolver, + configPath, + ], + { encoding: 'utf8' }, + ); + assert.equal(result.status, 0, result.stderr); + assert.equal(win32StorybookTargetId(directory), result.stdout.trim()); + }); + + it('binds generated target IDs without treating an arbitrary Win32 instance as the expected host', () => { + const targetId = `win32-${'1'.repeat(16)}`; + const { envelope } = desktopEvidenceFixture(JSON.stringify({ ...aggregate, targetId })); + const report = parseNativeReport(JSON.parse(envelope)); + const bound = parseEvidenceContext({ ...context, reportSha256: reportDigest(envelope), targetId }); + const input = options({ report, reportSha256: reportDigest(envelope), context: bound, expectedTargetId: targetId }); + assert.equal(assessCheckbox(input).evidence.status, 'matched'); + assert.equal(assessCheckbox({ ...input, expectedTargetId: undefined }).evidence.status, 'unverified'); + assert.equal(assessCheckbox({ ...input, expectedTargetId: `win32-${'2'.repeat(16)}` }).evidence.status, 'mismatched'); + assert.equal(assessCheckbox({ ...input, context: { ...bound, targetId: 'agenticstorybook-win32' } }).evidence.status, 'mismatched'); + assert.equal(assessCheckbox({ ...input, context: undefined, expectedTargetId: undefined }).evidence.status, 'unverified'); + }); + + it('combines canonical supervisor and smoke evidence without counting navigation as requirement coverage', () => { + const result = parseNativeReport(JSON.parse(desktopEvidenceFixture(smoke).envelope)); + assert.equal(result.format, 'desktop-test'); + assert.equal(result.harness?.status, 'passed'); + assert.equal(result.cases.length, 1); + assert.equal(result.cases[0].status, 'skipped'); + assert.equal(result.harness?.retries, 0); + }); + + it('loads a canonical run with its declared companion, preserving both sets of bytes', () => { + const directory = fixture(); + const { canonical } = desktopEvidenceFixture(smoke); + const evidence = path.join(directory, 'evidence', 'win32', 'static'); + mkdirSync(evidence, { recursive: true }); + writeFileSync(path.join(directory, 'run.json'), canonical); + writeFileSync(path.join(evidence, 'run.json'), smoke); + const content = readNativeEvidence(path.join(directory, 'run.json')); + assert.equal(JSON.parse(content).desktopRun, canonical); + assert.equal(JSON.parse(content).storyReport, smoke); + assert.equal(parseNativeReport(JSON.parse(content)).format, 'desktop-test'); + }); + + it('rejects a missing companion instead of silently accepting supervisor success', () => { + const directory = fixture(); + writeFileSync(path.join(directory, 'run.json'), desktopEvidenceFixture(smoke).canonical); + assert.throws(() => readNativeEvidence(path.join(directory, 'run.json')), /ENOENT/); + }); + + it('retains cleanup failure even when the Storybook evidence passed', () => { + const { envelope } = desktopEvidenceFixture(smoke, { + status: 'failed', + failures: [{ failureId: 'cleanup', kind: 'cleanup', error: { name: 'Error', message: 'session cleanup failed' } }], + }); + const result = parseNativeReport(JSON.parse(envelope)); + assert.equal(result.status, 'failed'); + assert.equal(result.harness?.failureCount, 1); + assert.match(result.error ?? '', /supervisor status: failed/); + }); + + it('preserves unfinished supervisor execution rather than qualifying its recorded cases', () => { + const result = parseNativeReport(JSON.parse(desktopEvidenceFixture(smoke, { status: 'incomplete' }).envelope)); + assert.equal(result.status, 'failed'); + assert.equal(result.harness?.status, 'incomplete'); + }); + + it('does not accept retry/flaky passes as a clean qualifying execution', () => { + const passedSmoke = JSON.stringify({ ...aggregate, wdio: [nativeCase] }); + const { run } = desktopEvidenceFixture(passedSmoke); + const retried = run.tests[1]; + retried.attempts.unshift({ ...retried.attempts[0] }); + retried.attempts[1].attempt = 1; + run.counts.retries = 1; + const result = parseNativeReport({ + schemaVersion: 1, + format: 'desktop-test', + desktopRun: JSON.stringify(run), + storyReport: passedSmoke, + }); + assert.equal(result.harness?.retries, 1); + assert.equal(result.status, 'failed'); + assert.match(result.error ?? '', /Retried or flaky/); + }); + + for (const [name, mutate] of [ + [ + 'counts', + (run: Record) => { + run.counts = {}; + }, + ], + [ + 'duplicated artifact', + (run: Record) => { + run.artifacts = [{ path: 'evidence/win32/static/run.json' }, { path: 'evidence\\win32\\static\\run.json' }]; + }, + ], + [ + 'wrong run kind', + (run: Record) => { + run.kind = 'lifecycle'; + }, + ], + [ + 'outside timestamp', + (run: Record) => { + run.startedAt = '2026-10-22T00:00:00.000Z'; + }, + ], + [ + 'invalid schema', + (run: Record) => { + run.schemaVersion = 2; + }, + ], + ] as const) { + it(`rejects inconsistent ${name}`, () => { + const run: Record = JSON.parse(desktopEvidenceFixture(smoke).canonical); + mutate(run); + assert.throws(() => + parseNativeReport({ schemaVersion: 1, format: 'desktop-test', desktopRun: JSON.stringify(run), storyReport: smoke }), + ); + }); + } + + it('rejects a callback status that contradicts the canonical result', () => { + const { canonical } = desktopEvidenceFixture(smoke); + assert.throws( + () => + parseNativeReport({ + schemaVersion: 1, + format: 'desktop-test', + desktopRun: canonical, + storyReport: JSON.stringify({ ...aggregate, wdio: [nativeCase] }), + }), + /does not match/, + ); + }); + + it('rejects a nested evidence envelope instead of recursively trusting it as a companion', () => { + const { canonical, envelope } = desktopEvidenceFixture(smoke); + assert.throws( + () => + parseNativeReport({ + schemaVersion: 1, + format: 'desktop-test', + desktopRun: canonical, + storyReport: envelope, + }), + /companion/, + ); + }); + + it('rejects successful supervisor status when selected work is unfinished', () => { + const { envelope } = desktopEvidenceFixture(smoke, { + work: [{ workId: 'unfinished', label: 'selected test cohort', status: 'not-run' }], + }); + assert.throws(() => parseNativeReport(JSON.parse(envelope)), /incomplete execution/); + }); + + it('rejects a native identity that is inconsistent with its canonical test ID', () => { + const { run } = desktopEvidenceFixture(smoke); + run.tests[1].testId = `csf-${'0'.repeat(64)}`; + assert.throws( + () => + parseNativeReport({ + schemaVersion: 1, + format: 'desktop-test', + desktopRun: JSON.stringify(run), + storyReport: smoke, + }), + /authored callback identity/, + ); + }); + + it('accepts the new direct optional-cohort skipped status without calling it a pass', () => { + const result = parseNativeReport({ + ...direct, + status: 'skipped', + tests: [{ ...nativeCase, status: 'skipped', skipReason: 'physical input disabled' }], + }); + assert.equal(result.cases[0].status, 'skipped'); + assert.equal(result.status, 'failed'); + assert.match(result.error ?? '', /skipped/); + }); + }); + + it('records a raw historical pass without accepting it as current qualification', () => { + const report = assessCheckbox(options({ context: undefined, expectedBuildId: undefined, maxAgeHours: undefined })); + assert.equal(report.evidence.status, 'unverified'); + assert.equal(report.requirements[1].check?.recordedOutcome, 'passed'); + assert.equal(report.requirements[1].check?.evidenceOutcome, 'unverified'); + assert.equal(report.requirements[1].outcome, 'insufficient-evidence'); + }); + + it('uses aggregate wdio results, not enclosing success or portable tests', () => { + const report = assessCheckbox(options({ report: parseNativeReport(aggregate), context: undefined })); + assert.equal(report.evidence.runStatus, 'passed'); + assert.equal(report.evidence.caseCounts.skipped, 1); + assert.equal(report.evidence.caseCounts.passed, 0); + assert.equal(report.requirements[1].check?.recordedOutcome, 'skipped'); + assert.match(report.requirements[1].check?.reason ?? '', /physical pointer/); + assert.equal(report.assessmentOutcome, 'insufficient-evidence'); + }); + + it('keeps a matched skip as a skip and never counts it as a pass', () => { + const report = assessCheckbox( + options({ + report: parseNativeReport(aggregate), + context: { ...context, physicalClick: false }, + }), + ); + assert.equal(report.evidence.status, 'matched'); + assert.equal(report.requirements[1].check?.evidenceOutcome, 'skipped'); + assert.equal(report.requirements[1].outcome, 'insufficient-evidence'); + }); + + it('blocks mapped evidence on a case failure without attributing the precise broken requirement', () => { + const report = assessCheckbox( + options({ + report: parseNativeReport({ ...direct, status: 'failed', tests: [{ ...nativeCase, status: 'failed', error: 'assertion failed' }] }), + }), + ); + assert.equal(report.assessmentOutcome, 'failed'); + assert.equal(report.requirements[1].outcome, 'blocked-by-failure'); + assert.equal(report.requirements[3].outcome, 'blocked-by-failure'); + assert.equal(report.requirements[1].check?.reason, 'assertion failed'); + }); + + it('separates runner errors from product assertion failures and does not accept partial-run passes', () => { + const report = assessCheckbox(options({ report: parseNativeReport({ ...direct, status: 'failed', error: 'driver disconnected' }) })); + assert.equal(report.assessmentOutcome, 'execution-failed'); + assert.equal(report.evidence.runError, 'driver disconnected'); + assert.equal(report.requirements[1].check?.recordedOutcome, 'passed'); + assert.equal(report.requirements[1].check?.evidenceOutcome, 'unverified'); + }); + + for (const report of [ + undefined, + parseNativeReport({ ...direct, tests: [] }), + parseNativeReport({ ...direct, tests: [{ ...nativeCase, storyId: 'components-button--default' }] }), + ]) { + it(`reports missing coverage for ${report === undefined ? 'absent report' : report.cases.length ? 'unrelated cases' : 'empty cases'}`, () => { + const result = assessCheckbox(options({ report, context: undefined })); + assert.equal(result.requirements[1].check?.recordedOutcome, 'not-run'); + assert.equal(result.requirements[1].outcome, 'insufficient-evidence'); + }); + } + + it('detects endpoint and native-host mismatches even without provenance context', () => { + const report = assessCheckbox(options({ report: parseNativeReport({ ...direct, endpoint: 'windows' }), context: undefined })); + assert.equal(report.assessmentHealth, 'blocked'); + assert.equal(report.evidence.status, 'mismatched'); + const host = assessCheckbox(options({ report: parseNativeReport({ ...aggregate, targetId: 'different-host' }), context: undefined })); + assert.equal(host.evidence.status, 'mismatched'); + }); + + for (const [field, value] of Object.entries({ + reportSha256: '1'.repeat(64), + sourceCommit: '2'.repeat(40), + contractDigest: '3'.repeat(64), + platformManifestDigest: '4'.repeat(64), + buildId: 'another-build', + finishedAt: '2026-10-22T00:00:00.000Z', + physicalClick: false, + })) { + it(`rejects mismatched ${field} provenance`, () => { + const report = assessCheckbox(options({ context: parseEvidenceContext({ ...context, [field]: value }) })); + assert.equal(report.evidence.status, 'mismatched'); + assert.equal(report.requirements[1].check?.evidenceOutcome, 'mismatched'); + assert.equal(report.evidenceSupportedStatus, null); + }); + } + + it('does not accept a context timestamp that contradicts the aggregate', () => { + const report = assessCheckbox( + options({ report: parseNativeReport(aggregate), context: { ...context, finishedAt: '2026-10-21T20:01:00.000Z' } }), + ); + assert.equal(report.evidence.status, 'mismatched'); + }); + + it('requires explicit freshness/build policy and a commit-identifiable working tree', () => { + for (const missing of [{ maxAgeHours: undefined }, { expectedBuildId: undefined }, { workingTreeDirty: true }]) { + assert.equal(assessCheckbox(options(missing)).evidence.status, 'unverified'); + } + assert.equal(assessCheckbox(options({ maxAgeHours: 1 })).evidence.status, 'matched'); + const stale = assessCheckbox(options({ maxAgeHours: 0.5 })); + assert.equal(stale.assessmentHealth, 'stale'); + assert.equal(stale.requirements[1].check?.evidenceOutcome, 'stale'); + assert.equal(stale.evidenceSupportedStatus, null); + }); + + it('invalidates changed reviewed inputs and never promotes an unmapped requirement', () => { + const report = assessCheckbox( + options({ + contract: { + ...contract, + mappingCurrent: false, + requirements: [...contract.requirements, { id: 'CBX-007', summary: 'New contract requirement', declaredEvidence: [] }], + }, + }), + ); + assert.equal(report.assessmentHealth, 'stale'); + assert.equal(report.requirements[1].check?.evidenceOutcome, 'stale'); + assert.equal(report.requirements[6].check, null); + assert.equal(report.requirements[6].outcome, 'insufficient-evidence'); + }); + + it('fingerprints normalized contract/story content and detects a changed source file', () => { + const directory = fixture(); + const component = 'packages/agentic/components/src/components/checkbox'; + for (const file of [ + 'SPEC.md', + 'spec/source.json', + 'spec/interaction.md', + 'spec/accessibility.md', + 'spec/tokens.yaml', + 'spec/usage.md', + 'checkbox.stories.tsx', + ]) { + const destination = path.join(directory, component, file); + mkdirSync(path.dirname(destination), { recursive: true }); + copyFileSync(path.join(root, component, file), destination); + writeFileSync(destination, readFileSync(destination, 'utf8').replace(/\r?\n/g, '\r\n')); + } + const helper = path.join(directory, checkboxNativeAssertionPath); + mkdirSync(path.dirname(helper), { recursive: true }); + writeFileSync(helper, readFileSync(path.join(root, checkboxNativeAssertionPath), 'utf8').replace(/\r?\n/g, '\r\n')); + assert.equal(readCheckboxContract(directory).mappingCurrent, true); + writeFileSync(path.join(directory, component, 'checkbox.stories.tsx'), '// changed test\n'); + assert.equal(readCheckboxContract(directory).mappingCurrent, false); + }); + + it('invalidates native mappings when only the assertion-bearing helper changes', () => { + const changed = readCheckboxContractFrom((file) => { + const content = readFileSync(path.join(root, file), 'utf8'); + return file === checkboxNativeAssertionPath ? `${content}\n// changed native assertion\n` : content; + }); + assert.notEqual(changed.digest, contract.digest); + assert.equal(changed.mappingCurrent, false); + }); + + it('produces deterministic decisions and escapes untrusted Markdown', () => { + assert.deepEqual(assessCheckbox(options()), assessCheckbox(options())); + for (const tag of ['script', 'SCRIPT', 'ScRiPt']) { + const report = assessCheckbox( + options({ + report: parseNativeReport({ + ...direct, + tests: [{ ...nativeCase, status: 'skipped', skipReason: `<${tag}>|[link](bad)\nsecond line` }], + }), + context: undefined, + }), + ); + const output = formatAssessmentMarkdown(report); + assert.match(output, /not established/); + assert.ok(output.includes(`\\<${tag}\\>\\|\\[link\\]`)); + assert.equal(output.includes(`<${tag}>`), false); + } + }); +}); + +describe('native evidence validation', () => { + const malformed: [string, unknown][] = [ + ['non-object input', null], + ['missing case array', { ...direct, tests: undefined }], + ['unknown case status', { ...direct, tests: [{ ...nativeCase, status: 'pending' }] }], + ['unknown root status', { ...direct, status: 'success' }], + ['bad platform digest', { ...direct, platformManifestDigest: 'bad' }], + ['negative duration', { ...direct, tests: [{ ...nativeCase, durationMs: -1 }] }], + ['non-finite duration', { ...direct, tests: [{ ...nativeCase, durationMs: Infinity }] }], + ['missing duration', { ...direct, tests: [{ ...nativeCase, durationMs: undefined }] }], + ['duplicate native identity', { ...direct, tests: [nativeCase, nativeCase] }], + ['missing skip reason', { ...direct, tests: [{ ...nativeCase, status: 'skipped' }] }], + ['invalid error', { ...direct, status: 'failed', error: {} }], + ['invalid case error', { ...direct, tests: [{ ...nativeCase, error: {} }] }], + ['passed report with failed case', { ...direct, tests: [{ ...nativeCase, status: 'failed' }] }], + ['passed report with runner error', { ...direct, error: 'driver failed' }], + ['passed case with error', { ...direct, tests: [{ ...nativeCase, error: 'failed' }] }], + ['passed case with evidence error', { ...direct, tests: [{ ...nativeCase, evidenceErrors: ['capture failed'] }] }], + ['invalid evidence errors', { ...direct, tests: [{ ...nativeCase, evidenceErrors: 'capture failed' }] }], + ['unknown smoke schema', { ...aggregate, schemaVersion: 2 }], + ['missing smoke wdio', { ...aggregate, wdio: undefined }], + ['bad smoke digest', { ...aggregate, manifest: { platform: 'bad', portable: 'c'.repeat(64) } }], + ['contradictory smoke digest', { ...aggregate, platformManifestDigest: 'd'.repeat(64) }], + ['smoke time reversal', { ...aggregate, startedAt: '2026-10-22T00:00:00.000Z' }], + ['impossible date', { ...aggregate, finishedAt: '2026-02-30T20:00:00.000Z' }], + ]; + for (const [label, value] of malformed) it(`rejects ${label}`, () => assert.throws(() => parseNativeReport(value))); + + for (const [field, value] of Object.entries({ + schemaVersion: 2, + reportSha256: 'bad', + sourceCommit: 'HEAD', + buildId: '', + contractDigest: 'bad', + platformManifestDigest: 'bad', + endpoint: 'windows', + renderer: 'fabric', + targetId: 'another-host', + finishedAt: '2026-02-30T20:00:00.000Z', + physicalClick: 'true', + })) { + it(`rejects invalid context ${field}`, () => assert.throws(() => parseEvidenceContext({ ...context, [field]: value }))); + } + it('round-trips a valid execution context', () => assert.deepEqual(parseEvidenceContext(context), context)); + it('rejects context without report, invalid target identity, and invalid age options', () => { + assert.throws(() => assessCheckbox(options({ report: undefined }))); + assert.throws(() => assessCheckbox(options({ sourceCommit: 'HEAD' }))); + for (const maxAgeHours of [0, -1, NaN, Infinity]) assert.throws(() => assessCheckbox(options({ maxAgeHours }))); + }); +}); + +describe('assess-control command', () => { + function run(args: string[]) { + return spawnSync(process.execPath, [path.join(root, 'scripts', 'src', 'cli.ts'), 'assess-control', ...args], { + cwd: root, + encoding: 'utf8', + }); + } + + it('consumes runtime unit JSON without inventing provenance, and protects unit inputs from overwrite', () => { + const directory = fixture(); + const input = path.join(directory, 'jest.json'); + const raw = JSON.stringify(checkboxJestFixture()); + writeFileSync(input, raw); + const result = run(['--component', 'checkbox', '--unit-evidence', input, '--output', directory]); + assert.equal(result.status, 0, result.stderr || result.stdout); + const report = JSON.parse(readFileSync(path.join(directory, 'assessment.json'), 'utf8')); + assert.equal(report.unitEvidence.caseCounts.passed, 42); + assert.equal(report.unitEvidence.status, 'unverified'); + assert.equal(report.evidence.status, 'missing'); + assert.equal(report.inputs.unitReport, input); + const other = fixture(); + linkSync(input, path.join(other, 'assessment.json')); + assert.notEqual(run(['--component', 'checkbox', '--unit-evidence', input, '--output', other]).status, 0); + assert.equal(readFileSync(input, 'utf8'), raw); + assert.notEqual(run(['--component', 'checkbox', '--run-unit-tests', '--unit-evidence', input]).status, 0); + assert.notEqual(run(['--component', 'checkbox', '--unit-context', input]).status, 0); + assert.notEqual(run(['--component', 'checkbox', '--unit-build-id', 'missing-report']).status, 0); + }); + + it('persists readable/JSON reports and leaves input bytes unchanged', () => { + const directory = fixture(); + const input = path.join(directory, 'run.json'); + writeFileSync(input, JSON.stringify(aggregate)); + const before = readFileSync(input, 'utf8'); + const output = path.join(directory, 'reports'); + const result = run(['--component', 'checkbox', '--evidence', input, '--output', output]); + assert.equal(result.status, 0, result.stderr || result.stdout); + assert.match(result.stdout, /insufficient-evidence; readiness not established/); + const report = JSON.parse(readFileSync(path.join(output, 'assessment.json'), 'utf8')); + assert.equal(report.requirements[1].check.recordedOutcome, 'skipped'); + assert.equal(report.evidenceSupportedStatus, null); + assert.match(readFileSync(path.join(output, 'assessment.md'), 'utf8'), /physical pointer input/); + assert.equal(readFileSync(input, 'utf8'), before); + }); + + it('produces an explicit evidence inventory when no report is supplied', () => { + const directory = fixture(); + const result = run(['--component', 'checkbox', '--output', directory]); + assert.equal(result.status, 0, result.stderr || result.stdout); + assert.equal(JSON.parse(readFileSync(path.join(directory, 'assessment.json'), 'utf8')).evidence.status, 'missing'); + }); + + it('fails for malformed/missing input, unsupported controls, and attempted input overwrite', () => { + const directory = fixture(); + const input = path.join(directory, 'assessment.json'); + for (const content of ['{', raw]) { + writeFileSync(input, content); + const result = run(['--component', 'checkbox', '--evidence', input, '--output', directory]); + assert.notEqual(result.status, 0); + assert.equal(readFileSync(input, 'utf8'), content); + } + assert.notEqual(run(['--component', 'checkbox', '--evidence', path.join(directory, 'missing.json'), '--output', directory]).status, 0); + assert.notEqual(run(['--component', 'button', '--output', directory]).status, 0); + }); + + it('does not overwrite evidence through a different hard-linked filename', () => { + const directory = fixture(); + const input = path.join(directory, 'run.json'); + writeFileSync(input, raw); + linkSync(input, path.join(directory, 'assessment.json')); + const result = run(['--component', 'checkbox', '--evidence', input, '--output', directory]); + assert.notEqual(result.status, 0); + assert.equal(readFileSync(input, 'utf8'), raw); + }); + + it('rejects JSON and Markdown outputs that alias one another', () => { + const directory = fixture(); + const output = path.join(directory, 'assessment.json'); + writeFileSync(output, 'existing report'); + linkSync(output, path.join(directory, 'assessment.md')); + const result = run(['--component', 'checkbox', '--output', directory]); + assert.notEqual(result.status, 0); + assert.equal(readFileSync(output, 'utf8'), 'existing report'); + }); + + it('updates SPEC only on explicit opt-in, preserves its contract and records honest local identity', () => { + const directory = fixture(); + const checkout = path.join(directory, 'repo'); + mkdirSync(checkout); + const invokeGit = (...args: string[]) => { + const result = spawnSync('git', args, { cwd: checkout, encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr || result.stdout); + return result.stdout.trim(); + }; + invokeGit('init', '-q'); + invokeGit('config', 'core.autocrlf', 'false'); + const component = path.dirname(assessmentSpecPath); + for (const file of [ + 'SPEC.md', + 'spec/source.json', + 'spec/interaction.md', + 'spec/accessibility.md', + 'spec/tokens.yaml', + 'spec/usage.md', + 'checkbox.stories.tsx', + ]) { + const target = path.join(checkout, component, file); + mkdirSync(path.dirname(target), { recursive: true }); + copyFileSync(path.join(root, component, file), target); + } + const helper = path.join(checkout, checkboxNativeAssertionPath); + mkdirSync(path.dirname(helper), { recursive: true }); + copyFileSync(path.join(root, checkboxNativeAssertionPath), helper); + writeFileSync(path.join(checkout, assessmentPackagePath), JSON.stringify({ version: '1.2.3' })); + invokeGit('add', '.'); + invokeGit( + '-c', + 'user.name=Fixture', + '-c', + 'user.email=fixture@example.invalid', + '-c', + 'commit.gpgsign=false', + 'commit', + '-qm', + 'Fixture', + ); + const revision = invokeGit('rev-parse', 'HEAD'); + const specPath = path.join(checkout, assessmentSpecPath); + const original = readFileSync(specPath, 'utf8'); + const input = path.join(directory, 'run.json'); + writeFileSync(input, raw); + const localRun = (extra: string[] = []) => + spawnSync( + process.execPath, + [ + path.join(root, 'scripts', 'src', 'cli.ts'), + 'assess-control', + '--component', + 'checkbox', + '--evidence', + input, + '--output', + path.join(directory, 'output'), + ...extra, + ], + { cwd: checkout, encoding: 'utf8' }, + ); + const ordinary = localRun(); + assert.equal(ordinary.status, 0, ordinary.stderr || ordinary.stdout); + assert.equal(readFileSync(specPath, 'utf8'), original); + assert.equal(invokeGit('status', '--porcelain'), ''); + writeFileSync(path.join(checkout, 'uncommitted.txt'), 'Local source is dirty.\n'); + const update = localRun(['--update-spec']); + assert.equal(update.status, 0, update.stderr || update.stdout); + const snapshot = splitSpecAssessment(readFileSync(specPath, 'utf8')); + assert.equal(snapshot.contract, splitSpecAssessment(original).contract); + assert.equal(snapshot.assessment?.packageVersion, '1.2.3'); + assert.equal(snapshot.assessment?.sourceCommit, revision); + assert.equal(snapshot.assessment?.testedCommit, null); + assert.equal(snapshot.assessment?.dirty, true); + assert.equal(snapshot.assessment?.provenance, 'unverified'); + assert.equal(snapshot.assessment?.outcome, 'insufficient-evidence'); + const summary = path.join(path.dirname(specPath), 'assessment', 'ASSESSMENT.md'); + assert.match(readFileSync(summary, 'utf8'), /Not a bound CI publication/); + assert.equal(readFileSync(input, 'utf8'), raw); + linkSync(summary, path.join(directory, 'summary-alias.md')); + const linked = localRun(['--update-spec']); + assert.notEqual(linked.status, 0); + assert.match(linked.stderr || linked.stdout, /regular, unlinked/); + assert.equal(splitSpecAssessment(readFileSync(specPath, 'utf8')).contract, splitSpecAssessment(original).contract); + }); +}); + +describe('generated SPEC assessment snapshot', () => { + const original = splitSpecAssessment(readFileSync(path.join(root, assessmentSpecPath), 'utf8')).contract; + const snapshot = specAssessmentFor(assessCheckbox(options()), '0.7.0'); + + for (const eol of ['\n', '\r\n']) { + it(`preserves all contract bytes and lifecycle with ${JSON.stringify(eol)} line endings`, () => { + const spec = original.replace(/\r\n/g, '\n').replace(/\n/g, eol); + const updated = updateSpecAssessment(spec, snapshot); + assert.deepEqual(splitSpecAssessment(updated), { contract: spec, assessment: snapshot }); + assert.equal(updateSpecAssessment(updated, snapshot), updated); + assert.equal((updated.match(/status: implemented/g) ?? []).length, 1); + assert.match(updated, /assessment-stage: 'not-determined'/); + }); + } + + it('keeps the reviewed mapping digest stable only for the generated block', () => { + const read = (spec: string) => + readCheckboxContractFrom((file) => (file === assessmentSpecPath ? spec : readFileSync(path.join(root, file), 'utf8'))); + const generated = updateSpecAssessment(original, snapshot); + assert.equal(read(generated).digest, contract.digest); + assert.equal(read(generated).mappingCurrent, true); + assert.notEqual(read(`${generated}\nNew human contract requirement.\n`).digest, contract.digest); + assert.equal(read(`${generated}\nNew human contract requirement.\n`).mappingCurrent, false); + assert.notEqual(read(generated.replace('status: implemented', 'status: contract-reviewed')).digest, contract.digest); + }); + + it('does not invent a tested commit from missing, dirty, mismatched, or stale evidence', () => { + for (const report of [ + assessCheckbox(options({ report: undefined, context: undefined, reportSha256: undefined })), + assessCheckbox(options({ workingTreeDirty: true })), + assessCheckbox(options({ context: { ...context, sourceCommit: 'b'.repeat(40) } })), + assessCheckbox(options({ maxAgeHours: 0.01 })), + ]) { + const metadata = specAssessmentFor(report, '0.7.0'); + assert.equal(metadata.testedCommit, null); + assert.equal(metadata.sourceCommit, sourceCommit); + assert.equal(metadata.provenance, report.evidence.status); + assert.equal(metadata.health, report.assessmentHealth); + assert.equal(metadata.outcome, report.assessmentOutcome); + assert.match(updateSpecAssessment(original, metadata), /assessment-tested-commit: null/); + } + assert.equal(snapshot.testedCommit, sourceCommit); + }); + + it('preserves native assertion and execution failures rather than promoting a maturity stage', () => { + const failure = { ...direct, status: 'failed', tests: [{ ...nativeCase, status: 'failed', error: 'Native assertion failed.' }] }; + const bytes = JSON.stringify(failure); + const failed = assessCheckbox( + options({ + report: parseNativeReport(failure), + reportSha256: reportDigest(bytes), + context: { ...context, reportSha256: reportDigest(bytes) }, + }), + ); + assert.equal(specAssessmentFor(failed, '0.7.0').outcome, 'failed'); + const blocked = assessCheckbox(options({ executionFailure: 'Native helper exited unexpectedly.' })); + assert.equal(specAssessmentFor(blocked, '0.7.0').outcome, 'execution-failed'); + }); + + it('rejects malformed, duplicated, unknown, misplaced, noncanonical and falsely bound metadata', () => { + const valid = updateSpecAssessment(original, snapshot); + const malformed = [ + original.replace('status: implemented', 'status: implemented\nassessment-outcome: "failed"'), + valid.replace('assessment-schema: 1', 'assessment-schema: 2'), + valid.replace('assessment-schema: 1', 'assessment-schema: 1\nassessment-schema: 1'), + valid.replace('assessment-schema: 1', 'assessment-schema: 1\nassessment-extra: "arbitrary contract content"'), + valid.replace("assessment-outcome: 'insufficient-evidence'", "assessment-outcome: 'production-ready'"), + valid.replace("assessment-stage: 'not-determined'", "assessment-stage: 'beta'"), + valid.replace("assessment-package-version: '0.7.0'", "assessment-package-version: '0.7.0'\ninjected: true"), + valid.replace("assessment-generated-at: '2026-10-21T21:00:00.000Z'", "assessment-generated-at: '2026-02-30T21:00:00.000Z'"), + valid.replace('assessment-dirty: false', 'assessment-dirty: true'), + valid.replace('assessment-tested-commit:', 'assessment-wrong-commit:'), + valid.replace('# furn-assessment:end', ''), + `${valid}\n# furn-assessment:begin\n`, + `${original}\n# furn-assessment:begin\n# furn-assessment:end\n`, + valid.replace('assessment-schema: 1', 'assessment-schema: 1'), + valid.replace("assessment-target: 'win32-paper-storybook'", "assessment-target: 'windows-fabric-storybook'"), + valid.replace("assessment-report: './assessment/ASSESSMENT.md'", "assessment-report: '../other-file.md'"), + valid.replace('assessment-schema: 1', 'assessment-schema: 1\n__proto__: {"assessment-schema":1}'), + ]; + for (const content of malformed) assert.throws(() => splitSpecAssessment(content)); + assert.throws(() => updateSpecAssessment('No front matter', snapshot), /front matter/); + assert.throws(() => updateSpecAssessment('---\nunterminated', snapshot), /unterminated/); + assert.throws(() => updateSpecAssessment(original, { ...snapshot, packageVersion: 'not-a-version' }), /identity/); + assert.throws(() => updateSpecAssessment(original, { ...snapshot, testedCommit: null }), /matched, clean/); + }); +}); diff --git a/scripts/src/tasks/controlAssessment.ts b/scripts/src/tasks/controlAssessment.ts new file mode 100644 index 0000000000..b7b57ccc59 --- /dev/null +++ b/scripts/src/tasks/controlAssessment.ts @@ -0,0 +1,702 @@ +import { createHash } from 'node:crypto'; +import { readFileSync, realpathSync } from 'node:fs'; +import path from 'node:path'; + +import { splitSpecAssessment } from './controlAssessmentSpec.ts'; +import { assessCheckboxUnits } from './controlAssessmentUnit.ts'; +import type { UnitInput } from './controlAssessmentUnit.ts'; + +const componentPath = 'packages/agentic/components/src/components/checkbox'; +export const checkboxNativeAssertionPath = 'packages/agentic/components/src/common/desktopFocus.wdio.ts'; +const contractFiles = [ + 'SPEC.md', + 'spec/source.json', + 'spec/interaction.md', + 'spec/accessibility.md', + 'spec/tokens.yaml', + 'spec/usage.md', + 'checkbox.stories.tsx', +]; +// October 29 review: the mapped default callback delegates checked-state assertions to the fingerprinted native helper. +const reviewedDigest = '8e05597cabf63743066b9dc9dd929d11e8c48fad89e3a7eabb853efbc6c58b6a'; +const storyId = 'components-checkbox--default'; +const testName = 'toggles native checked state in both directions'; + +const coverage: Record = { + 'CBX-001': { + remaining: + 'Map the additional existing defaults/root-prop assertions and ingest the existing compiler/type result; no native duplicate is inherently required.', + }, + 'CBX-002': { + observed: 'Uncontrolled unchecked -> checked -> unchecked through physical clicks.', + remaining: + 'Map the additional existing JS controlled/uncontrolled press and callback-count assertions, including defaultStatus precedence; native duplication is not required for state logic.', + }, + 'CBX-003': { + remaining: + 'Map the additional existing anatomy/order, text-gating and warning assertions and native wrapping cases; rendered geometry needs layout evidence.', + }, + 'CBX-004': { + observed: 'Native checkbox role (not the complete accessibility requirement).', + remaining: + 'Map the additional existing inner-node and JS semantics assertions and native naming cases; actual native action dispatch needs platform evidence when claimed.', + }, + 'CBX-005': { + remaining: + 'Map the additional existing secondary-text/state, interaction/status and caller-style assertions; resolved styles can be established by unit tests.', + }, + 'CBX-006': { remaining: 'Win32 focus modality, disabled/detached targets, and dual-ring geometry; collect focus/visual evidence.' }, +}; + +type Requirement = { id: string; summary: string; declaredEvidence: string[] }; +export type CheckboxContract = { + digest: string; + mappingCurrent: boolean; + lifecycle: string; + conformance: string; + requirements: Requirement[]; +}; +type CaseStatus = 'passed' | 'failed' | 'skipped'; +type NativeCase = { storyId: string; testName: string; status: CaseStatus; durationMs: number; reason?: string }; +export type NativeReport = { + endpoint: 'win32' | 'windows' | 'macos'; + platformManifestDigest: string; + status: 'passed' | 'failed'; + error?: string; + targetId?: string; + startedAt?: string; + finishedAt?: string; + cases: NativeCase[]; + format: 'wdio' | 'storybook-smoke' | 'desktop-test'; + harness?: { + runId: string; + status: string; + retries: number; + flaky: number; + unsupported: number; + incomplete: number; + failureCount: number; + }; +}; +export type EvidenceContext = { + schemaVersion: 1; + reportSha256: string; + sourceCommit: string; + buildId: string; + contractDigest: string; + platformManifestDigest: string; + endpoint: 'win32'; + renderer: 'paper'; + targetId: string; + finishedAt: string; + physicalClick: boolean; +}; +type Provenance = { status: 'missing' | 'unverified' | 'matched' | 'mismatched' | 'stale'; reasons: string[] }; +type CheckOutcome = CaseStatus | 'not-run' | 'unverified' | 'mismatched' | 'stale'; +export type AssessmentOptions = { + contract: CheckboxContract; + sourceCommit: string; + workingTreeDirty: boolean; + now: Date; + report?: NativeReport; + reportSha256?: string; + context?: EvidenceContext; + expectedBuildId?: string; + expectedTargetId?: string; + maxAgeHours?: number; + executionFailure?: string; + unit?: UnitInput; +}; + +function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +export function object(value: unknown, label: string): Record { + if (!isObject(value)) throw new Error(`${label} must be an object.`); + return value; +} + +export function text(value: unknown, label: string): string { + if (typeof value !== 'string' || !value.trim()) throw new Error(`${label} must be a nonempty string.`); + return value; +} + +export function hex(value: unknown, length: number, label: string): string { + const result = text(value, label); + if (!new RegExp(`^[0-9a-f]{${length}}$`).test(result)) throw new Error(`${label} must be ${length} lowercase hex characters.`); + return result; +} + +function choice(value: unknown, choices: readonly T[], label: string): T { + const result = choices.find((entry) => entry === value); + if (result === undefined) throw new Error(`${label} must be one of: ${choices.join(', ')}.`); + return result; +} + +export function timestamp(value: unknown, label: string): string { + const result = text(value, label); + const time = Date.parse(result); + if ( + !/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d{1,3})?Z$/.test(result) || + !Number.isFinite(time) || + new Date(time).toISOString().slice(0, 19) !== result.slice(0, 19) + ) { + throw new Error(`${label} must be a valid UTC ISO timestamp.`); + } + return result; +} + +export function reportDigest(content: string): string { + return createHash('sha256').update(content).digest('hex'); +} + +export function win32StorybookTargetId(root: string): string { + // Mirror desktop-test's target identity protocol without loading PR configuration in the trusted publisher. + const appRoot = realpathSync(path.join(root, 'apps', 'storybook')); + return `win32-${reportDigest(`${appRoot}\0win32\0paper`).slice(0, 16)}`; +} + +function isWin32StorybookTarget(value: string): boolean { + return value === 'agenticstorybook-win32' || /^win32-[0-9a-f]{16}$/.test(value); +} + +export function readCheckboxContract(root: string): CheckboxContract { + return readCheckboxContractFrom((file) => readFileSync(path.join(root, file), 'utf8')); +} + +export function readCheckboxContractFrom(read: (file: string) => string): CheckboxContract { + const files = new Map( + contractFiles.map((file) => { + const content = read(`${componentPath}/${file}`).replace(/\r\n/g, '\n'); + return [file, file === 'SPEC.md' ? splitSpecAssessment(content).contract : content]; + }), + ); + const hash = createHash('sha256'); + for (const [file, content] of files) hash.update(`${file}\0${content}\0`); + hash.update(`${checkboxNativeAssertionPath}\0${read(checkboxNativeAssertionPath).replace(/\r\n/g, '\n')}\0`); + const digest = hash.digest('hex'); + const source = object(JSON.parse(files.get('spec/source.json')!), 'Checkbox source metadata'); + if (source.component !== 'checkbox' || source.schemaVersion !== 2 || !Array.isArray(source.requirements)) { + throw new Error('Expected Checkbox source metadata schemaVersion 2 and a requirements array.'); + } + const summaries = new Map( + [...files.get('SPEC.md')!.matchAll(/^- \*\*(CBX-\d{3}):\*\* (.+(?:\n {2}.+)*)/gm)].map((match) => [ + match[1], + match[2].replace(/\s+/g, ' ').trim(), + ]), + ); + const requirements = source.requirements.map((value) => { + const entry = object(value, 'Requirement'); + const id = text(entry.id, 'Requirement id'); + const summary = summaries.get(id); + if (!summary || !Array.isArray(entry.evidence)) throw new Error(`Missing SPEC requirement or evidence list for ${id}.`); + return { id, summary, declaredEvidence: entry.evidence.map((file) => text(file, `${id} evidence file`)) }; + }); + if (new Set(requirements.map(({ id }) => id)).size !== requirements.length || summaries.size !== requirements.length) { + throw new Error('Checkbox SPEC and metadata must contain the same unique requirement IDs.'); + } + return { + digest, + mappingCurrent: digest === reviewedDigest, + lifecycle: text(source.lifecycle, 'Contract lifecycle'), + conformance: text(source.conformance, 'Contract conformance'), + requirements, + }; +} + +export function parseNativeReport(value: unknown): NativeReport { + const data = object(value, 'Native report'); + if (data.format === 'desktop-test') return parseDesktopEvidence(data); + const format = 'wdio' in data || 'schemaVersion' in data || 'manifest' in data ? 'storybook-smoke' : 'wdio'; + const entries = format === 'storybook-smoke' ? data.wdio : data.tests; + if (!Array.isArray(entries)) throw new Error('Native report must contain a tests or wdio array.'); + const error = data.error; + if (error !== undefined && typeof error !== 'string') throw new Error('Native report error must be a string.'); + const rawStatus = choice(data.status, ['passed', 'failed', 'skipped', 'incomplete'], 'Report status'); + const status = rawStatus === 'passed' ? 'passed' : 'failed'; + if (status === 'passed' && error !== undefined) throw new Error('Passed report cannot contain a runner error.'); + let platformManifestDigest: string; + let targetId: string | undefined; + let startedAt: string | undefined; + let finishedAt: string | undefined; + if (format === 'storybook-smoke') { + if (data.schemaVersion !== 1 || !Array.isArray(data.tests)) + throw new Error('Storybook smoke report requires schemaVersion 1 and a tests array.'); + const manifest = object(data.manifest, 'Smoke manifest'); + platformManifestDigest = hex(manifest.platform, 64, 'Smoke platform manifest'); + hex(manifest.portable, 64, 'Smoke portable manifest'); + if (data.platformManifestDigest !== undefined && data.platformManifestDigest !== platformManifestDigest) + throw new Error('Conflicting report platform digests.'); + targetId = text(data.targetId, 'Smoke targetId'); + text(data.runId, 'Smoke runId'); + text(data.platformName, 'Smoke platformName'); + finishedAt = timestamp(data.finishedAt, 'Smoke finishedAt'); + startedAt = timestamp(data.startedAt, 'Smoke startedAt'); + if (Date.parse(startedAt) > Date.parse(finishedAt)) throw new Error('Smoke report finished before it started.'); + } else { + platformManifestDigest = hex(data.platformManifestDigest, 64, 'Report platformManifestDigest'); + } + const identities = new Set(); + const cases = entries.map((value, index): NativeCase => { + const entry = object(value, `Native case ${index}`); + const id = text(entry.storyId, 'Case storyId'); + const name = entry.testName === undefined ? 'default' : text(entry.testName, 'Case testName'); + const identity = JSON.stringify([id, name]); + if (identities.has(identity)) throw new Error(`Duplicate native case: ${id} / ${name}.`); + identities.add(identity); + const caseStatus = choice(entry.status, ['passed', 'failed', 'skipped'], 'Case status'); + if (typeof entry.durationMs !== 'number' || !Number.isFinite(entry.durationMs) || entry.durationMs < 0) { + throw new Error(`Invalid durationMs for ${id} / ${name}.`); + } + const caseError = entry.error; + if (caseError !== undefined && typeof caseError !== 'string') throw new Error('Case error must be a string.'); + if ( + entry.evidenceErrors !== undefined && + (!Array.isArray(entry.evidenceErrors) || entry.evidenceErrors.some((value) => typeof value !== 'string')) + ) + throw new Error('Case evidenceErrors must be a string array.'); + if (caseStatus === 'passed' && (caseError !== undefined || (Array.isArray(entry.evidenceErrors) && entry.evidenceErrors.length))) + throw new Error('Passed case cannot contain execution or evidence errors.'); + if (status === 'passed' && caseStatus === 'failed') throw new Error('Passed report cannot contain a failed case.'); + return { + storyId: id, + testName: name, + status: caseStatus, + durationMs: entry.durationMs, + reason: caseStatus === 'skipped' ? text(entry.skipReason, 'Skipped case reason') : caseError, + }; + }); + return { + endpoint: choice(data.endpoint, ['win32', 'windows', 'macos'], 'Report endpoint'), + platformManifestDigest, + targetId, + finishedAt, + status, + error: error ?? (rawStatus === 'skipped' || rawStatus === 'incomplete' ? `Native execution is ${rawStatus}.` : undefined), + cases, + format, + startedAt, + }; +} + +const desktopStatuses = ['passed', 'failed', 'skipped', 'unsupported', 'cancelled', 'not-run', 'incomplete'] as const; +const smokeArtifact = 'evidence/win32/static/run.json'; + +export function hasStorybookArtifact(run: Record): boolean { + if (!Array.isArray(run.artifacts)) throw new Error('Desktop supervisor report has no artifact inventory.'); + return run.artifacts.some( + (value) => text(object(value, 'Desktop artifact').path, 'Artifact path').replaceAll('\\', '/') === smokeArtifact, + ); +} + +/** Keep the canonical supervisor result and its Storybook assertions together, including their exact bytes. */ +export function readNativeEvidence(file: string): string { + const content = readFileSync(file, 'utf8'); + const data = object(JSON.parse(content), 'Native evidence'); + if (data.kind !== 'tests' && data.kind !== 'lifecycle') return content; + const root = realpathSync(path.dirname(file)); + const companion = realpathSync(path.join(root, ...smokeArtifact.split('/'))); + const relative = path.relative(root, companion); + if (relative.startsWith('..') || path.isAbsolute(relative)) throw new Error('Storybook evidence must remain inside its desktop run.'); + return JSON.stringify({ schemaVersion: 1, format: 'desktop-test', desktopRun: content, storyReport: readFileSync(companion, 'utf8') }); +} + +function parseDesktopEvidence(data: Record): NativeReport { + if (data.schemaVersion !== 1) throw new Error('Unsupported desktop evidence envelope.'); + const run = object(JSON.parse(text(data.desktopRun, 'Desktop run bytes')), 'Desktop run'); + if (run.schemaVersion !== 1 || run.kind !== 'tests') throw new Error('Expected a desktop-test schemaVersion 1 test run.'); + const runId = text(run.runId, 'Desktop run ID'); + const startedAt = timestamp(run.startedAt, 'Desktop startedAt'); + const finishedAt = timestamp(run.finishedAt, 'Desktop finishedAt'); + if (Date.parse(startedAt) > Date.parse(finishedAt)) throw new Error('Desktop run finished before it started.'); + const status = choice(run.status, desktopStatuses, 'Desktop status'); + if (!Array.isArray(run.tests) || !Array.isArray(run.work) || !Array.isArray(run.failures) || !Array.isArray(run.artifacts)) { + throw new Error('Desktop run requires tests, work, failures and artifacts arrays.'); + } + const artifactMatches = run.artifacts.filter( + (value) => text(object(value, 'Desktop artifact').path, 'Artifact path').replaceAll('\\', '/') === smokeArtifact, + ); + if (artifactMatches.length !== 1) throw new Error('Desktop run must declare exactly one Win32 Storybook smoke artifact.'); + const storyData = object(JSON.parse(text(data.storyReport, 'Storybook report bytes')), 'Storybook smoke evidence'); + if (!('manifest' in storyData) || storyData.format === 'desktop-test') throw new Error('Expected a Storybook smoke companion.'); + const report = parseNativeReport(storyData); + if (report.format !== 'storybook-smoke' || report.endpoint !== 'win32') throw new Error('Expected Win32 Storybook smoke evidence.'); + if (Date.parse(report.startedAt!) < Date.parse(startedAt) || Date.parse(report.finishedAt!) > Date.parse(finishedAt)) { + throw new Error('Storybook evidence is outside its enclosing desktop execution.'); + } + const counts = { selected: 0, executed: 0, passed: 0, failed: 0, skipped: 0, unsupported: 0, incomplete: 0, retries: 0, flaky: 0 }; + const cases = new Map(); + const ids = new Set(); + for (const value of run.tests) { + const test = object(value, 'Desktop test'); + const id = text(test.testId, 'Desktop test ID'); + if (ids.has(id)) throw new Error('Duplicate desktop test ID.'); + ids.add(id); + text(test.title, 'Desktop test title'); + const testStatus = choice(test.status, desktopStatuses, 'Desktop test status'); + if (!Array.isArray(test.attempts) || !test.attempts.length) throw new Error('Desktop test needs recorded attempts.'); + let previous = -1; + let failedAttempt = false; + let executed = false; + let identity: string | undefined; + const attempts = test.attempts.map((value) => { + const attempt = object(value, 'Desktop attempt'); + if (!Number.isSafeInteger(attempt.attempt) || Number(attempt.attempt) <= previous) throw new Error('Invalid desktop attempt order.'); + previous = Number(attempt.attempt); + const result = choice(attempt.status, desktopStatuses, 'Desktop attempt status'); + if (typeof attempt.durationMs !== 'number' || !Number.isFinite(attempt.durationMs) || attempt.durationMs < 0) { + throw new Error('Invalid desktop attempt duration.'); + } + if (!['passed', 'failed'].includes(result)) text(attempt.reason, 'Desktop nonexecution reason'); + if (result === 'failed') object(attempt.failure, 'Desktop failed attempt'); + if (result === 'passed' && attempt.failure !== undefined) throw new Error('Passed desktop attempt contains a failure.'); + failedAttempt ||= result === 'failed'; + executed ||= result === 'passed' || result === 'failed'; + if (attempt.context !== undefined) { + const context = object(attempt.context, 'Desktop attempt context'); + if (id.startsWith('csf-')) { + const key = JSON.stringify([text(context.story, 'Desktop story'), text(context.test, 'Desktop test name')]); + if (identity !== undefined && identity !== key) throw new Error('Desktop test identity changed across attempts.'); + identity = key; + if (id !== `csf-${reportDigest(key)}`) throw new Error('Desktop test ID does not match the authored callback identity.'); + if (context.endpoint !== report.endpoint) throw new Error('Desktop attempt endpoint differs from Storybook evidence.'); + } + } + return result; + }); + if (attempts.at(-1) !== testStatus) throw new Error('Desktop final status disagrees with its last attempt.'); + const flaky = testStatus === 'passed' && failedAttempt; + if (test.flaky !== flaky) throw new Error('Desktop flaky flag disagrees with recorded attempts.'); + if (identity !== undefined) { + if (cases.has(identity)) throw new Error('Duplicate desktop story/test identity.'); + cases.set(identity, testStatus); + } + counts.selected++; + counts.executed += Number(executed); + counts.retries += test.attempts.length - 1; + counts.flaky += Number(flaky); + if (testStatus === 'passed' || testStatus === 'failed' || testStatus === 'skipped' || testStatus === 'unsupported') + counts[testStatus]++; + else counts.incomplete++; + } + const declaredCounts = object(run.counts, 'Desktop counts'); + for (const [name, value] of Object.entries(counts)) { + if (declaredCounts[name] !== value) throw new Error(`Desktop ${name} count disagrees with recorded attempts.`); + } + const work = run.work.map((value) => choice(object(value, 'Desktop selected work').status, desktopStatuses, 'Desktop work status')); + let infrastructureFailed = false; + for (const value of run.failures) { + const failure = object(value, 'Desktop failure'); + text(failure.failureId, 'Desktop failure ID'); + const kind = choice(failure.kind, ['test', 'infrastructure', 'cleanup', 'evidence', 'reporting'], 'Desktop failure kind'); + if (typeof object(failure.error, 'Desktop error').message !== 'string') throw new Error('Desktop error message must be a string.'); + infrastructureFailed ||= kind !== 'test'; + } + if ( + status === 'passed' && + (infrastructureFailed || + counts.failed || + counts.incomplete || + !counts.executed || + work.some((value) => ['failed', 'not-run', 'incomplete', 'cancelled'].includes(value))) + ) + throw new Error('Passed desktop run contains failures or incomplete execution.'); + for (const entry of report.cases) { + const recorded = cases.get(JSON.stringify([entry.storyId, entry.testName])); + if (recorded !== entry.status) throw new Error('Storybook case does not match canonical desktop results.'); + } + const issues = [ + ...(report.error ? [report.error] : []), + ...(status !== 'passed' ? [`Desktop supervisor status: ${status}.`] : []), + ...(run.failures.length ? [`Desktop run recorded ${run.failures.length} failure(s), including prior attempts.`] : []), + ...(counts.retries || counts.flaky ? ['Retried or flaky desktop execution is not qualifying evidence in this pilot.'] : []), + ]; + return { + ...report, + format: 'desktop-test', + startedAt, + finishedAt, + status: status === 'passed' && report.status === 'passed' && !issues.length ? 'passed' : 'failed', + error: issues.length ? issues.join(' ') : undefined, + harness: { + runId, + status, + retries: counts.retries, + flaky: counts.flaky, + unsupported: counts.unsupported, + incomplete: counts.incomplete, + failureCount: run.failures.length, + }, + }; +} + +export function parseEvidenceContext(value: unknown): EvidenceContext { + const data = object(value, 'Evidence context'); + if (data.schemaVersion !== 1) throw new Error('Evidence context schemaVersion must be 1.'); + if (typeof data.physicalClick !== 'boolean') throw new Error('Evidence context physicalClick must be boolean.'); + const targetId = text(data.targetId, 'Context targetId'); + if (!isWin32StorybookTarget(targetId)) throw new Error('Context targetId is not a supported Win32 Storybook target.'); + return { + schemaVersion: 1, + reportSha256: hex(data.reportSha256, 64, 'Context reportSha256'), + sourceCommit: hex(data.sourceCommit, 40, 'Context sourceCommit'), + buildId: text(data.buildId, 'Context buildId'), + contractDigest: hex(data.contractDigest, 64, 'Context contractDigest'), + platformManifestDigest: hex(data.platformManifestDigest, 64, 'Context platformManifestDigest'), + endpoint: choice(data.endpoint, ['win32'], 'Context endpoint'), + renderer: choice(data.renderer, ['paper'], 'Context renderer'), + targetId, + finishedAt: timestamp(data.finishedAt, 'Context finishedAt'), + physicalClick: data.physicalClick, + }; +} + +function provenance(options: AssessmentOptions): Provenance { + const { report, context, contract } = options; + if (!report) return { status: 'missing', reasons: ['No native execution report supplied.'] }; + const mismatches: string[] = []; + if (report.endpoint !== 'win32') mismatches.push(`Report endpoint is ${report.endpoint}, not win32.`); + if (report.targetId !== undefined && !isWin32StorybookTarget(report.targetId)) mismatches.push('Report targets a different native host.'); + if (report.targetId !== undefined && options.expectedTargetId !== undefined && report.targetId !== options.expectedTargetId) + mismatches.push('Report target does not match the expected native instance.'); + if (context) { + if (context.reportSha256 !== options.reportSha256) mismatches.push('Execution context is bound to different report bytes.'); + if (context.sourceCommit !== options.sourceCommit) mismatches.push('Execution source revision does not match the target revision.'); + if (context.contractDigest !== contract.digest) mismatches.push('Execution contract/mapping inputs do not match the current inputs.'); + if (context.platformManifestDigest !== report.platformManifestDigest) + mismatches.push('Execution platform manifest does not match the report.'); + if (report.targetId !== undefined && context.targetId !== report.targetId) + mismatches.push('Execution target does not match the report.'); + if (options.expectedTargetId !== undefined && context.targetId !== options.expectedTargetId) + mismatches.push('Execution target does not match the expected native instance.'); + if (report.finishedAt !== undefined && Date.parse(report.finishedAt) !== Date.parse(context.finishedAt)) + mismatches.push('Execution timestamp does not match the report.'); + if (options.expectedBuildId !== undefined && context.buildId !== options.expectedBuildId) + mismatches.push('Execution build ID does not match the requested build.'); + if (Date.parse(context.finishedAt) > options.now.getTime()) mismatches.push('Execution timestamp is in the future.'); + if ( + !context.physicalClick && + report.cases.some((entry) => entry.storyId === storyId && entry.testName === testName && entry.status === 'passed') + ) { + mismatches.push('Recorded Checkbox pass contradicts the execution physical-click capability.'); + } + } + if (mismatches.length) return { status: 'mismatched', reasons: mismatches }; + if ( + context && + options.maxAgeHours !== undefined && + options.now.getTime() - Date.parse(context.finishedAt) > options.maxAgeHours * 3_600_000 + ) { + return { status: 'stale', reasons: ['Execution evidence exceeds the explicitly requested maximum age.'] }; + } + const reasons: string[] = []; + if (!context) reasons.push('Report has no bound execution context; source/build/freshness cannot be established.'); + if (context?.targetId !== 'agenticstorybook-win32' && context && options.expectedTargetId === undefined) + reasons.push('No expected native target instance was supplied.'); + if (options.workingTreeDirty) reasons.push('Target working tree has uncommitted changes; a commit alone does not identify these inputs.'); + if (options.expectedBuildId === undefined) reasons.push('No expected build ID was supplied.'); + if (options.maxAgeHours === undefined) reasons.push('No evidence-age policy was supplied.'); + return { status: reasons.length ? 'unverified' : 'matched', reasons }; +} + +export function assessCheckbox(options: AssessmentOptions) { + hex(options.sourceCommit, 40, 'Target sourceCommit'); + if (!Number.isFinite(options.now.getTime())) throw new Error('Assessment time is invalid.'); + if (options.maxAgeHours !== undefined && (!Number.isFinite(options.maxAgeHours) || options.maxAgeHours <= 0)) { + throw new Error('Maximum evidence age must be a positive finite number.'); + } + if (options.context && !options.report) throw new Error('Execution context requires a native report.'); + if (options.report) hex(options.reportSha256, 64, 'Supplied report content hash'); + if (options.expectedBuildId !== undefined) text(options.expectedBuildId, 'Expected build ID'); + if (options.expectedTargetId !== undefined && !isWin32StorybookTarget(options.expectedTargetId)) + throw new Error('Expected target ID is not a supported Win32 Storybook target.'); + const evidence = provenance(options); + const mappingCurrent = options.contract.mappingCurrent; + const unitEvidence = assessCheckboxUnits({ + input: options.unit, + sourceCommit: options.sourceCommit, + workingTreeDirty: options.workingTreeDirty, + contractDigest: options.contract.digest, + contractMappingCurrent: mappingCurrent, + now: options.now, + maxAgeHours: options.maxAgeHours, + }); + const runFailed = options.report?.status === 'failed' || Boolean(options.report?.error) || Boolean(options.executionFailure); + const observed = options.report?.cases.find((entry) => entry.storyId === storyId && entry.testName === testName); + const eligible = evidence.status === 'matched' && mappingCurrent; + const requirements = options.contract.requirements.map((requirement) => { + const mapping = coverage[requirement.id]; + const mapped = Boolean(mapping?.observed); + let outcome: CheckOutcome = 'not-run'; + if (mapped && observed) { + if (!mappingCurrent || evidence.status === 'stale') outcome = 'stale'; + else if (evidence.status === 'mismatched') outcome = 'mismatched'; + else if (!eligible) outcome = 'unverified'; + else if (observed.status === 'passed' && runFailed) outcome = 'unverified'; + else outcome = observed.status; + } + const check = mapped + ? { + storyId, + testName, + scope: mapping.observed!, + coverage: 'partial' as const, + recordedOutcome: observed?.status ?? 'not-run', + evidenceOutcome: outcome, + reason: observed?.reason, + } + : null; + const unitChecks = unitEvidence.checks.filter(({ requirements }) => requirements.includes(requirement.id)); + const failed = outcome === 'failed' || unitChecks.some(({ evidenceOutcome }) => evidenceOutcome === 'failed'); + const partial = outcome === 'passed' || unitChecks.some(({ evidenceOutcome }) => evidenceOutcome === 'passed'); + return { + ...requirement, + outcome: failed ? 'blocked-by-failure' : partial ? 'partial' : 'insufficient-evidence', + check, + unitChecks, + remainingCoverage: mapping?.remaining ?? 'No reviewed executable mapping exists for this requirement.', + }; + }); + return { + schemaVersion: 2, + component: 'checkbox', + generation: 'agentic', + target: { + endpoint: 'win32', + renderer: 'paper', + host: 'storybook', + sourceCommit: options.sourceCommit, + workingTreeDirty: options.workingTreeDirty, + expectedBuildId: options.expectedBuildId ?? null, + }, + generatedAt: options.now.toISOString(), + advisory: true, + declaredStatus: null, + evidenceSupportedStatus: null, + approvedStatus: null, + policyStatus: 'pending', + assessmentHealth: + !mappingCurrent || evidence.status === 'stale' || unitEvidence.status === 'stale' || unitEvidence.mappingCurrent === false + ? 'stale' + : evidence.status === 'mismatched' || unitEvidence.status === 'mismatched' + ? 'blocked' + : 'complete', + assessmentOutcome: + requirements.some(({ outcome }) => outcome === 'blocked-by-failure') || unitEvidence.outcome === 'failed' + ? 'failed' + : (eligible && runFailed) || unitEvidence.outcome === 'execution-failed' + ? 'execution-failed' + : 'insufficient-evidence', + contract: { + path: `${componentPath}/SPEC.md`, + digest: options.contract.digest, + lifecycle: options.contract.lifecycle, + conformance: options.contract.conformance, + mappingVersion: 'checkbox-win32-v1', + mappingCurrent, + }, + evidence: { + ...evidence, + reportSha256: options.reportSha256 ?? null, + platformManifestDigest: options.report?.platformManifestDigest ?? null, + runStatus: options.report?.status ?? null, + runError: options.executionFailure ?? options.report?.error ?? null, + finishedAt: options.report?.finishedAt ?? null, + format: options.report?.format ?? null, + harness: options.report?.harness ?? null, + context: options.context ?? null, + caseCounts: { + passed: options.report?.cases.filter(({ status }) => status === 'passed').length ?? 0, + failed: options.report?.cases.filter(({ status }) => status === 'failed').length ?? 0, + skipped: options.report?.cases.filter(({ status }) => status === 'skipped').length ?? 0, + }, + }, + unitEvidence, + requirements, + limitations: [ + 'Only the existing named native Checkbox case is mapped; passing it covers parts of CBX-002 and CBX-004, not either full requirement.', + 'Unit evidence is mocked JavaScript execution, not native Win32 UIA, focus, visual, or physical-input qualification.', + 'Evidence requirements follow the claim: unit/type checks can suffice for logic/API/style requirements; native tests are not mandatory duplicates for every requirement.', + 'The macOS focus case and non-Win32 platform tables are recorded but do not map to Win32 SPEC coverage.', + 'Type/build/visual/manual result adapters are not implemented. A Jest pass of a types test would not prove type checking.', + 'Maturity gates and release approvals are not defined by this pilot. No lifecycle status is inferred or changed.', + 'A multi-assertion native case failure blocks its mapped evidence; it does not identify which individual requirement assertion failed.', + ...(!mappingCurrent + ? ['Contract or story inputs changed since mapping review; review the mapping before accepting its evidence.'] + : []), + ...(runFailed + ? ['The enclosing execution failed or reported an error; recorded passes are not accepted as qualifying evidence.'] + : []), + ...(unitEvidence.runFailed ? ['The enclosing unit command failed; its recorded passes cannot qualify.'] : []), + ], + }; +} + +export type ControlAssessment = ReturnType; + +function markdown(value: string): string { + return value.replace(/[\\|`*_[\]<>]/g, '\\$&').replace(/[\r\n]+/g, ' '); +} + +export function formatAssessmentMarkdown(report: ControlAssessment): string { + return [ + '# Checkbox / Win32 advisory assessment', + '', + '**Readiness: not established.** Maturity policy is pending; no stage is assigned.', + '', + `Assessment health: ${report.assessmentHealth}. Outcome: ${report.assessmentOutcome}.`, + `Target source: \`${report.target.sourceCommit}\`. Working tree dirty: ${report.target.workingTreeDirty}.`, + `Native execution provenance: ${report.evidence.status}. Mapping current: ${report.contract.mappingCurrent}.`, + `Recorded native cases across the input report: ${report.evidence.caseCounts.passed} passed, ${report.evidence.caseCounts.failed} failed, ${report.evidence.caseCounts.skipped} skipped.`, + ...(report.evidence.harness + ? [ + `Desktop supervisor: ${report.evidence.harness.status}; retries: ${report.evidence.harness.retries}; flaky: ${report.evidence.harness.flaky}; unsupported: ${report.evidence.harness.unsupported}; incomplete: ${report.evidence.harness.incomplete}; failures: ${report.evidence.harness.failureCount}.`, + ] + : []), + '', + `Unit execution provenance: ${report.unitEvidence.status}. Outcome: ${report.unitEvidence.outcome}. Mapping current: ${report.unitEvidence.mappingCurrent ?? 'not collected'}.`, + `Unit environment: ${report.unitEvidence.environment}.`, + `Recorded unit cases: ${report.unitEvidence.caseCounts.passed} passed, ${report.unitEvidence.caseCounts.failed} failed, ${report.unitEvidence.caseCounts.skipped} skipped.`, + `Recorded unit retries: ${report.unitEvidence.retryCounts.retries}; retry-success cases: ${report.unitEvidence.retryCounts.flaky}. Enclosing package retries: ${report.unitEvidence.packageRetryCounts?.retries ?? 0}.`, + ...(report.unitEvidence.packageSuites !== null + ? [ + `Unit input contained ${report.unitEvidence.packageSuites} package suites; only the Checkbox runtime suite is mapped. Type-suite Jest passes are not type-check evidence.`, + ] + : []), + '', + '| Requirement | Evidence outcome | Recorded native case | Qualifying partial unit cases | Remaining coverage |', + '| --- | --- | --- | --- | --- |', + ...report.requirements.map( + (entry) => + `| ${markdown(entry.id)} | ${entry.outcome} | ${entry.check ? `${entry.check.recordedOutcome} (partial mapping; ${entry.check.evidenceOutcome})` : 'No executable mapping'} | ${entry.unitChecks.filter(({ evidenceOutcome }) => evidenceOutcome === 'passed').length} / ${entry.unitChecks.length} mapped | ${markdown(entry.remainingCoverage)} |`, + ), + '', + '## Execution evidence', + '', + ...report.requirements + .filter(({ check }) => check) + .map( + (entry) => + `- ${entry.id}: ${markdown(entry.check!.scope)} Recorded: ${entry.check!.recordedOutcome}.${entry.check!.reason ? ` Reason: ${markdown(entry.check!.reason!)}` : ''}`, + ), + ...report.evidence.reasons.map((reason) => `- ${markdown(reason)}`), + '', + '## JavaScript unit evidence', + '', + ...report.unitEvidence.reasons.map((reason) => `- ${markdown(reason)}`), + ...(report.unitEvidence.executionFailure ? [`- ${markdown(report.unitEvidence.executionFailure)}`] : []), + '| Test | Win32 requirement mapping | Recorded | Evidence outcome | Retries | Scope |', + '| --- | --- | --- | --- | --- | --- |', + ...report.unitEvidence.checks.map( + (check) => + `| ${markdown(check.fullName)} | ${check.requirements.join(', ') || 'Not mapped to Win32'} | ${check.recordedOutcome} | ${check.evidenceOutcome} | ${check.retries} | ${markdown(check.scope)}${check.reason ? ` ${markdown(check.reason)}` : ''}${check.retryReasons.length ? ` Prior retry failures: ${markdown(check.retryReasons.join('\n'))}` : ''} |`, + ), + ...report.unitEvidence.unmappedCases.map((check) => `- Unmapped recorded unit case: ${markdown(check.fullName)} (${check.status}).`), + '', + '## Limitations', + '', + ...report.limitations.map((limitation) => `- ${markdown(limitation)}`), + '', + ].join('\n'); +} diff --git a/scripts/src/tasks/controlAssessmentCi.test.ts b/scripts/src/tasks/controlAssessmentCi.test.ts new file mode 100644 index 0000000000..486c9aea1c --- /dev/null +++ b/scripts/src/tasks/controlAssessmentCi.test.ts @@ -0,0 +1,1034 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, it } from 'node:test'; +import { fileURLToPath } from 'node:url'; + +import { + assessmentSummaryPath, + assertFresh, + assessCiBundle, + beginCiExecution, + combineCiEvidence, + finishCiExecution, + formatCiSummary, + formatCiSpec, + git, + inputDigest, + isReportOnlyUpdate, + parseBundle, + parseIdentity, + parseReportReference, +} from './controlAssessmentCi.ts'; +import type { CiBundle } from './controlAssessmentCi.ts'; +import { checkboxNativeAssertionPath, readCheckboxContract, reportDigest, win32StorybookTargetId } from './controlAssessment.ts'; +import { assessmentPackagePath, assessmentSpecPath, splitSpecAssessment } from './controlAssessmentSpec.ts'; +import { checkboxJestFixture, componentJestFixture, desktopEvidenceFixture } from './controlAssessmentFixtures.ts'; +import { readCheckboxUnitMappingFrom, unitMappingFiles } from './controlAssessmentUnit.ts'; +import { runJestWithUnitCapture } from './controlAssessmentUnitExecution.ts'; +import { + AssessmentGitHub, + inspectReportUpdate, + preparePublication, + publishAssessment, + verifyReportUpdate, +} from './controlAssessmentGithub.ts'; + +const repositoryRoot = fileURLToPath(new URL('../../../', import.meta.url)); +const contractPath = 'packages/agentic/components/src/components/checkbox'; +const now = new Date('2026-10-21T20:00:03.000Z'); +const directories: string[] = []; +const reportDirectory = ['apps', 'storybook', 'artifacts', 'win32', 'assessment-ci']; +const desktopDirectory = ['apps', 'storybook', 'artifacts', 'desktop']; +const nativePath = [...desktopDirectory, 'synthetic-desktop-run', 'run.json']; + +function put(root: string, file: string, content: string): void { + const target = path.join(root, file); + mkdirSync(path.dirname(target), { recursive: true }); + writeFileSync(target, content); +} + +function commit(root: string, message: string): string { + git(root, ['add', '.']); + git(root, ['commit', '-qm', message]); + return git(root, ['rev-parse', 'HEAD']); +} + +function merge(root: string, base: string, head: string): string { + git(root, ['checkout', '-q', '--detach', base]); + git(root, ['merge', '--no-ff', '-qm', 'Synthetic PR merge', head]); + return git(root, ['rev-parse', 'HEAD']); +} + +function fixture() { + const directory = mkdtempSync(path.join(tmpdir(), 'furn-ci-pilot-')); + directories.push(directory); + const root = path.join(directory, 'repo'); + mkdirSync(root); + mkdirSync(path.join(root, 'apps', 'storybook'), { recursive: true }); + git(root, ['init', '-q', '-b', 'main']); + git(root, ['config', 'user.name', 'Fixture']); + git(root, ['config', 'user.email', 'fixture@example.invalid']); + git(root, ['config', 'core.autocrlf', 'false']); + git(root, ['config', 'commit.gpgsign', 'false']); + put(root, '.gitignore', 'apps/storybook/artifacts/\n'); + const originalManifest = JSON.parse(readFileSync(path.join(repositoryRoot, assessmentPackagePath), 'utf8')); + put( + root, + assessmentPackagePath, + JSON.stringify({ + name: originalManifest.name, + version: '0.7.0', + furn: { jestPlatform: originalManifest.furn.jestPlatform }, + scripts: { test: originalManifest.scripts.test }, + }), + ); + for (const file of unitMappingFiles) put(root, file, readFileSync(path.join(repositoryRoot, file), 'utf8')); + put(root, checkboxNativeAssertionPath, readFileSync(path.join(repositoryRoot, checkboxNativeAssertionPath), 'utf8')); + for (const file of [ + 'SPEC.md', + 'spec/source.json', + 'spec/interaction.md', + 'spec/accessibility.md', + 'spec/tokens.yaml', + 'spec/usage.md', + 'checkbox.stories.tsx', + ]) { + const target = path.join(root, contractPath, file); + mkdirSync(path.dirname(target), { recursive: true }); + if (file === 'SPEC.md') { + writeFileSync(target, splitSpecAssessment(readFileSync(path.join(repositoryRoot, contractPath, file), 'utf8')).contract); + } else { + copyFileSync(path.join(repositoryRoot, contractPath, file), target); + } + } + put(root, 'source.ts', 'export const value = 0;\n'); + const baseCommit = commit(root, 'Synthetic base'); + git(root, ['checkout', '-qb', 'pilot/checkbox']); + put(root, 'source.ts', 'export const value = 1;\n'); + const headCommit = commit(root, 'Synthetic control change'); + const testedCommit = merge(root, baseCommit, headCommit); + const event = path.join(directory, 'event.json'); + const payload = (head = headCommit, base = baseCommit) => ({ + pull_request: { + number: 123, + head: { sha: head, ref: 'pilot/checkbox', repo: { full_name: 'example/furn' } }, + base: { sha: base, ref: 'main', repo: { full_name: 'example/furn' } }, + }, + }); + writeFileSync(event, JSON.stringify(payload())); + const env: NodeJS.ProcessEnv = { + GITHUB_EVENT_PATH: event, + GITHUB_REPOSITORY: 'example/furn', + GITHUB_SHA: testedCommit, + GITHUB_RUN_ID: '456', + GITHUB_RUN_ATTEMPT: '1', + FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES: 'keyboard,physicalClick,wheel', + SMOKE_OUTCOME: 'success', + }; + return { directory, root, baseCommit, headCommit, testedCommit, event, payload, env }; +} +type Fixture = ReturnType; + +function nativeReport(overrides: Record = {}): string { + return JSON.stringify({ + schemaVersion: 1, + endpoint: 'win32', + platformName: 'windows', + targetId: 'agenticstorybook-win32', + runId: 'synthetic-native-run', + startedAt: '2026-10-21T20:00:01.000Z', + finishedAt: '2026-10-21T20:00:02.000Z', + manifest: { platform: 'b'.repeat(64), portable: 'c'.repeat(64) }, + status: 'passed', + tests: [], + wdio: [ + { + storyId: 'components-checkbox--default', + testName: 'toggles native checked state in both directions', + status: 'skipped', + durationMs: 0, + skipReason: 'This test requires physical pointer input.', + }, + ], + ...overrides, + }); +} + +function execute(f: Fixture): CiBundle { + rmSync(path.join(f.root, ...desktopDirectory, 'synthetic-desktop-run'), { recursive: true, force: true }); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putNative(f.root, nativeReport()); + return finishCiExecution(f.root, f.env, now); +} + +function putNative(root: string, smoke: string): void { + mkdirSync(path.join(root, 'apps', 'storybook'), { recursive: true }); + const capture = path.join(root, ...reportDirectory, 'execution-start.json'); + const targetId = existsSync(capture) ? JSON.parse(readFileSync(capture, 'utf8')).targetId : win32StorybookTargetId(root); + const boundSmoke = JSON.stringify({ ...JSON.parse(smoke), targetId }); + put(root, path.join(...nativePath), desktopEvidenceFixture(boundSmoke).canonical); + put(root, path.join(...desktopDirectory, 'synthetic-desktop-run', 'evidence', 'win32', 'static', 'run.json'), boundSmoke); +} + +function putUnit(f: Fixture, modify: (capture: Record) => void = () => {}): void { + const started = JSON.parse(readFileSync(path.join(f.root, ...reportDirectory, 'execution-start.json'), 'utf8')); + const report = JSON.stringify(checkboxJestFixture()); + const mapping = readCheckboxUnitMappingFrom((file) => readFileSync(path.join(f.root, file), 'utf8')); + const capture = { + schemaVersion: 1, + identity: started.identity, + exitCode: 0, + report, + executionFailure: null, + collectionStartedAt: '2026-10-21T20:00:00.000Z', + context: { + schemaVersion: 1, + platform: 'macos', + sourceCommit: f.testedCommit, + workingTreeDirty: false, + buildId: 'example/furn/actions/runs/456/attempts/1', + contractDigest: readCheckboxContract(f.root).digest, + mappingDigest: mapping.digest, + reportSha256: reportDigest(report), + startedAt: '2026-10-21T20:00:00.100Z', + finishedAt: '2026-10-21T20:00:01.500Z', + exitCode: 0, + }, + }; + modify(capture); + put(f.root, path.join(...reportDirectory, 'unit-execution.json'), JSON.stringify(capture)); + f.env.UNIT_OUTCOME = 'success'; +} + +function reportCommit( + f: Fixture, + summary: string, + spec = readFileSync(path.join(f.root, ...reportDirectory, 'SPEC.md'), 'utf8'), +): { head: string; merged: string } { + git(f.root, ['checkout', '-q', 'pilot/checkbox']); + put(f.root, assessmentSummaryPath, summary); + put(f.root, assessmentSpecPath, spec); + const head = commit(f.root, 'Synthetic generated report'); + const merged = merge(f.root, f.baseCommit, head); + writeFileSync(f.event, JSON.stringify(f.payload(head, f.baseCommit))); + f.env.GITHUB_SHA = merged; + return { head, merged }; +} + +function fakeApi(f: Fixture, overrides: Record = {}) { + const calls: { url: string; options?: RequestInit }[] = []; + const fetchImplementation: typeof fetch = async (input, options) => { + const url = String(input); + calls.push({ url, options }); + const route = url.slice('https://api.github.com'.length); + const defaults: Record = { + '/repos/example/furn/actions/runs/456/attempts/1': { + id: 456, + run_attempt: 1, + event: 'pull_request', + status: 'completed', + path: '.github/workflows/pr.yml', + head_sha: f.headCommit, + head_branch: 'pilot/checkbox', + head_repository: { full_name: 'example/furn' }, + }, + '/repos/example/furn/actions/runs/456/attempts/1/jobs?per_page=100&page=1': { + total_count: 1, + jobs: [{ name: 'PR source validation', conclusion: 'success' }], + }, + '/repos/example/furn/actions/runs/456/artifacts?per_page=100&page=1': { + total_count: 1, + artifacts: [{ name: 'checkbox-win32-assessment-1', expired: false }], + }, + '/repos/example/furn/pulls/123': { + number: 123, + state: 'open', + base: { sha: f.baseCommit, ref: 'main', repo: { full_name: 'example/furn' } }, + head: { sha: f.headCommit, ref: 'pilot/checkbox', repo: { full_name: 'example/furn' } }, + merge_commit_sha: f.testedCommit, + }, + '/graphql': { data: { createCommitOnBranch: { commit: { oid: 'd'.repeat(40) } } } }, + }; + const value = route in overrides ? overrides[route] : defaults[route]; + assert.notEqual(value, undefined, `Unexpected API request: ${route}`); + return new Response(JSON.stringify(value), { status: 200 }); + }; + return { github: new AssessmentGitHub({ token: 'synthetic-token', fetchImplementation }), calls }; +} + +afterEach(() => { + for (const directory of directories.splice(0)) rmSync(directory, { recursive: true, force: true }); +}); + +describe('CI execution binding', () => { + it('assesses existing units when native execution is unavailable and rejects a different native attempt', () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putUnit(f); + put( + f.root, + path.join('apps', 'storybook', 'artifacts', 'assessment-unit-ci', 'unit-execution.json'), + readFileSync(path.join(f.root, ...reportDirectory, 'unit-execution.json'), 'utf8'), + ); + f.env.NATIVE_OUTCOME = 'failure'; + const bundle = combineCiEvidence(f.root, f.env, now); + const report = assessCiBundle(f.root, bundle); + assert.equal(report.evidence.status, 'missing'); + assert.equal(report.unitEvidence.status, 'matched'); + assert.equal(report.requirements[0].outcome, 'partial'); + assert.equal(splitSpecAssessment(formatCiSpec(f.root, bundle)).assessment?.testedCommit, null); + put( + f.root, + path.join(...reportDirectory, 'bundle.json'), + JSON.stringify({ ...bundle, identity: { ...bundle.identity, runAttempt: 2 } }), + ); + assert.throws(() => combineCiEvidence(f.root, f.env, now), /different CI inputs/); + }); + + for (const mode of ['passed', 'failed'] as const) { + it(`reuses exactly one existing package invocation and retains its ${mode} exit status across jobs`, async () => { + const f = fixture(); + const command = path.join(repositoryRoot, 'scripts', 'src', 'tasks', 'controlAssessmentCiTask.ts'); + const run = (argument: string) => + spawnSync(process.execPath, [command, argument], { + cwd: f.root, + env: { ...process.env, ...f.env }, + encoding: 'utf8', + }); + const begin = run('begin-unit'); + assert.equal(begin.status, 0, begin.stderr || begin.stdout); + const unitDirectory = path.join(f.root, 'apps', 'storybook', 'artifacts', 'assessment-unit-ci'); + let invocations = 0; + const originalArgs = ['--runTestsByPath', 'checkbox.test.tsx', 'checkbox.types.test.ts']; + const unit = await runJestWithUnitCapture( + path.join(f.root, 'packages', 'agentic', 'components'), + 'macos', + originalArgs, + async (args) => { + invocations++; + assert.deepEqual(args.slice(0, originalArgs.length), originalArgs); + const data = componentJestFixture(); + const time = Date.now(); + data.startTime = time; + for (const suite of data.testResults) Object.assign(suite, { startTime: time, endTime: time }); + const outputIndex = args.indexOf('--outputFile'); + assert.ok(outputIndex >= 0, 'Existing invocation must receive its capture output path.'); + writeFileSync(args[outputIndex + 1], JSON.stringify(data)); + return mode === 'passed' ? 0 : 1; + }, + { ...f.env, FURN_ASSESSMENT_UNIT_CAPTURE: unitDirectory }, + ); + assert.equal(unit, mode === 'passed' ? 0 : 1); + assert.equal(invocations, 1); + // Another workspace may fail the aggregate without invalidating this successful component command. + f.env.UNIT_OUTCOME = 'failure'; + const nativeBegin = run('begin'); + assert.equal(nativeBegin.status, 0, nativeBegin.stderr || nativeBegin.stdout); + const time = new Date().toISOString(); + putNative(f.root, nativeReport({ startedAt: time, finishedAt: time })); + const finish = run('finish'); + assert.equal(finish.status, 0, finish.stderr || finish.stdout); + const combined = run('combine'); + assert.equal(combined.status, 0, combined.stderr || combined.stdout); + const report = JSON.parse(readFileSync(path.join(f.root, ...reportDirectory, 'assessment.json'), 'utf8')); + assert.equal(report.unitEvidence.status, 'matched'); + assert.equal(report.unitEvidence.caseCounts.passed, 42); + assert.equal(report.unitEvidence.packageSuites, 2); + assert.equal(report.unitEvidence.packageCounts.passed, 43); + assert.equal(report.unitEvidence.outcome, mode === 'passed' ? 'passed' : 'execution-failed'); + assert.equal(invocations, 1, 'Assessment must never invoke Jest again.'); + }); + } + + it('collects bound unit cases beside a native skip without claiming native qualification', () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putUnit(f); + putNative(f.root, nativeReport()); + const bundle = finishCiExecution(f.root, f.env, now); + const report = assessCiBundle(f.root, bundle); + assert.equal(bundle.schemaVersion, 2); + assert.equal(report.unitEvidence.status, 'matched'); + assert.equal(report.unitEvidence.caseCounts.passed, 42); + assert.equal(report.requirements[1].check?.evidenceOutcome, 'skipped'); + assert.equal(report.requirements[1].outcome, 'partial'); + assert.equal(report.requirements[5].outcome, 'insufficient-evidence'); + assert.equal(report.assessmentOutcome, 'insufficient-evidence'); + assert.equal(report.evidenceSupportedStatus, null); + assert.match(formatCiSummary(f.root, bundle), /Recorded unit cases: 42 passed/); + assert.equal(splitSpecAssessment(formatCiSpec(f.root, bundle)).assessment?.testedCommit, f.testedCommit); + const context = bundle.unit!.context!; + for (const change of [ + { sourceCommit: f.headCommit }, + { reportSha256: 'f'.repeat(64) }, + { buildId: 'example/furn/actions/runs/456/attempts/2' }, + { finishedAt: '2026-10-22T20:00:00.000Z' }, + { workingTreeDirty: true }, + ]) { + assert.throws( + () => assessCiBundle(f.root, { ...bundle, unit: { ...bundle.unit!, context: { ...context, ...change } } }), + /binding|dirty/, + ); + } + }); + + it('preserves unit step failure instead of accepting passing JSON from a failed parent command', () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putUnit(f); + putNative(f.root, nativeReport()); + f.env.UNIT_OUTCOME = 'failure'; + const report = assessCiBundle(f.root, finishCiExecution(f.root, f.env, now)); + assert.equal(report.unitEvidence.outcome, 'execution-failed'); + assert.equal(report.assessmentOutcome, 'execution-failed'); + assert.equal(report.requirements[1].outcome, 'insufficient-evidence'); + }); + + it('keeps absent/skipped unit collection unavailable and accepts old native-only bundles explicitly', () => { + const f = fixture(); + f.env.UNIT_OUTCOME = 'skipped'; + const bundle = execute(f); + const report = assessCiBundle(f.root, bundle); + assert.equal(report.unitEvidence.status, 'missing'); + assert.match(report.unitEvidence.executionFailure ?? '', /skipped/); + const { unit, ...nativeOnly } = bundle; + assert.ok(unit); + const legacy = parseBundle(JSON.stringify({ ...nativeOnly, schemaVersion: 1 })); + assert.equal(legacy.unit, undefined); + assert.equal(assessCiBundle(f.root, legacy).unitEvidence.status, 'missing'); + assert.throws(() => parseBundle(JSON.stringify({ ...bundle, schemaVersion: 1 })), /Legacy/); + assert.throws( + () => parseBundle(JSON.stringify({ ...bundle, unit: { report: '{}', context: null, executionFailure: null } })), + /together/, + ); + }); + + it('rejects stale unit captures from a previous interval or workflow attempt', () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putNative(f.root, nativeReport()); + putUnit(f, (capture) => { + (capture.context as Record).startedAt = '2026-10-20T20:00:00.000Z'; + }); + assert.throws(() => finishCiExecution(f.root, f.env, now), /predates/); + putUnit(f, (capture) => { + (capture.identity as Record).runAttempt = 2; + }); + assert.throws(() => finishCiExecution(f.root, f.env, now), /different CI/); + }); + + it('captures before execution, retains an honest hosted skip, and writes usable reports', () => { + const f = fixture(); + const bundle = execute(f); + const report = assessCiBundle(f.root, bundle); + assert.equal(bundle.identity.testedCommit, f.testedCommit); + assert.equal(bundle.identity.headCommit, f.headCommit); + assert.equal(bundle.context?.physicalClick, false); + assert.equal(report.evidence.status, 'matched'); + assert.equal(report.requirements[1].check?.recordedOutcome, 'skipped'); + assert.equal(report.requirements[1].check?.evidenceOutcome, 'skipped'); + assert.equal(report.evidenceSupportedStatus, null); + assert.equal(report.assessmentOutcome, 'insufficient-evidence'); + const markdown = readFileSync(path.join(f.root, ...reportDirectory, 'ASSESSMENT.md'), 'utf8'); + assert.match(markdown, /AUTO-GENERATED\. DO NOT EDIT/); + assert.match(markdown, /physical pointer input/); + assert.equal(parseReportReference(markdown).inputDigest, inputDigest(f.root, f.testedCommit)); + assert.deepEqual(parseBundle(readFileSync(path.join(f.root, ...reportDirectory, 'bundle.json'), 'utf8')), bundle); + }); + + it('never binds an old report when pre-execution capture did not run', () => { + const f = fixture(); + putNative(f.root, nativeReport()); + const bundle = finishCiExecution(f.root, { ...f.env, SMOKE_OUTCOME: 'skipped' }, now); + assert.equal(bundle.nativeReport, null); + assert.equal(bundle.context, null); + assert.match(bundle.executionFailure ?? '', /skipped/); + assert.equal(assessCiBundle(f.root, bundle).evidence.status, 'missing'); + }); + + it('preserves but never binds desktop runs that existed before capture', () => { + const f = fixture(); + putNative(f.root, nativeReport()); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + const bundle = finishCiExecution(f.root, f.env, now); + assert.equal(bundle.nativeReport, null); + assert.match(bundle.executionFailure ?? '', /did not produce/); + assert.equal(JSON.parse(readFileSync(path.join(f.root, ...nativePath), 'utf8')).runId, 'synthetic-desktop-run'); + }); + + it('rejects multiple new desktop runs rather than choosing the newest report', () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putNative(f.root, nativeReport()); + put(f.root, path.join(...desktopDirectory, 'another-run', 'run.json'), desktopEvidenceFixture(nativeReport()).canonical); + assert.throws(() => finishCiExecution(f.root, f.env, now), /Ambiguous/); + }); + + it('rejects a canonical run ID that differs from its captured artifact directory', () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putNative(f.root, nativeReport()); + put(f.root, path.join(...nativePath), desktopEvidenceFixture(nativeReport(), { runId: 'different-run' }).canonical); + assert.throws(() => finishCiExecution(f.root, f.env, now), /artifact directory/); + }); + + it('rejects another generated target instance instead of accepting the Win32 prefix alone', () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putNative(f.root, nativeReport()); + put( + f.root, + path.join(...desktopDirectory, 'synthetic-desktop-run', 'evidence', 'win32', 'static', 'run.json'), + nativeReport({ targetId: `win32-${'0'.repeat(16)}` }), + ); + assert.throws(() => finishCiExecution(f.root, f.env, now), /different native instance/); + }); + + for (const missing of ['undeclared companion', 'unwritten companion', 'unfinished canonical run'] as const) { + it(`reports unavailable evidence for ${missing}`, () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + if (missing === 'unfinished canonical run') { + put(f.root, path.join(...desktopDirectory, 'synthetic-desktop-run', 'events.jsonl'), '{}\n'); + } else { + put( + f.root, + path.join(...nativePath), + desktopEvidenceFixture(nativeReport(), missing === 'undeclared companion' ? { artifacts: [] } : {}).canonical, + ); + } + const bundle = finishCiExecution(f.root, f.env, now); + assert.equal(bundle.nativeReport, null); + assert.equal(bundle.context, null); + assert.match(bundle.executionFailure ?? '', /evidence|report/); + assert.equal(assessCiBundle(f.root, bundle).assessmentOutcome, 'insufficient-evidence'); + assert.match(assessCiBundle(f.root, bundle).evidence.runError ?? '', /evidence|report/); + }); + } + + it('preserves a canonical cleanup failure in the generated assessment', () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putNative(f.root, nativeReport()); + put( + f.root, + path.join(...nativePath), + desktopEvidenceFixture(nativeReport(), { + status: 'failed', + failures: [{ failureId: 'cleanup', kind: 'cleanup', error: { name: 'Error', message: '' } }], + }).canonical, + ); + const bundle = finishCiExecution(f.root, f.env, now); + assert.equal(assessCiBundle(f.root, bundle).assessmentOutcome, 'execution-failed'); + assert.match(assessCiBundle(f.root, bundle).evidence.runError ?? '', /Desktop supervisor status: failed/); + assert.match(formatCiSummary(f.root, bundle), /Desktop supervisor: failed/); + }); + + it('does not accept passes from a smoke step that failed after writing its report', () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putNative(f.root, nativeReport()); + const bundle = finishCiExecution(f.root, { ...f.env, SMOKE_OUTCOME: 'failure' }, now); + assert.equal(assessCiBundle(f.root, bundle).assessmentOutcome, 'execution-failed'); + assert.match(assessCiBundle(f.root, bundle).evidence.runError ?? '', /failure/); + }); + + it('rejects dirty tracked and nonignored inputs instead of inventing a clean execution', () => { + const f = fixture(); + put(f.root, 'source.ts', 'dirty\n'); + assert.throws(() => beginCiExecution(f.root, f.env, now), /inputs changed/); + put(f.root, 'source.ts', 'export const value = 1;\n'); + put(f.root, 'extra.ts', 'untracked\n'); + assert.throws(() => finishCiExecution(f.root, f.env, now), /inputs changed/); + }); + + it('requires hosted capability disabling and the real merged checkout', () => { + const f = fixture(); + assert.throws(() => beginCiExecution(f.root, { ...f.env, FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES: '' }, now), /physicalClick/); + assert.throws(() => beginCiExecution(f.root, { ...f.env, GITHUB_SHA: f.headCommit }, now), /checkout/); + const bundle = execute(f); + assert.throws( + () => assessCiBundle(f.root, { ...bundle, identity: { ...bundle.identity, testedCommit: f.headCommit } }), + /merge commit/, + ); + }); + + it('rejects old report timestamps, changed source trees, and incorrect report hashes', () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putNative(f.root, nativeReport({ startedAt: '2026-10-21T19:00:00.000Z' })); + assert.throws(() => finishCiExecution(f.root, f.env, now), /predates/); + const bundle = execute(f); + assert.throws(() => assessCiBundle(f.root, { ...bundle, identity: { ...bundle.identity, inputDigest: 'a'.repeat(64) } }), /digest/); + const context = bundle.context; + assert.ok(context); + assert.throws(() => assessCiBundle(f.root, { ...bundle, context: { ...context, reportSha256: 'a'.repeat(64) } }), /binding/); + }); + + it('rejects a physical pass on the explicitly noninteractive hosted runner', () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putNative( + f.root, + nativeReport({ + wdio: [ + { + storyId: 'components-checkbox--default', + testName: 'toggles native checked state in both directions', + status: 'passed', + durationMs: 10, + }, + ], + }), + ); + assert.throws(() => finishCiExecution(f.root, f.env, now), /physical-click/); + }); + + it('runs the actual CI entry point and persists files without dependencies or network writes', () => { + const f = fixture(); + const command = path.join(repositoryRoot, 'scripts', 'src', 'tasks', 'controlAssessmentCiTask.ts'); + const run = (argument: string) => + spawnSync(process.execPath, [command, argument], { + cwd: f.root, + env: { ...process.env, ...f.env }, + encoding: 'utf8', + }); + const begin = run('begin'); + assert.equal(begin.status, 0, begin.stderr || begin.stdout); + const time = new Date().toISOString(); + putNative(f.root, nativeReport({ startedAt: time, finishedAt: time })); + const finish = run('finish'); + assert.equal(finish.status, 0, finish.stderr || finish.stdout); + assert.match(finish.stdout, /No maturity stage assigned/); + assert.equal(parseBundle(readFileSync(path.join(f.root, ...reportDirectory, 'bundle.json'), 'utf8')).identity.headCommit, f.headCommit); + }); +}); + +describe('same-PR source fingerprint and evidence reuse', () => { + it('verifies unit evidence through the same exact two-file publication and follow-up path', async () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putUnit(f); + putNative(f.root, nativeReport()); + const bundle = finishCiExecution(f.root, f.env, now); + const summary = formatCiSummary(f.root, bundle); + const update = reportCommit(f, summary); + const { github } = fakeApi(f); + const candidate = await inspectReportUpdate(f.root, f.env, github, now); + assert.equal(candidate.reportOnly, true); + verifyReportUpdate(f.root, candidate, bundle, now); + assert.equal(inputDigest(f.root, update.merged), bundle.identity.inputDigest); + const tampered = { + ...bundle, + unit: { ...bundle.unit!, report: bundle.unit!.report!.replace('"numPassedTests":42', '"numPassedTests":41') }, + }; + assert.throws(() => verifyReportUpdate(f.root, candidate, tampered, now), /counts|binding/); + }); + + it('preserves the legacy raw tree digest before generated SPEC metadata exists', () => { + const f = fixture(); + const raw = git(f.root, ['ls-tree', '-rz', '--full-tree', f.testedCommit]); + assert.equal(inputDigest(f.root, f.testedCommit), createHash('sha256').update(raw).digest('hex')); + }); + + it('verifies the report-only follow-up without relabeling its commit as tested', async () => { + const f = fixture(); + const bundle = execute(f); + const summary = formatCiSummary(f.root, bundle); + const update = reportCommit(f, summary); + assert.equal(inputDigest(f.root, update.merged), bundle.identity.inputDigest); + assert.equal(isReportOnlyUpdate(f.root, parseReportReference(summary), update.head, update.merged), true); + const { github } = fakeApi(f); + const candidate = await inspectReportUpdate(f.root, f.env, github, now); + assert.equal(candidate.reportOnly, true); + verifyReportUpdate(f.root, candidate, bundle, now); + assert.notEqual(bundle.identity.testedCommit, update.merged); + assert.notEqual(bundle.identity.headCommit, update.head); + }); + + for (const file of [ + 'source.ts', + 'yarn.lock', + '.github/workflows/pr.yml', + `${contractPath}/SPEC.md`, + `${contractPath}/checkbox.test.tsx`, + `${contractPath}/assessment/not-a-generated-report.ts`, + assessmentPackagePath, + ]) { + it(`does not exclude source input ${file}`, () => { + const f = fixture(); + const bundle = execute(f); + git(f.root, ['checkout', '-q', 'pilot/checkbox']); + put(f.root, file, `${readFileIfPresent(f.root, file)}\n// changed input\n`); + put(f.root, assessmentSummaryPath, formatCiSummary(f.root, bundle)); + const changed = commit(f.root, 'Synthetic source and report changes'); + const merged = merge(f.root, f.baseCommit, changed); + assert.notEqual(inputDigest(f.root, merged), bundle.identity.inputDigest); + assert.equal(isReportOnlyUpdate(f.root, { ...bundle.identity, generatedAt: bundle.generatedAt }, changed, merged), false); + }); + } + + it('records the tested package version and merge SHA, never the publication SHA', () => { + const f = fixture(); + const bundle = execute(f); + const spec = formatCiSpec(f.root, bundle); + const metadata = splitSpecAssessment(spec).assessment; + assert.equal(metadata?.packageVersion, '0.7.0'); + assert.equal(metadata?.sourceCommit, f.testedCommit); + assert.equal(metadata?.testedCommit, f.testedCommit); + assert.equal(metadata?.dirty, false); + assert.equal(metadata?.provenance, 'matched'); + const update = reportCommit(f, formatCiSummary(f.root, bundle)); + assert.notEqual(metadata?.testedCommit, update.head); + assert.notEqual(metadata?.testedCommit, update.merged); + }); + + it('rejects altered generated SPEC fields even though their structural input digest is unchanged', async () => { + const f = fixture(); + const bundle = execute(f); + const spec = formatCiSpec(f.root, bundle).replace("assessment-package-version: '0.7.0'", "assessment-package-version: '99.0.0'"); + const update = reportCommit(f, formatCiSummary(f.root, bundle), spec); + assert.equal(inputDigest(f.root, update.merged), bundle.identity.inputDigest); + const candidate = await inspectReportUpdate(f.root, f.env, fakeApi(f).github, now); + assert.ok(candidate.reportOnly); + assert.throws(() => verifyReportUpdate(f.root, candidate, bundle, now), /SPEC.*trusted rendering/); + }); + + it('falls back to full CI for an old summary-only publication without a SPEC update', async () => { + const f = fixture(); + const bundle = execute(f); + git(f.root, ['checkout', '-q', 'pilot/checkbox']); + put(f.root, assessmentSummaryPath, formatCiSummary(f.root, bundle)); + const head = commit(f.root, 'Legacy summary-only update'); + f.env.GITHUB_SHA = merge(f.root, f.baseCommit, head); + writeFileSync(f.event, JSON.stringify(f.payload(head))); + assert.equal((await inspectReportUpdate(f.root, f.env, fakeApi(f).github, now)).reportOnly, false); + }); + + it('does not normalize away executable mode or human contract changes in a two-file publication', () => { + for (const mode of [false, true]) { + const f = fixture(); + const bundle = execute(f); + git(f.root, ['checkout', '-q', 'pilot/checkbox']); + put(f.root, assessmentSummaryPath, formatCiSummary(f.root, bundle)); + put(f.root, assessmentSpecPath, `${formatCiSpec(f.root, bundle)}${mode ? '' : '\nHuman contract changed.\n'}`); + git(f.root, ['add', '.']); + if (mode) { + chmodSync(path.join(f.root, assessmentSpecPath), 0o755); + git(f.root, ['update-index', '--chmod=+x', assessmentSpecPath]); + } + git(f.root, ['commit', '-qm', 'Changed actual SPEC inputs']); + const head = git(f.root, ['rev-parse', 'HEAD']); + if (mode) assert.match(git(f.root, ['ls-tree', head, '--', assessmentSpecPath]), /^100755 blob /); + const merged = merge(f.root, f.baseCommit, head); + assert.notEqual(inputDigest(f.root, merged), bundle.identity.inputDigest); + assert.equal(isReportOnlyUpdate(f.root, { ...bundle.identity, generatedAt: bundle.generatedAt }, head, merged), false); + } + }); + + it('renders from original PR head contract bytes but takes the version and assessment from the tested merge', async () => { + const f = fixture(); + const headSpec = readFileSync(path.join(f.root, assessmentSpecPath), 'utf8'); + git(f.root, ['checkout', '-q', 'main']); + put(f.root, assessmentSpecPath, `${headSpec}\nBase-only contract note.\n`); + put(f.root, assessmentPackagePath, JSON.stringify({ version: '0.8.0' })); + f.baseCommit = commit(f.root, 'New main contract and version'); + f.testedCommit = merge(f.root, f.baseCommit, f.headCommit); + f.env.GITHUB_SHA = f.testedCommit; + writeFileSync(f.event, JSON.stringify(f.payload(f.headCommit, f.baseCommit))); + const bundle = execute(f); + const spec = formatCiSpec(f.root, bundle); + assert.equal(splitSpecAssessment(spec).contract, headSpec); + assert.equal(splitSpecAssessment(spec).assessment?.packageVersion, '0.8.0'); + assert.equal(splitSpecAssessment(spec).assessment?.health, 'stale'); + const update = reportCommit(f, formatCiSummary(f.root, bundle)); + assert.equal(inputDigest(f.root, update.merged), bundle.identity.inputDigest); + const candidate = await inspectReportUpdate(f.root, f.env, fakeApi(f).github, now); + verifyReportUpdate(f.root, candidate, bundle, now); + }); + + it('rejects non-UTF-8 SPEC bytes instead of folding distinct source bytes together', () => { + const f = fixture(); + const bundle = execute(f); + git(f.root, ['checkout', '-q', 'pilot/checkbox']); + writeFileSync(path.join(f.root, assessmentSpecPath), Buffer.concat([Buffer.from(formatCiSpec(f.root, bundle)), Buffer.from([0xff])])); + const head = commit(f.root, 'Invalid SPEC encoding'); + assert.throws(() => inputDigest(f.root, head), /valid UTF-8/); + }); + + it('detects changes in earlier commits, not just the last report commit', () => { + const f = fixture(); + const bundle = execute(f); + git(f.root, ['checkout', '-q', 'pilot/checkbox']); + put(f.root, 'source.ts', 'changed after testing\n'); + commit(f.root, 'Synthetic earlier code commit'); + put(f.root, assessmentSummaryPath, formatCiSummary(f.root, bundle)); + const head = commit(f.root, 'Synthetic latest report-only commit'); + const merged = merge(f.root, f.baseCommit, head); + assert.equal(isReportOnlyUpdate(f.root, { ...bundle.identity, generatedAt: bundle.generatedAt }, head, merged), false); + }); + + it('invalidates reuse when another control PR changes the base branch', () => { + const f = fixture(); + const bundle = execute(f); + const update = reportCommit(f, formatCiSummary(f.root, bundle)); + git(f.root, ['checkout', '-q', 'main']); + put(f.root, 'other-control.ts', 'another merged change\n'); + const newBase = commit(f.root, 'Synthetic concurrent PR'); + const merged = merge(f.root, newBase, update.head); + assert.equal(isReportOnlyUpdate(f.root, { ...bundle.identity, generatedAt: bundle.generatedAt }, update.head, merged), false); + }); + + it('recovers from concurrent generated-report conflicts by resolving source and reassessing the combined tree', () => { + const f = fixture(); + const firstBundle = execute(f); + const first = reportCommit(f, formatCiSummary(f.root, firstBundle)); + git(f.root, ['checkout', '-q', 'main']); + git(f.root, ['merge', '--ff-only', first.head]); + git(f.root, ['checkout', '-qb', 'pilot/second', f.baseCommit]); + put(f.root, 'other-control.ts', 'second PR source\n'); + const secondHead = commit(f.root, 'Synthetic simultaneous control PR'); + const secondMerge = merge(f.root, f.baseCommit, secondHead); + const reassess = (head: string, base: string, tested: string) => { + const event = { + pull_request: { + number: 124, + head: { sha: head, ref: 'pilot/second', repo: { full_name: 'example/furn' } }, + base: { sha: base, ref: 'main', repo: { full_name: 'example/furn' } }, + }, + }; + writeFileSync(f.event, JSON.stringify(event)); + const env = { ...f.env, GITHUB_RUN_ID: '789', GITHUB_SHA: tested }; + rmSync(path.join(f.root, ...desktopDirectory, 'synthetic-desktop-run'), { recursive: true, force: true }); + beginCiExecution(f.root, env, new Date('2026-10-21T20:00:00.000Z')); + putNative(f.root, nativeReport()); + return finishCiExecution(f.root, env, now); + }; + const secondBundle = reassess(secondHead, f.baseCommit, secondMerge); + git(f.root, ['checkout', '-q', 'pilot/second']); + put(f.root, assessmentSummaryPath, formatCiSummary(f.root, secondBundle)); + put(f.root, assessmentSpecPath, formatCiSpec(f.root, secondBundle)); + commit(f.root, 'Synthetic second generated report'); + const conflict = spawnSync('git', ['merge', '--no-ff', '--no-edit', 'main'], { cwd: f.root, encoding: 'utf8' }); + assert.equal(conflict.status, 1, conflict.stdout || conflict.stderr); + assert.deepEqual( + git(f.root, ['diff', '--name-only', '--diff-filter=U']).split('\n').sort(), + [assessmentSummaryPath, assessmentSpecPath].sort(), + ); + put(f.root, assessmentSummaryPath, formatCiSummary(f.root, firstBundle)); + put(f.root, assessmentSpecPath, formatCiSpec(f.root, firstBundle)); + const updatedHead = commit(f.root, 'Resolve generated report with main snapshot temporarily'); + const updatedMerge = merge(f.root, first.head, updatedHead); + const refreshed = reassess(updatedHead, first.head, updatedMerge); + assert.notEqual(refreshed.identity.inputDigest, secondBundle.identity.inputDigest); + assert.equal(refreshed.identity.baseCommit, first.head); + git(f.root, ['checkout', '-q', 'pilot/second']); + put(f.root, assessmentSummaryPath, formatCiSummary(f.root, refreshed)); + put(f.root, assessmentSpecPath, formatCiSpec(f.root, refreshed)); + const reportHead = commit(f.root, 'Regenerate second PR assessment after integration'); + const reportMerge = merge(f.root, first.head, reportHead); + assert.equal(isReportOnlyUpdate(f.root, { ...refreshed.identity, generatedAt: refreshed.generatedAt }, reportHead, reportMerge), true); + }); + + it('rejects a manually altered report body even when the embedded source reference is valid', async () => { + const f = fixture(); + const bundle = execute(f); + reportCommit(f, `${formatCiSummary(f.root, bundle)}\nManual readiness claim\n`); + const candidate = await inspectReportUpdate(f.root, f.env, fakeApi(f).github, now); + assert.equal(candidate.reportOnly, true); + assert.throws(() => verifyReportUpdate(f.root, candidate, bundle, now), /trusted rendering/); + }); + + it('preserves failed source checks rather than blessing them through skipped jobs', async () => { + const f = fixture(); + const bundle = execute(f); + reportCommit(f, formatCiSummary(f.root, bundle)); + const { github } = fakeApi(f, { + '/repos/example/furn/actions/runs/456/attempts/1/jobs?per_page=100&page=1': { + total_count: 1, + jobs: [{ name: 'PR source validation', conclusion: 'failure' }], + }, + }); + const candidate = await inspectReportUpdate(f.root, f.env, github, now); + assert.ok(candidate.reportOnly); + assert.equal(candidate.sourceResult, 'failure'); + verifyReportUpdate(f.root, candidate, bundle, now); + }); + + it('fails closed for a missing artifact or a source attempt with the wrong head', async () => { + const f = fixture(); + const bundle = execute(f); + reportCommit(f, formatCiSummary(f.root, bundle)); + await assert.rejects( + inspectReportUpdate( + f.root, + f.env, + fakeApi(f, { + '/repos/example/furn/actions/runs/456/artifacts?per_page=100&page=1': { total_count: 0, artifacts: [] }, + }).github, + now, + ), + /missing or expired/, + ); + await assert.rejects( + inspectReportUpdate( + f.root, + f.env, + fakeApi(f, { + '/repos/example/furn/actions/runs/456/attempts/1': { + id: 456, + run_attempt: 1, + event: 'pull_request', + status: 'completed', + path: '.github/workflows/pr.yml', + head_sha: 'a'.repeat(40), + head_branch: 'pilot/checkbox', + head_repository: { full_name: 'example/furn' }, + }, + }).github, + now, + ), + /source PR workflow/, + ); + }); + + it('runs full CI when there is no reusable report or its explicit reuse age has expired', async () => { + const f = fixture(); + assert.equal((await inspectReportUpdate(f.root, f.env, fakeApi(f).github, now)).reportOnly, false); + const bundle = execute(f); + reportCommit(f, formatCiSummary(f.root, bundle)); + const later = new Date(now.getTime() + 169 * 3_600_000); + const candidate = await inspectReportUpdate(f.root, f.env, fakeApi(f).github, later); + assert.equal(candidate.reportOnly, false); + assert.throws(() => assertFresh(bundle.generatedAt, later), /reuse limit/); + }); +}); + +describe('trusted report publication', () => { + const expected = { repository: 'example/furn', runId: '456', runAttempt: 1, defaultBranch: 'main' }; + + it('writes exactly the allowlisted summary and SPEC paths with one atomic expected-head check', async () => { + const f = fixture(); + const bundle = execute(f); + const { github, calls } = fakeApi(f); + const publication = await preparePublication(f.root, bundle, github, expected, now); + assert.ok(publication.eligible); + assert.equal(calls.filter(({ options }) => options?.method === 'POST').length, 0); + const sha = await publishAssessment(publication, github); + assert.equal(sha, 'd'.repeat(40)); + const mutation = calls.find(({ url }) => url.endsWith('/graphql')); + assert.ok(mutation); + const payload = JSON.parse(String(mutation.options?.body)); + assert.equal(payload.variables.input.expectedHeadOid, f.headCommit); + assert.deepEqual(payload.variables.input.branch, { repositoryNameWithOwner: 'example/furn', branchName: 'pilot/checkbox' }); + assert.equal(payload.variables.input.fileChanges.additions.length, 2); + assert.equal(payload.variables.input.fileChanges.additions[0].path, assessmentSummaryPath); + assert.equal(payload.variables.input.fileChanges.additions[1].path, assessmentSpecPath); + assert.equal(Buffer.from(payload.variables.input.fileChanges.additions[1].contents, 'base64').toString(), formatCiSpec(f.root, bundle)); + assert.equal(payload.variables.input.fileChanges.deletions, undefined); + }); + + for (const [label, overrides] of [ + ['newer source commit', { head: { sha: 'a'.repeat(40), ref: 'pilot/checkbox', repo: { full_name: 'example/furn' } } }], + ['closed PR', { state: 'closed' }], + ['fork PR', { head: { sha: 'a'.repeat(40), ref: 'pilot/checkbox', repo: { full_name: 'contributor/furn' } } }], + ['conflicted PR', { merge_commit_sha: null }], + ['protected main branch as source', { head: { sha: 'a'.repeat(40), ref: 'main', repo: { full_name: 'example/furn' } } }], + ] as const) { + it(`does not write for ${label}`, async () => { + const f = fixture(); + const bundle = execute(f); + const pr = { + number: 123, + state: 'open', + base: { sha: f.baseCommit, ref: 'main', repo: { full_name: 'example/furn' } }, + head: { sha: f.headCommit, ref: 'pilot/checkbox', repo: { full_name: 'example/furn' } }, + merge_commit_sha: f.testedCommit, + ...overrides, + }; + const { github, calls } = fakeApi(f, { '/repos/example/furn/pulls/123': pr }); + const publication = await preparePublication(f.root, bundle, github, expected, now); + assert.equal(publication.eligible, false); + assert.equal(calls.filter(({ options }) => options?.method === 'POST').length, 0); + }); + } + + it('rejects an artifact from a different source run before requesting a PR or creating a commit', async () => { + const f = fixture(); + const bundle = execute(f); + const { github, calls } = fakeApi(f); + await assert.rejects(preparePublication(f.root, bundle, github, { ...expected, runId: '999' }, now), /triggering workflow/); + assert.equal(calls.length, 0); + }); + + it('surfaces a branch race rejected by GitHub rather than reporting publication success', async () => { + const f = fixture(); + const bundle = execute(f); + const { github } = fakeApi(f, { '/graphql': { errors: [{ message: 'expectedHeadOid mismatch' }] } }); + const publication = await preparePublication(f.root, bundle, github, expected, now); + await assert.rejects(publishAssessment(publication, github), /rejected the assessment commit/); + }); + + it('surfaces permission errors without logging credentials or returning an empty success', async () => { + const fetchImplementation: typeof fetch = async () => new Response('{}', { status: 403 }); + const github = new AssessmentGitHub({ token: 'synthetic-token', fetchImplementation }); + await assert.rejects(github.request('/repos/example/furn/pulls/123'), (error: Error) => { + assert.match(error.message, /403/); + assert.doesNotMatch(error.message, /synthetic-token/); + return true; + }); + }); +}); + +describe('CI input schema and workflow wiring', () => { + it('rejects invalid IDs, unsafe references, malformed markers, and oversized bundles', () => { + for (const value of [null, {}, { repository: '../repo' }]) assert.throws(() => parseIdentity(value)); + assert.throws(() => parseReportReference('# Manual report')); + assert.throws(() => parseReportReference('')); + assert.throws(() => parseBundle('x'.repeat(2_000_001)), /size limit/); + assert.throws(() => assertFresh('2026-10-22T20:00:00.000Z', now), /future/); + }); + + it('keeps required checks alive, captures before smoke, and runs publisher code only from trusted main', () => { + const workflow = readFileSync(path.join(repositoryRoot, '.github', 'workflows', 'pr.yml'), 'utf8'); + assert.match(workflow, /name: PR\non: pull_request/); + assert.doesNotMatch(workflow, /paths-ignore:/); + assert.match(workflow, / {2}pr:\n {4}name: PR\n {4}if: always\(\)/); + assert.match(workflow, /The original source validation did not pass/); + const win32 = workflow.slice(workflow.indexOf(' win32-storybook:'), workflow.indexOf(' check-changesets:')); + assert.ok(win32.indexOf('controlAssessmentCiTask.ts begin') < win32.indexOf('name: Smoke test Win32 Storybook')); + assert.ok(win32.indexOf('controlAssessmentCiTask.ts finish') > win32.indexOf('name: Smoke test Win32 Storybook')); + assert.match(win32, /disabled-input-features: keyboard,physicalClick,wheel/); + assert.match(win32, /checkbox-win32-assessment-\$\{\{ github.run_attempt \}\}/); + const js = workflow.slice(workflow.indexOf(' js:'), workflow.indexOf(' bundle-size:')); + assert.match(js, /FURN_ASSESSMENT_UNIT_CAPTURE:/); + assert.equal(js.match(/run: yarn lage buildci/g)?.length, 1); + assert.doesNotMatch(workflow, /--run-unit-tests|controlAssessmentCiTask\.ts unit(?:\s|$)/); + assert.match(workflow, /needs: \[assessment-inputs, js, win32-storybook\]/); + assert.match(workflow, /run: node scripts\/src\/tasks\/controlAssessmentCiTask\.ts combine/); + const jest = readFileSync(path.join(repositoryRoot, 'scripts', 'src', 'tasks', 'jest.ts'), 'utf8'); + assert.match(jest, /runJestWithUnitCapture\(\s*context.root/); + const publisher = readFileSync(path.join(repositoryRoot, '.github', 'workflows', 'pr-control-assessment.yml'), 'utf8'); + assert.match(publisher, /workflow_run:/); + assert.match(publisher, /ref: \$\{\{ github.event.repository.default_branch \}\}/); + assert.doesNotMatch(publisher, /pull_request_target|run: yarn|npm install|ref: \$\{\{ github.event.workflow_run.head_sha/); + assert.ok( + publisher.indexOf('Validate publication without write credentials') < publisher.indexOf('Create narrowly scoped publisher token'), + ); + assert.match(publisher, /permission-contents: write/); + assert.doesNotMatch(publisher, /permission-workflows:|permission-administration:|skip-token-revoke:/); + }); +}); + +function readFileIfPresent(root: string, file: string): string { + try { + return readFileSync(path.join(root, file), 'utf8'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return ''; + throw error; + } +} diff --git a/scripts/src/tasks/controlAssessmentCi.ts b/scripts/src/tasks/controlAssessmentCi.ts new file mode 100644 index 0000000000..1d9721d843 --- /dev/null +++ b/scripts/src/tasks/controlAssessmentCi.ts @@ -0,0 +1,564 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; + +import { + assessCheckbox, + formatAssessmentMarkdown, + hasStorybookArtifact, + hex, + object, + parseEvidenceContext, + parseNativeReport, + readCheckboxContractFrom, + readNativeEvidence, + reportDigest, + text, + timestamp, + win32StorybookTargetId, +} from './controlAssessment.ts'; +import type { EvidenceContext } from './controlAssessment.ts'; +import { + assessmentPackagePath, + assessmentSpecPath, + specAssessmentFor, + splitSpecAssessment, + updateSpecAssessment, +} from './controlAssessmentSpec.ts'; +import { parseUnitContext, readCheckboxUnitMappingFrom } from './controlAssessmentUnit.ts'; +import type { UnitContext } from './controlAssessmentUnit.ts'; + +export const assessmentSummaryPath = 'packages/agentic/components/src/components/checkbox/assessment/ASSESSMENT.md'; +export const assessmentArtifactName = 'checkbox-win32-assessment'; +export const assessmentMaxAgeHours = 168; +export const assessmentWorkflowPath = '.github/workflows/pr.yml'; +const marker = '') || first.length > 4096) { + throw new Error('Missing or invalid generated assessment reference.'); + } + const encoded = first.slice(marker.length, -4); + if (!/^[A-Za-z0-9+/]+={0,2}$/.test(encoded)) throw new Error('Invalid assessment reference encoding.'); + const data = object(JSON.parse(Buffer.from(encoded, 'base64').toString('utf8')), 'Report reference'); + return { ...parseIdentity(data), generatedAt: timestamp(data.generatedAt, 'Report generation time') }; +} + +export function assertFresh(generatedAt: string, now: Date): void { + const age = now.getTime() - Date.parse(timestamp(generatedAt, 'Assessment generation time')); + if (!Number.isFinite(age) || age < 0 || age > assessmentMaxAgeHours * 3_600_000) { + throw new Error(`Assessment is from the future or older than the explicit ${assessmentMaxAgeHours}-hour CI reuse limit.`); + } +} + +export function assertTestedInputs(root: string, identity: CiIdentity): void { + ensureCommit(root, identity.testedCommit); + const parents = git(root, ['rev-list', '--parents', '-n', '1', identity.testedCommit]).split(' '); + if (parents.length !== 3 || parents[1] !== identity.baseCommit || parents[2] !== identity.headCommit) { + throw new Error('CI evidence must identify the actual PR merge commit and both source parents.'); + } + if (inputDigest(root, identity.testedCommit) !== identity.inputDigest) throw new Error('CI input digest does not match the tested tree.'); +} + +export function assessCiBundle(root: string, bundle: CiBundle) { + assertTestedInputs(root, bundle.identity); + const read = (file: string) => { + const content = gitFile(root, bundle.identity.testedCommit, file); + if (content === undefined) throw new Error(`Tested contract file is missing: ${file}.`); + return content; + }; + const contract = readCheckboxContractFrom(read); + if (bundle.context && Date.parse(bundle.context.finishedAt) > Date.parse(bundle.generatedAt)) { + throw new Error('Native execution finished after its assessment was generated.'); + } + const expectedBuildId = `${bundle.identity.repository}/actions/runs/${bundle.identity.runId}/attempts/${bundle.identity.runAttempt}`; + const report = assessCheckbox({ + contract, + sourceCommit: bundle.identity.testedCommit, + workingTreeDirty: false, + now: new Date(bundle.generatedAt), + expectedBuildId, + expectedTargetId: bundle.context?.targetId, + maxAgeHours: assessmentMaxAgeHours, + report: bundle.nativeReport === null ? undefined : parseNativeReport(JSON.parse(bundle.nativeReport)), + reportSha256: bundle.nativeReport === null ? undefined : reportDigest(bundle.nativeReport), + context: bundle.context ?? undefined, + executionFailure: bundle.executionFailure ?? undefined, + unit: bundle.unit + ? { + mapping: bundle.unit.report === null ? undefined : readCheckboxUnitMappingFrom(read), + report: bundle.unit.report ?? undefined, + context: bundle.unit.context ?? undefined, + executionFailure: bundle.unit.executionFailure ?? undefined, + expectedBuildId, + } + : undefined, + }); + if (report.evidence.status === 'mismatched' || report.evidence.status === 'stale') { + throw new Error(`CI evidence binding failed: ${report.evidence.reasons.join(' ')}`); + } + if (report.unitEvidence.status === 'mismatched' || report.unitEvidence.status === 'stale') { + throw new Error(`CI unit evidence binding failed: ${report.unitEvidence.reasons.join(' ')}`); + } + if (bundle.unit?.context?.workingTreeDirty) throw new Error('CI unit evidence was captured from dirty source inputs.'); + return report; +} + +export function formatCiSummary(root: string, bundle: CiBundle): string { + const report = assessCiBundle(root, bundle); + const reference: ReportReference = { ...bundle.identity, generatedAt: bundle.generatedAt }; + const encoded = Buffer.from(JSON.stringify(reference)).toString('base64'); + return [ + `${marker}${encoded} -->`, + '', + '', + ...formatAssessmentMarkdown(report).trimEnd().split('\n'), + '', + '## CI source and freshness', + '', + `Assessed at: ${bundle.generatedAt}.`, + `Pull request source: \`${bundle.identity.headCommit}\`. Tested base: \`${bundle.identity.baseCommit}\`.`, + `All tracked input digest: \`${bundle.identity.inputDigest}\`.`, + `Execution: https://github.com/${bundle.identity.repository}/actions/runs/${bundle.identity.runId}/attempts/${bundle.identity.runAttempt}`, + '', + 'This is a snapshot of that execution, not a claim that the latest revision has passed.', + 'Only this generated summary and the exact validated assessment block in Checkbox SPEC front matter are excluded from the input digest. All other contract bytes, code, tests, dependencies, workflows, and configuration remain inputs.', + 'A report-only commit is not relabeled as a tested commit. CI may reuse the original results only after checking unchanged inputs and the original required PR result.', + ...(bundle.executionFailure + ? ['', `CI execution limitation: ${bundle.executionFailure.replace(/[<>]/g, '').replace(/[\r\n]/g, ' ')}`] + : []), + '', + ].join('\n'); +} + +export function formatCiSpec(root: string, bundle: CiBundle): string { + const report = assessCiBundle(root, bundle); + for (const commit of [bundle.identity.headCommit, bundle.identity.testedCommit]) { + if (!git(root, ['ls-tree', commit, '--', assessmentSpecPath]).startsWith('100644 blob ')) { + throw new Error('Assessment publication requires a regular nonexecutable SPEC file.'); + } + } + const manifest = gitFile(root, bundle.identity.testedCommit, assessmentPackagePath); + const spec = gitFile(root, bundle.identity.headCommit, assessmentSpecPath); + if (manifest === undefined || spec === undefined) throw new Error('Tested component manifest or source SPEC is missing.'); + const version = text(object(JSON.parse(manifest), 'Tested component manifest').version, 'Tested component version'); + return updateSpecAssessment(spec, specAssessmentFor(report, version)); +} + +function identityFromEnvironment(root: string, env: NodeJS.ProcessEnv): CiIdentity { + const eventPath = text(env.GITHUB_EVENT_PATH, 'GITHUB_EVENT_PATH'); + const event = object(JSON.parse(readFileSync(eventPath, 'utf8')), 'PR event'); + const pr = object(event.pull_request, 'PR event pull_request'); + const head = object(pr.head, 'PR head'); + const base = object(pr.base, 'PR base'); + const testedCommit = hex(env.GITHUB_SHA, 40, 'GITHUB_SHA'); + const identity = parseIdentity({ + repository: env.GITHUB_REPOSITORY, + pullRequest: pr.number, + headRepository: object(head.repo, 'PR head repository').full_name, + headRef: head.ref, + headCommit: head.sha, + baseCommit: base.sha, + testedCommit, + runId: env.GITHUB_RUN_ID, + runAttempt: Number(env.GITHUB_RUN_ATTEMPT), + inputDigest: inputDigest(root, testedCommit), + }); + if (git(root, ['rev-parse', 'HEAD']) !== testedCommit) throw new Error('CI checkout does not match GITHUB_SHA.'); + assertTestedInputs(root, identity); + return identity; +} + +function assertCleanInputs(root: string): void { + if (git(root, ['status', '--porcelain', '--untracked-files=all', '--', '.', `:(exclude)${assessmentSummaryPath}`])) { + throw new Error('CI tracked or nonignored inputs changed outside the generated assessment summary.'); + } +} + +export function beginCiExecution(root: string, env: NodeJS.ProcessEnv, now = new Date()): void { + const directory = path.join(root, ...artifactDirectory); + const identity = identityFromEnvironment(root, env); + assertCleanInputs(root); + if ( + !(env.FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES ?? '') + .split(',') + .map((value) => value.trim()) + .includes('physicalClick') + ) { + throw new Error('The hosted pilot requires explicit physicalClick disabling; do not claim interactive-runner qualification.'); + } + const contract = readCheckboxContractFrom((file) => readFileSync(path.join(root, file), 'utf8')); + mkdirSync(directory, { recursive: true }); + const start: ExecutionStart = { + schemaVersion: 2, + identity, + contractDigest: contract.digest, + startedAt: now.toISOString(), + existingRuns: desktopRuns(root), + targetId: win32StorybookTargetId(root), + }; + writeFileSync(path.join(directory, 'execution-start.json'), `${JSON.stringify(start, null, 2)}\n`); +} + +export function beginCiUnitCollection(root: string, env: NodeJS.ProcessEnv, now = new Date()): void { + const directory = path.join(root, 'apps', 'storybook', 'artifacts', 'assessment-unit-ci'); + const identity = identityFromEnvironment(root, env); + assertCleanInputs(root); + mkdirSync(directory, { recursive: true }); + if (existsSync(path.join(directory, 'unit-execution.json'))) throw new Error('Unit results already exist before CI collection.'); + writeFileSync( + path.join(directory, 'unit-collection-start.json'), + `${JSON.stringify( + { + schemaVersion: 1, + identity, + startedAt: now.toISOString(), + buildId: `${identity.repository}/actions/runs/${identity.runId}/attempts/${identity.runAttempt}`, + }, + null, + 2, + )}\n`, + ); +} + +export function finishCiExecution(root: string, env: NodeJS.ProcessEnv, now = new Date()): CiBundle { + const directory = path.join(root, ...artifactDirectory); + mkdirSync(directory, { recursive: true }); + const identity = identityFromEnvironment(root, env); + assertCleanInputs(root); + const startPath = path.join(directory, 'execution-start.json'); + const started = existsSync(startPath) ? object(JSON.parse(readFileSync(startPath, 'utf8')), 'Execution start') : undefined; + if (started && started.schemaVersion !== 2) throw new Error('Unsupported pre-execution capture schema.'); + if (started && JSON.stringify(parseIdentity(started.identity)) !== JSON.stringify(identity)) { + throw new Error('CI inputs or identity changed during native execution.'); + } + if (started && (!Array.isArray(started.existingRuns) || started.existingRuns.some((name) => typeof name !== 'string'))) { + throw new Error('Invalid pre-execution desktop run inventory.'); + } + const freshRuns = started ? desktopRuns(root).filter((name) => !(started.existingRuns as string[]).includes(name)) : []; + if (freshRuns.length > 1) throw new Error('Ambiguous native evidence: more than one desktop run started during smoke.'); + const reportPath = freshRuns.length ? path.join(root, ...desktopRunsPath, freshRuns[0], 'run.json') : undefined; + let nativeReport: string | null = null; + let collectionFailure: string | undefined; + if (reportPath && existsSync(reportPath)) { + const canonical = object(JSON.parse(readFileSync(reportPath, 'utf8')), 'Desktop supervisor report'); + if (hasStorybookArtifact(canonical)) { + if (existsSync(path.join(path.dirname(reportPath), 'evidence', 'win32', 'static', 'run.json'))) { + nativeReport = readNativeEvidence(reportPath); + } else { + collectionFailure = 'Desktop supervisor declared Win32 Storybook smoke evidence, but the companion report is missing.'; + } + } else { + collectionFailure = 'Desktop supervisor produced no Win32 Storybook smoke evidence; native assessment is unavailable.'; + } + } + let context: EvidenceContext | null = null; + if (nativeReport !== null) { + const parsed = parseNativeReport(JSON.parse(nativeReport)); + if (parsed.format !== 'desktop-test' || !parsed.finishedAt || !parsed.startedAt) { + throw new Error('CI requires the canonical desktop-test result and its timestamped smoke evidence.'); + } + const began = timestamp(started?.startedAt, 'Execution start time'); + if (Date.parse(parsed.startedAt) < Date.parse(began)) { + throw new Error('Desktop report predates this execution.'); + } + if (parsed.harness?.runId !== freshRuns[0]) throw new Error('Desktop run ID does not match its artifact directory.'); + if (parsed.targetId !== text(started?.targetId, 'Execution target ID')) + throw new Error('Desktop report targets a different native instance from the pre-execution capture.'); + context = parseEvidenceContext({ + schemaVersion: 1, + reportSha256: reportDigest(nativeReport), + sourceCommit: identity.testedCommit, + buildId: `${identity.repository}/actions/runs/${identity.runId}/attempts/${identity.runAttempt}`, + contractDigest: hex(started?.contractDigest, 64, 'Execution contract digest'), + platformManifestDigest: parsed.platformManifestDigest, + endpoint: 'win32', + renderer: 'paper', + targetId: text(started?.targetId, 'Execution target ID'), + finishedAt: parsed.finishedAt, + physicalClick: false, + }); + } + const outcome = text(env.SMOKE_OUTCOME, 'SMOKE_OUTCOME'); + if (!['success', 'failure', 'cancelled', 'skipped'].includes(outcome)) throw new Error('Unsupported smoke step outcome.'); + const executionFailure = + outcome !== 'success' + ? `Win32 smoke step outcome: ${outcome}. Recorded results do not establish successful native execution.` + : nativeReport === null + ? (collectionFailure ?? 'Win32 smoke did not produce a current native report.') + : null; + const unit = readCiUnitCapture(directory, identity, env.UNIT_OUTCOME); + const bundle = parseBundle( + JSON.stringify({ schemaVersion: 2, identity, generatedAt: now.toISOString(), nativeReport, context, executionFailure, unit }), + ); + writeCiAssessment(root, directory, bundle, env); + return bundle; +} + +function readCiUnitCapture(directory: string, identity: CiIdentity, unitOutcome?: string): NonNullable { + const unitPath = path.join(directory, 'unit-execution.json'); + let unit: NonNullable = { report: null, context: null, executionFailure: null }; + if (unitOutcome !== undefined && !['success', 'failure', 'cancelled', 'skipped'].includes(unitOutcome)) { + throw new Error('Unsupported unit step outcome.'); + } + if (existsSync(unitPath)) { + const captured = object(JSON.parse(readFileSync(unitPath, 'utf8')), 'Unit execution capture'); + if (captured.schemaVersion !== 1 || JSON.stringify(parseIdentity(captured.identity)) !== JSON.stringify(identity)) { + throw new Error('Unit execution capture belongs to different CI inputs or attempt.'); + } + const context = captured.context === null ? null : parseUnitContext(captured.context); + if (!Number.isSafeInteger(captured.exitCode) || Number(captured.exitCode) < 0) throw new Error('Invalid unit capture exit code.'); + if (context && Date.parse(context.startedAt) < Date.parse(timestamp(captured.collectionStartedAt, 'Unit collection start time'))) { + throw new Error('Unit report predates this CI execution.'); + } + if (captured.report !== null && typeof captured.report !== 'string') throw new Error('Invalid captured unit report.'); + if (captured.executionFailure !== null && typeof captured.executionFailure !== 'string') + throw new Error('Invalid captured unit failure.'); + unit = { + report: captured.report, + context, + executionFailure: + unitOutcome !== undefined && unitOutcome !== 'success' + ? `Checkbox unit step outcome: ${unitOutcome}.` + : Number(captured.exitCode) !== 0 + ? `Checkbox unit collector failed with exit ${captured.exitCode}.` + : captured.executionFailure, + }; + } else if (unitOutcome !== undefined) { + unit.executionFailure = `Checkbox unit step outcome: ${unitOutcome}; no current execution capture was produced.`; + } + return unit; +} + +export function combineCiEvidence(root: string, env: NodeJS.ProcessEnv, now = new Date()): CiBundle { + const directory = path.join(root, ...artifactDirectory); + mkdirSync(directory, { recursive: true }); + const identity = identityFromEnvironment(root, env); + assertCleanInputs(root); + const nativePath = path.join(directory, 'bundle.json'); + const native = existsSync(nativePath) ? parseBundle(readFileSync(nativePath, 'utf8')) : undefined; + if (native && JSON.stringify(native.identity) !== JSON.stringify(identity)) + throw new Error('Native artifact belongs to different CI inputs or attempt.'); + if (native) assertFresh(native.generatedAt, now); + const unitDirectory = path.join(root, 'apps', 'storybook', 'artifacts', 'assessment-unit-ci'); + const unit = readCiUnitCapture( + unitDirectory, + identity, + existsSync(path.join(unitDirectory, 'unit-execution.json')) ? undefined : env.UNIT_OUTCOME, + ); + const bundle = parseBundle( + JSON.stringify({ + schemaVersion: 2, + identity, + generatedAt: now.toISOString(), + nativeReport: native?.nativeReport ?? null, + context: native?.context ?? null, + executionFailure: + native?.executionFailure ?? + (native ? null : `Native job outcome: ${text(env.NATIVE_OUTCOME, 'NATIVE_OUTCOME')}; no native bundle was collected.`), + unit, + }), + ); + writeCiAssessment(root, directory, bundle, env); + return bundle; +} + +function writeCiAssessment(root: string, directory: string, bundle: CiBundle, env: NodeJS.ProcessEnv): void { + const report = assessCiBundle(root, bundle); + const summary = formatCiSummary(root, bundle); + writeFileSync(path.join(directory, 'bundle.json'), `${JSON.stringify(bundle, null, 2)}\n`); + writeFileSync(path.join(directory, 'assessment.json'), `${JSON.stringify(report, null, 2)}\n`); + writeFileSync(path.join(directory, 'ASSESSMENT.md'), summary); + writeFileSync(path.join(directory, 'SPEC.md'), formatCiSpec(root, bundle)); + if (bundle.unit?.report !== null && bundle.unit?.report !== undefined) { + writeFileSync(path.join(directory, 'jest.json'), bundle.unit.report); + writeFileSync(path.join(directory, 'unit-context.json'), `${JSON.stringify(bundle.unit.context, null, 2)}\n`); + } + if (env.GITHUB_STEP_SUMMARY) appendFileSync(env.GITHUB_STEP_SUMMARY, summary); +} + +export function isReportOnlyUpdate(root: string, reference: ReportReference, head: string, merged: string): boolean { + ensureCommit(root, reference.headCommit); + ensureCommit(root, head); + ensureCommit(root, merged); + const parents = git(root, ['rev-list', '--parents', '-n', '1', head]).split(' '); + if (parents.length !== 2 || parents[1] !== reference.headCommit) return false; + const changed = git(root, ['diff', '--name-only', reference.headCommit, head]).split('\n'); + if (changed.length !== 2 || changed.some((file) => ![assessmentSummaryPath, assessmentSpecPath].includes(file))) return false; + for (const file of [assessmentSummaryPath, assessmentSpecPath]) { + if (!git(root, ['ls-tree', merged, '--', file]).startsWith('100644 blob ')) return false; + } + const spec = gitFile(root, head, assessmentSpecPath); + if (spec === undefined || splitSpecAssessment(spec).assessment === undefined) return false; + assertTestedInputs(root, reference); + return inputDigest(root, merged) === reference.inputDigest; +} diff --git a/scripts/src/tasks/controlAssessmentCiTask.ts b/scripts/src/tasks/controlAssessmentCiTask.ts new file mode 100644 index 0000000000..f037d65c9d --- /dev/null +++ b/scripts/src/tasks/controlAssessmentCiTask.ts @@ -0,0 +1,133 @@ +import { appendFileSync, readFileSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { object, text } from './controlAssessment.ts'; +import { + assessmentArtifactName, + beginCiExecution, + beginCiUnitCollection, + combineCiEvidence, + finishCiExecution, + git, +} from './controlAssessmentCi.ts'; +import { + AssessmentGitHub, + inspectReportUpdate, + preparePublication, + publishAssessment, + readBundle, + verifyReportUpdate, +} from './controlAssessmentGithub.ts'; +import type { Candidate } from './controlAssessmentGithub.ts'; + +function output(name: string, value: string | boolean): void { + if (/[\r\n]/.test(String(value))) throw new Error('Invalid multiline workflow output.'); + appendFileSync(text(process.env.GITHUB_OUTPUT, 'GITHUB_OUTPUT'), `${name}=${value}\n`); +} + +function notice(message: string): void { + process.stdout.write(`${message}\n`); + if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${message}\n\n`); +} + +async function main(args = process.argv.slice(2)): Promise { + const env = process.env; + const root = git(process.cwd(), ['rev-parse', '--show-toplevel']); + const command = args[0]; + if (command === 'begin') { + beginCiExecution(root, env); + notice('Captured pre-execution Checkbox inputs. Hosted physical input remains disabled.'); + } else if (command === 'begin-unit') { + beginCiUnitCollection(root, env); + notice('Prepared capture of the existing component Jest invocation; no additional unit tests are launched.'); + } else if (command === 'combine') { + const bundle = combineCiEvidence(root, env); + notice( + `Assessed existing JS/native results from run ${bundle.identity.runId}, attempt ${bundle.identity.runAttempt}; no tests were rerun.`, + ); + } else if (command === 'finish') { + const bundle = finishCiExecution(root, env); + notice( + `Advisory Checkbox assessment generated for run ${bundle.identity.runId}, attempt ${bundle.identity.runAttempt}. No maturity stage assigned.`, + ); + } else if (command === 'inspect') { + const github = new AssessmentGitHub({ token: text(env.GITHUB_TOKEN, 'GITHUB_TOKEN') }); + const candidate = await inspectReportUpdate(root, env, github); + output('candidate', candidate.reportOnly); + if (candidate.reportOnly) { + const candidatePath = path.join(text(env.RUNNER_TEMP, 'RUNNER_TEMP'), 'checkbox-assessment-candidate.json'); + writeFileSync(candidatePath, JSON.stringify(candidate)); + output('candidate-path', candidatePath); + output('run-id', candidate.reference.runId); + output('artifact-name', `${assessmentArtifactName}-${candidate.reference.runAttempt}`); + notice('Report-only candidate found. Source jobs remain gated until the original evidence and rendered report are verified.'); + } else { + output('report-only', false); + notice(candidate.reason); + } + } else if (command === 'verify') { + const candidate = JSON.parse(readFileSync(text(env.CANDIDATE_PATH, 'CANDIDATE_PATH'), 'utf8')) as Candidate; + verifyReportUpdate(root, candidate, readBundle(text(env.BUNDLE_PATH, 'BUNDLE_PATH'))); + if (!candidate.reportOnly) throw new Error('No report-only candidate.'); + output('report-only', true); + output('source-result', candidate.sourceResult); + notice( + `Verified unchanged tested inputs and exact generated report/SPEC assessment fields. Reusing source validation result: ${candidate.sourceResult}.`, + ); + } else if (command === 'find-artifact') { + const event = object(JSON.parse(readFileSync(text(env.GITHUB_EVENT_PATH, 'GITHUB_EVENT_PATH'), 'utf8')), 'Workflow event'); + const run = object(event.workflow_run, 'Source workflow run'); + const github = new AssessmentGitHub({ token: text(env.GITHUB_TOKEN, 'GITHUB_TOKEN') }); + const found = await github.hasArtifact(text(env.GITHUB_REPOSITORY, 'GITHUB_REPOSITORY'), String(run.id), Number(run.run_attempt)); + output('found', found); + output('artifact-name', `${assessmentArtifactName}-${Number(run.run_attempt)}`); + notice( + found + ? 'Found the source attempt assessment bundle.' + : 'No assessment artifact exists for this attempt. Report-only verification runs do not execute or republish native assessments.', + ); + } else if (command === 'prepare-publication') { + const event = object(JSON.parse(readFileSync(text(env.GITHUB_EVENT_PATH, 'GITHUB_EVENT_PATH'), 'utf8')), 'Workflow event'); + const run = object(event.workflow_run, 'Source workflow run'); + const github = new AssessmentGitHub({ token: text(env.GITHUB_TOKEN, 'GITHUB_TOKEN') }); + const publication = await preparePublication(root, readBundle(text(env.BUNDLE_PATH, 'BUNDLE_PATH')), github, { + repository: text(env.GITHUB_REPOSITORY, 'GITHUB_REPOSITORY'), + runId: String(run.id), + runAttempt: Number(run.run_attempt), + defaultBranch: text(object(event.repository, 'Repository').default_branch, 'Default branch'), + }); + output('eligible', publication.eligible); + if (publication.eligible) { + notice('Validated the source attempt, native evidence, merged input tree, target PR, and both allowlisted assessment write paths.'); + } else { + notice(publication.reason); + } + } else if (command === 'publish') { + // Revalidate after acquiring the write credential; never trust a stale prepared mutation. + const event = object(JSON.parse(readFileSync(text(env.GITHUB_EVENT_PATH, 'GITHUB_EVENT_PATH'), 'utf8')), 'Workflow event'); + const run = object(event.workflow_run, 'Source workflow run'); + const readGitHub = new AssessmentGitHub({ token: text(env.GITHUB_TOKEN, 'GITHUB_TOKEN') }); + const publication = await preparePublication(root, readBundle(text(env.BUNDLE_PATH, 'BUNDLE_PATH')), readGitHub, { + repository: text(env.GITHUB_REPOSITORY, 'GITHUB_REPOSITORY'), + runId: String(run.id), + runAttempt: Number(run.run_attempt), + defaultBranch: text(object(event.repository, 'Repository').default_branch, 'Default branch'), + }); + if (!publication.eligible) { + notice(publication.reason); + return; + } + const writer = new AssessmentGitHub({ token: text(env.ASSESSMENT_WRITE_TOKEN, 'ASSESSMENT_WRITE_TOKEN') }); + const commit = await publishAssessment(publication, writer); + notice( + `Published the generated Checkbox summary and SPEC assessment fields in commit ${commit}. Contract content, code, dependencies and workflows are unchanged.`, + ); + } else { + throw new Error('Expected begin, begin-unit, finish, combine, inspect, verify, find-artifact, prepare-publication, or publish.'); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + await main(); +} diff --git a/scripts/src/tasks/controlAssessmentFixtures.ts b/scripts/src/tasks/controlAssessmentFixtures.ts new file mode 100644 index 0000000000..d915b340c7 --- /dev/null +++ b/scripts/src/tasks/controlAssessmentFixtures.ts @@ -0,0 +1,140 @@ +import { reportDigest } from './controlAssessment.ts'; +import { checkboxUnitCases, checkboxUnitTestPath } from './controlAssessmentUnit.ts'; + +/** Synthetic Jest protocol fixture, not an actual Checkbox test execution. */ +export function checkboxJestFixture(overrides: Record = {}) { + const assertions: { + fullName: string; + ancestorTitles: string[]; + title: string; + status: string; + failureMessages: string[]; + invocations?: number; + retryReasons?: string[]; + }[] = checkboxUnitCases.map(({ fullName }) => { + const win32 = fullName.includes(' accessibility actions '); + const words = fullName.split(' '); + const ancestorTitles = win32 ? ['Checkbox', `${words[1]} accessibility actions`] : ['Checkbox']; + const title = fullName.slice(ancestorTitles.join(' ').length + 1); + return { fullName, ancestorTitles, title, status: 'passed', failureMessages: [] }; + }); + return { + numTotalTestSuites: 1, + numPassedTestSuites: 1, + numFailedTestSuites: 0, + numPendingTestSuites: 0, + numTotalTests: assertions.length, + numPassedTests: assertions.length, + numFailedTests: 0, + numPendingTests: 0, + numTodoTests: 0, + numRuntimeErrorTestSuites: 0, + startTime: Date.parse('2026-10-21T20:00:00.500Z'), + success: true, + wasInterrupted: false, + testResults: [ + { + name: `C:\\synthetic\\${checkboxUnitTestPath.replace(/\//g, '\\')}`, + status: 'passed', + startTime: Date.parse('2026-10-21T20:00:00.600Z'), + endTime: Date.parse('2026-10-21T20:00:01.000Z'), + assertionResults: assertions, + }, + ], + ...overrides, + }; +} + +/** Synthetic existing-package run, including a type suite that must not qualify runtime/type claims. */ +export function componentJestFixture() { + const data = checkboxJestFixture(); + data.testResults.push({ + ...data.testResults[0], + name: data.testResults[0].name.replace('checkbox.test.tsx', 'checkbox.types.test.ts'), + assertionResults: [ + { + fullName: 'Checkbox slot types accepts the public checkbox contract as a slot prop', + ancestorTitles: ['Checkbox slot types'], + title: 'accepts the public checkbox contract as a slot prop', + status: 'passed', + failureMessages: [], + }, + ], + }); + data.numTotalTestSuites++; + data.numPassedTestSuites++; + data.numTotalTests++; + data.numPassedTests++; + return data; +} + +/** Synthetic collector output; this fixture is not native qualification evidence. */ +export function desktopEvidenceFixture(smoke: string, overrides: Record = {}) { + const story = JSON.parse(smoke); + const tests = [ + { + testId: 'navigation:components-checkbox--default', + title: 'Navigate components-checkbox--default', + status: 'passed', + flaky: false, + attempts: [{ attempt: 0, status: 'passed', durationMs: 1, context: { story: 'components-checkbox--default' } }], + }, + ...story.wdio.map( + (entry: { storyId: string; testName?: string; status: string; durationMs: number; skipReason?: string; error?: string }) => { + const name = entry.testName ?? 'default'; + return { + testId: `csf-${reportDigest(JSON.stringify([entry.storyId, name]))}`, + title: `${entry.storyId} / ${name}`, + status: entry.status, + flaky: false, + attempts: [ + { + attempt: 0, + status: entry.status, + durationMs: entry.durationMs, + ...(entry.skipReason ? { reason: entry.skipReason } : {}), + ...(entry.status === 'failed' + ? { + failure: { + failureId: 'test-failure', + kind: 'test', + error: { name: 'Error', message: entry.error ?? 'assertion failed' }, + }, + } + : {}), + context: { endpoint: story.endpoint, story: entry.storyId, test: name }, + }, + ], + }; + }, + ), + ]; + const run = { + schemaVersion: 1, + runId: 'synthetic-desktop-run', + kind: 'tests', + status: story.status, + startedAt: story.startedAt, + finishedAt: story.finishedAt, + durationMs: 1000, + tests, + work: [], + failures: [], + counts: { + selected: tests.length, + executed: tests.filter(({ status }) => status === 'passed' || status === 'failed').length, + passed: tests.filter(({ status }) => status === 'passed').length, + failed: tests.filter(({ status }) => status === 'failed').length, + skipped: tests.filter(({ status }) => status === 'skipped').length, + unsupported: 0, + incomplete: 0, + retries: 0, + flaky: 0, + }, + artifacts: [{ path: 'evidence\\win32\\static\\run.json', description: 'Story smoke results' }], + ...overrides, + }; + const canonical = JSON.stringify(run); + const envelope = JSON.stringify({ schemaVersion: 1, format: 'desktop-test', desktopRun: canonical, storyReport: smoke }); + return { run, canonical, envelope }; +} diff --git a/scripts/src/tasks/controlAssessmentGithub.ts b/scripts/src/tasks/controlAssessmentGithub.ts new file mode 100644 index 0000000000..bdd34ddeb3 --- /dev/null +++ b/scripts/src/tasks/controlAssessmentGithub.ts @@ -0,0 +1,272 @@ +import { readFileSync } from 'node:fs'; + +import { hex, object, text } from './controlAssessment.ts'; +import { + assessmentArtifactName, + assessmentSummaryPath, + assessmentWorkflowPath, + assertFresh, + assertTestedInputs, + ensureCommit, + formatCiSummary, + formatCiSpec, + git, + gitFile, + inputDigest, + isReportOnlyUpdate, + parseBundle, + parseReportReference, +} from './controlAssessmentCi.ts'; +import type { CiBundle, CiIdentity, ReportReference } from './controlAssessmentCi.ts'; +import { assessmentSpecPath } from './controlAssessmentSpec.ts'; + +type Run = { + id: number; + run_attempt: number; + event: string; + status: string; + path: string; + head_sha: string; + head_branch: string; + head_repository: { full_name: string } | null; +}; +type PullRequest = { + number: number; + state: string; + base: { sha: string; ref: string; repo: { full_name: string } }; + head: { sha: string; ref: string; repo: { full_name: string } | null }; + merge_commit_sha: string | null; +}; +type Jobs = { jobs: { name: string; conclusion: string | null }[]; total_count: number }; +type Artifacts = { artifacts: { name: string; expired: boolean }[]; total_count: number }; +type GitHubOptions = { token: string; fetchImplementation?: typeof fetch }; + +export class AssessmentGitHub { + private readonly token: string; + private readonly fetchImplementation: typeof fetch; + + constructor({ token, fetchImplementation = fetch }: GitHubOptions) { + this.token = text(token, 'GitHub token'); + this.fetchImplementation = fetchImplementation; + } + + async request(route: string, options: RequestInit = {}): Promise { + if (!route.startsWith('/repos/') && route !== '/graphql') throw new Error('Unexpected assessment API route.'); + const response = await this.fetchImplementation(`https://api.github.com${route}`, { + ...options, + redirect: 'error', + signal: AbortSignal.timeout(30_000), + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${this.token}`, + 'Content-Type': 'application/json', + 'X-GitHub-Api-Version': '2022-11-28', + }, + }); + if (!response.ok) throw new Error(`Assessment GitHub API request failed (${response.status}): ${route}.`); + return (await response.json()) as T; + } + + async sourceRun(identity: CiIdentity): Promise { + const run = await this.request(`/repos/${identity.repository}/actions/runs/${identity.runId}/attempts/${identity.runAttempt}`); + if ( + String(run.id) !== identity.runId || + run.run_attempt !== identity.runAttempt || + run.event !== 'pull_request' || + run.status !== 'completed' || + run.path !== assessmentWorkflowPath || + run.head_sha !== identity.headCommit || + run.head_branch !== identity.headRef || + run.head_repository?.full_name !== identity.headRepository + ) { + throw new Error('Assessment evidence does not match the completed source PR workflow attempt.'); + } + return run; + } + + async sourceValidation(identity: CiIdentity): Promise<'success' | 'failure'> { + const matches: Jobs['jobs'] = []; + for (let page = 1; page <= 20; page++) { + const data = await this.request( + `/repos/${identity.repository}/actions/runs/${identity.runId}/attempts/${identity.runAttempt}/jobs?per_page=100&page=${page}`, + ); + matches.push(...data.jobs.filter((job) => job.name === 'PR source validation')); + if (page * 100 >= data.total_count) { + if (matches.length !== 1) throw new Error('Source workflow must have exactly one PR source validation job.'); + return matches[0].conclusion === 'success' ? 'success' : 'failure'; + } + } + throw new Error('Source workflow job list exceeds the supported limit.'); + } + + async hasArtifact(repository: string, runId: string, runAttempt: number): Promise { + const name = `${assessmentArtifactName}-${runAttempt}`; + let matches = 0; + for (let page = 1; page <= 20; page++) { + const data = await this.request(`/repos/${repository}/actions/runs/${runId}/artifacts?per_page=100&page=${page}`); + matches += data.artifacts.filter((artifact) => artifact.name === name && !artifact.expired).length; + if (page * 100 >= data.total_count) { + if (matches > 1) throw new Error('Source workflow has ambiguous assessment artifacts.'); + return matches === 1; + } + } + throw new Error('Source workflow artifact list exceeds the supported limit.'); + } +} + +export type Candidate = + | { reportOnly: false; reason: string } + | { reportOnly: true; reference: ReportReference; summary: string; spec: string; sourceResult: 'success' | 'failure' }; + +export async function inspectReportUpdate( + root: string, + env: NodeJS.ProcessEnv, + github: AssessmentGitHub, + now = new Date(), +): Promise { + const event = object(JSON.parse(readFileSync(text(env.GITHUB_EVENT_PATH, 'GITHUB_EVENT_PATH'), 'utf8')), 'PR event'); + const pr = object(event.pull_request, 'PR event pull_request'); + const head = object(pr.head, 'PR head'); + const merge = hex(env.GITHUB_SHA, 40, 'PR merge revision'); + ensureCommit(root, merge); + const parents = git(root, ['rev-list', '--parents', '-n', '1', merge]).split(' '); + if (parents.length !== 3 || parents[1] !== object(pr.base, 'PR base').sha || parents[2] !== head.sha) { + throw new Error('Current CI merge revision does not match the PR event.'); + } + const summary = gitFile(root, merge, assessmentSummaryPath); + if (summary === undefined) return { reportOnly: false, reason: 'No checked-in assessment: run normal source CI.' }; + let reference: ReportReference; + try { + reference = parseReportReference(summary); + assertFresh(reference.generatedAt, now); + } catch (error) { + return { reportOnly: false, reason: `Assessment is not reusable; run normal source CI. ${String(error)}` }; + } + if ( + reference.repository !== env.GITHUB_REPOSITORY || + reference.pullRequest !== pr.number || + reference.headRepository !== object(head.repo, 'PR head repository').full_name || + reference.headRef !== head.ref + ) { + return { reportOnly: false, reason: 'Assessment belongs to a different PR: run normal source CI.' }; + } + if (!isReportOnlyUpdate(root, reference, hex(head.sha, 40, 'PR head revision'), merge)) { + return { + reportOnly: false, + reason: 'Code, contract, test, dependency, configuration, or branch inputs changed: run normal source CI.', + }; + } + await github.sourceRun(reference); + if (!(await github.hasArtifact(reference.repository, reference.runId, reference.runAttempt))) { + throw new Error('Report-only update cannot reuse missing or expired source evidence.'); + } + const spec = gitFile(root, hex(head.sha, 40, 'PR head revision'), assessmentSpecPath); + if (spec === undefined) throw new Error('Report-only update is missing its SPEC assessment snapshot.'); + return { reportOnly: true, reference, summary, spec, sourceResult: await github.sourceValidation(reference) }; +} + +export function verifyReportUpdate(root: string, candidate: Candidate, bundle: CiBundle, now = new Date()): void { + if (!candidate.reportOnly) throw new Error('No report-only candidate was identified.'); + assertFresh(bundle.generatedAt, now); + const reference: ReportReference = { ...bundle.identity, generatedAt: bundle.generatedAt }; + if (JSON.stringify(reference) !== JSON.stringify(candidate.reference)) throw new Error('Downloaded evidence belongs to a different run.'); + if (formatCiSummary(root, bundle) !== candidate.summary) { + throw new Error('Checked-in report is not the trusted rendering of its source evidence.'); + } + if (formatCiSpec(root, bundle) !== candidate.spec) + throw new Error('Checked-in SPEC is not the trusted rendering of its source evidence.'); +} + +export type Publication = + | { eligible: false; reason: string } + | { eligible: true; repository: string; branch: string; expectedHead: string; summary: string; spec: string; sourceRunId: string }; + +export async function preparePublication( + root: string, + bundle: CiBundle, + github: AssessmentGitHub, + expected: { repository: string; runId: string; runAttempt: number; defaultBranch: string }, + now = new Date(), +): Promise { + const identity = bundle.identity; + if (identity.repository !== expected.repository || identity.runId !== expected.runId || identity.runAttempt !== expected.runAttempt) { + throw new Error('Downloaded bundle is not from the triggering workflow attempt.'); + } + assertFresh(bundle.generatedAt, now); + await github.sourceRun(identity); + assertTestedInputs(root, identity); + const pr = await github.request(`/repos/${identity.repository}/pulls/${identity.pullRequest}`); + if (pr.state !== 'open') return { eligible: false, reason: 'PR is no longer open; no report commit was made.' }; + if (identity.headRepository !== identity.repository || pr.head.repo?.full_name !== identity.repository) { + return { eligible: false, reason: 'Fork PR: the pilot publishes CI artifacts only; no fork branch write is attempted.' }; + } + if (pr.base.repo.full_name !== identity.repository || pr.base.ref !== expected.defaultBranch || pr.head.ref === expected.defaultBranch) { + return { eligible: false, reason: 'The pilot writes only to a PR source branch targeting the default branch, never to main.' }; + } + if (pr.head.sha !== identity.headCommit || pr.head.ref !== identity.headRef) { + return { eligible: false, reason: 'PR head moved since testing; discard the stale publication.' }; + } + if (!pr.merge_commit_sha) return { eligible: false, reason: 'PR merge revision is unavailable or conflicted; resolve and rerun CI.' }; + ensureCommit(root, pr.merge_commit_sha); + const parents = git(root, ['rev-list', '--parents', '-n', '1', pr.merge_commit_sha]).split(' '); + if (parents.length !== 3 || parents[1] !== pr.base.sha || parents[2] !== pr.head.sha) { + return { eligible: false, reason: 'GitHub has not refreshed the current PR merge revision; rerun publication after it is available.' }; + } + if (inputDigest(root, pr.merge_commit_sha) !== identity.inputDigest) { + return { eligible: false, reason: 'Merged source inputs changed since testing; update the branch and rerun CI.' }; + } + const summary = formatCiSummary(root, bundle); + const spec = formatCiSpec(root, bundle); + if (Buffer.byteLength(summary) > 100_000) throw new Error('Generated assessment summary exceeds its publication limit.'); + if (Buffer.byteLength(spec) > 100_000) throw new Error('Generated SPEC exceeds its publication limit.'); + if ( + gitFile(root, identity.headCommit, assessmentSummaryPath) === summary && + gitFile(root, identity.headCommit, assessmentSpecPath) === spec + ) { + return { eligible: false, reason: 'The exact assessment is already present; no duplicate commit was made.' }; + } + return { + eligible: true, + repository: identity.repository, + branch: identity.headRef, + expectedHead: identity.headCommit, + summary, + spec, + sourceRunId: identity.runId, + }; +} + +export async function publishAssessment(publication: Publication, github: AssessmentGitHub): Promise { + if (!publication.eligible) throw new Error('Assessment publication was not validated.'); + const response = await github.request<{ + data?: { createCommitOnBranch?: { commit?: { oid?: string } } }; + errors?: unknown[]; + }>('/graphql', { + method: 'POST', + body: JSON.stringify({ + query: 'mutation($input: CreateCommitOnBranchInput!) { createCommitOnBranch(input: $input) { commit { oid } } }', + variables: { + input: { + branch: { repositoryNameWithOwner: publication.repository, branchName: publication.branch }, + expectedHeadOid: publication.expectedHead, + message: { + headline: 'chore: update Checkbox Win32 assessment', + body: `Auto-generated from CI run ${publication.sourceRunId}. No source inputs changed.`, + }, + fileChanges: { + additions: [ + { path: assessmentSummaryPath, contents: Buffer.from(publication.summary).toString('base64') }, + { path: assessmentSpecPath, contents: Buffer.from(publication.spec).toString('base64') }, + ], + }, + }, + }, + }), + }); + if (response.errors?.length) throw new Error('GitHub rejected the assessment commit; the branch may have moved or prohibit writes.'); + return hex(response.data?.createCommitOnBranch?.commit?.oid, 40, 'Published assessment commit'); +} + +export function readBundle(file: string): CiBundle { + return parseBundle(readFileSync(file, 'utf8')); +} diff --git a/scripts/src/tasks/controlAssessmentSpec.ts b/scripts/src/tasks/controlAssessmentSpec.ts new file mode 100644 index 0000000000..57ee8c402f --- /dev/null +++ b/scripts/src/tasks/controlAssessmentSpec.ts @@ -0,0 +1,177 @@ +import type { ControlAssessment } from './controlAssessment.ts'; + +export const assessmentSpecPath = 'packages/agentic/components/src/components/checkbox/SPEC.md'; +export const assessmentPackagePath = 'packages/agentic/components/package.json'; +const begin = '# furn-assessment:begin'; +const end = '# furn-assessment:end'; +const outcomes = ['failed', 'execution-failed', 'insufficient-evidence'] as const; +const provenances = ['missing', 'unverified', 'matched', 'mismatched', 'stale'] as const; +const healths = ['complete', 'stale', 'blocked'] as const; + +type SpecAssessment = { + packageVersion: string; + sourceCommit: string; + testedCommit: string | null; + dirty: boolean; + generatedAt: string; + outcome: (typeof outcomes)[number]; + provenance: (typeof provenances)[number]; + health: (typeof healths)[number]; +}; + +function member(value: unknown, values: readonly T[]): T { + const result = values.find((entry) => entry === value); + if (result === undefined) throw new Error('Unsupported generated SPEC assessment value.'); + return result; +} + +function validate(value: SpecAssessment): void { + if ( + typeof value.packageVersion !== 'string' || + value.packageVersion.length > 100 || + !/^\d+\.\d+\.\d+(?:-[A-Za-z0-9.-]+)?(?:\+[A-Za-z0-9.-]+)?$/.test(value.packageVersion) || + !/^[0-9a-f]{40}$/.test(value.sourceCommit) || + typeof value.dirty !== 'boolean' || + !/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z$/.test(value.generatedAt) || + !Number.isFinite(Date.parse(value.generatedAt)) || + new Date(value.generatedAt).toISOString() !== value.generatedAt + ) { + throw new Error('Invalid generated SPEC assessment identity.'); + } + member(value.outcome, outcomes); + member(value.provenance, provenances); + member(value.health, healths); + if (value.testedCommit !== (value.provenance === 'matched' && !value.dirty ? value.sourceCommit : null)) { + throw new Error('Only matched, clean execution evidence can identify a tested SPEC commit.'); + } +} + +function fields(value: SpecAssessment): [string, string | number | boolean | null][] { + validate(value); + return [ + ['assessment-schema', 1], + ['assessment-package-version', value.packageVersion], + ['assessment-source-commit', value.sourceCommit], + ['assessment-tested-commit', value.testedCommit], + ['assessment-dirty', value.dirty], + ['assessment-target', 'win32-paper-storybook'], + ['assessment-outcome', value.outcome], + ['assessment-health', value.health], + ['assessment-stage', 'not-determined'], + ['assessment-provenance', value.provenance], + ['assessment-generated-at', value.generatedAt], + ['assessment-report', './assessment/ASSESSMENT.md'], + ]; +} + +function block(value: SpecAssessment, eol: string): string { + return [ + begin, + ...fields(value).map(([key, scalar]) => `${key}: ${typeof scalar === 'string' ? `'${scalar}'` : JSON.stringify(scalar)}`), + end, + '', + ].join(eol); +} + +function header(spec: string): { eol: string; close: number } { + const eol = spec.startsWith('---\r\n') ? '\r\n' : '\n'; + if (spec.length > 1_000_000 || !spec.startsWith(`---${eol}`)) throw new Error('SPEC must start with bounded YAML front matter.'); + const delimiter = `${eol}---`; + const match = spec.indexOf(`${delimiter}${eol}`, 3); + const close = match >= 0 ? match + eol.length : spec.endsWith(delimiter) ? spec.length - 3 : -1; + if (close < 0) throw new Error('SPEC YAML front matter is unterminated.'); + return { eol, close }; +} + +export function splitSpecAssessment(spec: string): { contract: string; assessment?: SpecAssessment } { + const { eol, close } = header(spec); + const start = spec.indexOf(begin); + const finish = spec.indexOf(end); + let contract = spec; + let assessment: SpecAssessment | undefined; + if (start !== -1 || finish !== -1) { + const after = finish + end.length + eol.length; + if ( + start < 3 + eol.length || + finish <= start || + after > close || + spec[start - 1] !== '\n' || + spec.slice(finish - eol.length, finish) !== eol || + spec.slice(finish + end.length, after) !== eol || + spec.indexOf(begin, start + begin.length) !== -1 || + spec.indexOf(end, finish + end.length) !== -1 + ) { + throw new Error('Generated SPEC assessment must be one complete front-matter block.'); + } + const raw = spec.slice(start, after); + const values: Record = Object.create(null); + for (const line of raw.split(eol).slice(1, -2)) { + const separator = line.indexOf(': '); + if (separator < 1) throw new Error('Malformed generated SPEC assessment field.'); + const key = line.slice(0, separator); + if (Object.hasOwn(values, key)) throw new Error('Duplicate generated SPEC assessment field.'); + const scalar = line.slice(separator + 2); + values[key] = scalar.startsWith("'") && scalar.endsWith("'") ? scalar.slice(1, -1) : JSON.parse(scalar); + } + if ( + values['assessment-schema'] !== 1 || + values['assessment-target'] !== 'win32-paper-storybook' || + values['assessment-stage'] !== 'not-determined' || + values['assessment-report'] !== './assessment/ASSESSMENT.md' || + typeof values['assessment-package-version'] !== 'string' || + typeof values['assessment-source-commit'] !== 'string' || + (values['assessment-tested-commit'] !== null && typeof values['assessment-tested-commit'] !== 'string') || + typeof values['assessment-dirty'] !== 'boolean' || + typeof values['assessment-generated-at'] !== 'string' + ) { + throw new Error('Invalid generated SPEC assessment schema or identity.'); + } + assessment = { + packageVersion: values['assessment-package-version'], + sourceCommit: values['assessment-source-commit'], + testedCommit: values['assessment-tested-commit'], + dirty: values['assessment-dirty'], + generatedAt: values['assessment-generated-at'], + outcome: member(values['assessment-outcome'], outcomes), + provenance: member(values['assessment-provenance'], provenances), + health: member(values['assessment-health'], healths), + }; + if (block(assessment, eol) !== raw) throw new Error('Generated SPEC assessment is not its exact canonical rendering.'); + contract = spec.slice(0, start) + spec.slice(after); + } + if (/^assessment-/m.test(contract.slice(0, header(contract).close))) { + throw new Error('Assessment fields must appear only in the generated SPEC assessment block.'); + } + return { contract, assessment }; +} + +export function updateSpecAssessment(spec: string, assessment: SpecAssessment): string { + const { contract } = splitSpecAssessment(spec); + const { eol, close } = header(contract); + return contract.slice(0, close) + block(assessment, eol) + contract.slice(close); +} + +export function specAssessmentFor(report: ControlAssessment, packageVersion: string): SpecAssessment { + if ( + report.component !== 'checkbox' || + report.target.endpoint !== 'win32' || + report.target.renderer !== 'paper' || + report.target.host !== 'storybook' || + report.evidenceSupportedStatus !== null || + report.approvedStatus !== null + ) { + throw new Error('The SPEC assessment pilot supports only unassigned Checkbox Win32/Paper maturity.'); + } + const value: SpecAssessment = { + packageVersion, + sourceCommit: report.target.sourceCommit, + testedCommit: report.evidence.status === 'matched' && !report.target.workingTreeDirty ? report.target.sourceCommit : null, + dirty: report.target.workingTreeDirty, + generatedAt: report.generatedAt, + outcome: member(report.assessmentOutcome, outcomes), + provenance: member(report.evidence.status, provenances), + health: member(report.assessmentHealth, healths), + }; + validate(value); + return value; +} diff --git a/scripts/src/tasks/controlAssessmentUnit.test.ts b/scripts/src/tasks/controlAssessmentUnit.test.ts new file mode 100644 index 0000000000..9f2833be73 --- /dev/null +++ b/scripts/src/tasks/controlAssessmentUnit.test.ts @@ -0,0 +1,601 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, it } from 'node:test'; +import { fileURLToPath } from 'node:url'; + +import { + assessCheckbox, + checkboxNativeAssertionPath, + formatAssessmentMarkdown, + parseNativeReport, + readCheckboxContract, + reportDigest, +} from './controlAssessment.ts'; +import type { AssessmentOptions } from './controlAssessment.ts'; +import { checkboxJestFixture, componentJestFixture } from './controlAssessmentFixtures.ts'; +import { specAssessmentFor } from './controlAssessmentSpec.ts'; +import { + checkboxUnitCases, + checkboxUnitTestPath, + parseCheckboxJest, + parseUnitContext, + readCheckboxUnitMappingFrom, + unitMappingFiles, +} from './controlAssessmentUnit.ts'; +import { runCheckboxUnitTests, runJestWithUnitCapture } from './controlAssessmentUnitExecution.ts'; + +const root = fileURLToPath(new URL('../../../', import.meta.url)); +const read = (file: string) => readFileSync(path.join(root, file), 'utf8'); +const contract = readCheckboxContract(root); +const mapping = readCheckboxUnitMappingFrom(read); +const sourceCommit = 'a'.repeat(40); +const now = new Date('2026-10-21T20:00:03.000Z'); +const directories: string[] = []; + +function options(data = checkboxJestFixture()): AssessmentOptions { + const report = JSON.stringify(data); + return { + contract, + sourceCommit, + workingTreeDirty: false, + now, + maxAgeHours: 24, + unit: { + mapping, + report, + expectedBuildId: 'synthetic-unit-build', + context: parseUnitContext({ + schemaVersion: 1, + platform: 'macos', + sourceCommit, + workingTreeDirty: false, + buildId: 'synthetic-unit-build', + contractDigest: contract.digest, + mappingDigest: mapping.digest, + reportSha256: reportDigest(report), + startedAt: '2026-10-21T20:00:00.000Z', + finishedAt: '2026-10-21T20:00:02.000Z', + exitCode: 0, + }), + }, + }; +} + +afterEach(() => { + for (const directory of directories.splice(0)) rmSync(directory, { recursive: true, force: true }); +}); + +describe('Checkbox JavaScript unit evidence', () => { + it('maps Checkbox from a complete existing package report without counting its type suite', () => { + const parsed = parseCheckboxJest(JSON.stringify(componentJestFixture())); + assert.equal(parsed.packageSuites, 2); + assert.equal(parsed.packageCounts.passed, 43); + assert.equal(parsed.counts.passed, 42); + assert.equal(parsed.entries.length, 42); + const report = assessCheckbox(options(componentJestFixture())); + assert.equal(report.unitEvidence.outcome, 'passed'); + assert.equal(report.requirements[5].outcome, 'insufficient-evidence'); + assert.equal(specAssessmentFor(report, '0.7.0').testedCommit, null); + }); + + it('counts duplicate titles in unrelated parameterized suites without using them as Checkbox identities', () => { + for (const failed of [false, true]) { + const data = componentJestFixture(); + const other = data.testResults[1]; + other.name = 'packages/agentic/components/src/components/menu-button/menu-button.test.tsx'; + other.assertionResults.push({ ...other.assertionResults[0], failureMessages: [] }); + data.numTotalTests++; + data.numPassedTests++; + if (failed) { + other.assertionResults[1].status = 'failed'; + other.assertionResults[1].failureMessages.push('Synthetic unrelated duplicate-title failure'); + other.status = 'failed'; + data.success = false; + data.numPassedTests--; + data.numFailedTests++; + data.numPassedTestSuites--; + data.numFailedTestSuites++; + } + const report = assessCheckbox(options(data)); + assert.equal(report.unitEvidence.caseCounts.passed, 42); + assert.equal(report.unitEvidence.checks.length, 42); + assert.equal(report.unitEvidence.unmappedCases.length, 0); + assert.deepEqual(report.unitEvidence.packageCounts, { passed: failed ? 43 : 44, failed: failed ? 1 : 0, skipped: 0 }); + assert.equal(report.unitEvidence.outcome, failed ? 'execution-failed' : 'passed'); + } + }); + + it('retains a failed enclosing package run without attributing another suite failure to Checkbox', () => { + const data = componentJestFixture(); + data.success = false; + data.numPassedTests--; + data.numFailedTests++; + data.numPassedTestSuites--; + data.numFailedTestSuites++; + data.testResults[1].status = 'failed'; + data.testResults[1].assertionResults[0].status = 'failed'; + data.testResults[1].assertionResults[0].failureMessages.push('Synthetic other-suite failure'); + const report = assessCheckbox(options(data)); + assert.equal(report.unitEvidence.packageCounts!.failed, 1); + assert.equal(report.unitEvidence.caseCounts.failed, 0); + assert.equal(report.unitEvidence.caseCounts.passed, 42); + assert.equal(report.unitEvidence.outcome, 'execution-failed'); + assert.ok(report.requirements.every(({ outcome }) => outcome === 'insufficient-evidence')); + }); + + it('reviews all 42 exact cases but excludes other-platform actions and macOS focus from Win32 coverage', () => { + assert.equal(mapping.current, true); + assert.equal(checkboxUnitCases.length, 42); + assert.equal(new Set(checkboxUnitCases.map(({ fullName }) => fullName)).size, 42); + const report = assessCheckbox(options()); + assert.equal(report.schemaVersion, 2); + assert.equal(report.unitEvidence.status, 'matched'); + assert.equal(report.unitEvidence.outcome, 'passed'); + assert.equal(report.unitEvidence.caseCounts.passed, 42); + assert.equal(report.unitEvidence.checks.filter(({ requirements }) => requirements.length === 0).length, 17); + assert.deepEqual( + report.requirements.map(({ outcome }) => outcome), + ['partial', 'partial', 'partial', 'partial', 'partial', 'insufficient-evidence'], + ); + assert.deepEqual(report.requirements[5].unitChecks, []); + assert.equal(report.evidence.status, 'missing'); + assert.equal(report.assessmentOutcome, 'insufficient-evidence'); + assert.equal(report.evidenceSupportedStatus, null); + assert.equal(report.approvedStatus, null); + assert.equal(specAssessmentFor(report, '0.7.0').testedCommit, null); + const markdown = formatAssessmentMarkdown(report); + assert.match(markdown, /macOS Jest mock/); + assert.match(markdown, /not native Win32/); + assert.match(markdown, /Not mapped to Win32/); + assert.match(markdown, /0 \/ 0 mapped/); + }); + + it('pins assertion and mock configuration bytes, not merely familiar test names', () => { + for (const changed of unitMappingFiles) { + assert.equal( + readCheckboxUnitMappingFrom((file) => read(file) + (file === changed ? '\n// changed assertion or configuration\n' : '')).current, + false, + ); + } + assert.equal(readCheckboxUnitMappingFrom((file) => read(file).replace(/\r?\n/g, '\r\n')).current, true); + assert.equal( + readCheckboxUnitMappingFrom((file) => + file.endsWith('/package.json') ? JSON.stringify({ ...JSON.parse(read(file)), furn: { jestPlatform: 'win32' } }) : read(file), + ).current, + false, + ); + const input = options(); + input.unit!.mapping = { ...mapping, current: false }; + const report = assessCheckbox(input); + assert.equal(report.assessmentHealth, 'stale'); + assert.equal(report.unitEvidence.outcome, 'stale'); + assert.ok(report.unitEvidence.checks.every(({ evidenceOutcome }) => evidenceOutcome === 'stale')); + }); + + it('keeps a passing suite without execution context unverified', () => { + const input = options(); + input.unit!.context = undefined; + const report = assessCheckbox(input); + assert.equal(report.unitEvidence.status, 'unverified'); + assert.equal(report.unitEvidence.caseCounts.passed, 42); + assert.ok(report.requirements.every(({ outcome }) => outcome === 'insufficient-evidence')); + }); + + for (const missing of ['dirty target', 'dirty execution', 'build', 'age'] as const) { + it(`does not qualify unit evidence with ${missing}`, () => { + const input = options(); + if (missing === 'dirty target') input.workingTreeDirty = true; + if (missing === 'dirty execution') input.unit!.context!.workingTreeDirty = true; + if (missing === 'build') input.unit!.expectedBuildId = undefined; + if (missing === 'age') input.maxAgeHours = undefined; + assert.equal(assessCheckbox(input).unitEvidence.status, 'unverified'); + }); + } + + for (const field of ['sourceCommit', 'buildId', 'contractDigest', 'mappingDigest', 'reportSha256', 'startedAt', 'finishedAt'] as const) { + it(`blocks mismatched unit ${field}`, () => { + const input = options(); + const context = input.unit!.context!; + if (field === 'startedAt') context.startedAt = '2026-10-21T20:00:00.900Z'; + else if (field === 'finishedAt') context.finishedAt = '2026-10-21T20:01:00.000Z'; + else context[field] = field === 'buildId' ? 'different-build' : 'f'.repeat(field === 'sourceCommit' ? 40 : 64); + const report = assessCheckbox(input); + assert.equal(report.unitEvidence.status, 'mismatched'); + assert.equal(report.assessmentHealth, 'blocked'); + assert.ok(report.requirements.every(({ outcome }) => outcome === 'insufficient-evidence')); + }); + } + + it('preserves stale execution and contract mapping state independently', () => { + const input = options(); + input.now = new Date('2026-10-23T20:00:00.000Z'); + assert.equal(assessCheckbox(input).unitEvidence.status, 'stale'); + input.now = now; + input.contract = { ...contract, mappingCurrent: false }; + const report = assessCheckbox(input); + assert.equal(report.assessmentHealth, 'stale'); + assert.ok(report.unitEvidence.checks.every(({ evidenceOutcome }) => evidenceOutcome === 'stale')); + }); + + it('does not invent a passing default assertion when it is missing or replaced by an unmapped name', () => { + for (const replace of [false, true]) { + const data = checkboxJestFixture(); + if (replace) { + Object.assign(data.testResults[0].assertionResults[0], { fullName: 'Checkbox different assertion', title: 'different assertion' }); + } else { + data.testResults[0].assertionResults.shift(); + data.numTotalTests--; + data.numPassedTests--; + } + const report = assessCheckbox(options(data)); + assert.equal(report.requirements[0].outcome, 'insufficient-evidence'); + assert.equal(report.unitEvidence.checks[0].recordedOutcome, 'not-run'); + assert.equal(report.unitEvidence.unmappedCases.length, replace ? 1 : 0); + } + }); + + it('keeps skipped and todo assertions distinct from passing execution', () => { + for (const status of ['pending', 'todo']) { + const data = checkboxJestFixture(); + data.testResults[0].assertionResults[0].status = status; + data.testResults[0].status = status === 'pending' ? 'focused' : 'passed'; + data.numPassedTests--; + if (status === 'todo') data.numTodoTests++; + else data.numPendingTests++; + const report = assessCheckbox(options(data)); + assert.equal(report.unitEvidence.caseCounts.skipped, 1); + assert.equal(report.unitEvidence.checks[0].evidenceOutcome, 'skipped'); + assert.equal(report.requirements[0].outcome, 'insufficient-evidence'); + } + }); + + it('preserves partially skipped unrelated suites using the Jest focused status', () => { + const data = componentJestFixture(); + const other = data.testResults[1]; + other.status = 'focused'; + other.assertionResults.push({ + fullName: 'Other pending assertion', + ancestorTitles: ['Other'], + title: 'pending assertion', + status: 'pending', + failureMessages: [], + }); + data.numTotalTests++; + data.numPendingTests++; + const report = assessCheckbox(options(data)); + assert.deepEqual(report.unitEvidence.packageCounts, { passed: 43, failed: 0, skipped: 1 }); + assert.equal(report.unitEvidence.caseCounts.skipped, 0); + assert.equal(report.unitEvidence.outcome, 'passed'); + }); + + it('preserves retry-success history without qualifying its enclosing execution', () => { + for (const metadata of [ + { invocations: 2, retryReasons: ['Prior synthetic '] }, + { invocations: 2 }, + { retryReasons: ['Prior synthetic '] }, + ]) { + const data = checkboxJestFixture(); + Object.assign(data.testResults[0].assertionResults[0], metadata); + const report = assessCheckbox(options(data)); + const check = report.unitEvidence.checks[0]; + assert.equal(check.recordedOutcome, 'passed'); + assert.equal(check.evidenceOutcome, 'unverified'); + assert.equal(check.invocations, metadata.invocations ?? null); + assert.deepEqual(check.retryReasons, metadata.retryReasons ?? []); + assert.equal(check.retries, 1); + assert.equal(check.flaky, true); + assert.deepEqual(report.unitEvidence.retryCounts, { retries: 1, flaky: 1 }); + assert.equal(report.unitEvidence.runFailed, false); + assert.equal(report.unitEvidence.runRetried, true); + assert.equal(report.unitEvidence.outcome, 'retried'); + assert.ok(report.requirements.every(({ outcome }) => outcome === 'insufficient-evidence')); + const markdown = formatAssessmentMarkdown(report); + assert.match(markdown, /Retried or flaky unit execution/); + if (metadata.retryReasons) assert.ok(markdown.includes('Prior synthetic \\')); + } + }); + + it('retains unrelated package retries without attributing them to Checkbox cases', () => { + const data = componentJestFixture(); + data.testResults[1].assertionResults[0].invocations = 2; + const report = assessCheckbox(options(data)); + assert.deepEqual(report.unitEvidence.retryCounts, { retries: 0, flaky: 0 }); + assert.deepEqual(report.unitEvidence.packageRetryCounts, { retries: 1, flaky: 1 }); + assert.equal(report.unitEvidence.outcome, 'retried'); + assert.ok(report.unitEvidence.checks.every(({ retries, evidenceOutcome }) => retries === 0 && evidenceOutcome === 'unverified')); + }); + + it('does not call an all-skipped suite passed', () => { + const data = checkboxJestFixture(); + data.testResults[0].status = 'skipped'; + data.numPassedTestSuites = 0; + data.numPendingTestSuites = 1; + data.testResults[0].startTime = 0; + data.testResults[0].endTime = 0; + for (const entry of data.testResults[0].assertionResults) entry.status = 'pending'; + data.numPassedTests = 0; + data.numPendingTests = 42; + const report = assessCheckbox(options(data)); + assert.equal(report.unitEvidence.outcome, 'skipped'); + assert.ok(report.requirements.every(({ outcome }) => outcome === 'insufficient-evidence')); + }); + + it('blocks mapped failures and never accepts other passes from the enclosing failed unit command', () => { + const data = checkboxJestFixture(); + const entry = data.testResults[0].assertionResults.find(({ fullName }) => fullName.includes('disables interaction'))!; + entry.status = 'failed'; + entry.failureMessages = ['Synthetic assertion failure']; + entry.invocations = 2; + entry.retryReasons = ['Prior synthetic assertion failure']; + data.success = false; + data.testResults[0].status = 'failed'; + data.numPassedTestSuites = 0; + data.numFailedTestSuites = 1; + data.numPassedTests--; + data.numFailedTests++; + const report = assessCheckbox(options(data)); + assert.equal(report.unitEvidence.outcome, 'failed'); + assert.equal(report.assessmentOutcome, 'failed'); + assert.equal(report.requirements[1].outcome, 'blocked-by-failure'); + assert.equal(report.requirements[3].outcome, 'blocked-by-failure'); + assert.equal(report.requirements[4].outcome, 'insufficient-evidence'); + assert.match(formatAssessmentMarkdown(report), /Synthetic assertion failure/); + assert.equal(specAssessmentFor(report, '0.7.0').outcome, 'failed'); + }); + + it('retains runner/parent-command failure even if all recorded assertions passed', () => { + for (const failure of ['interrupted', 'unsuccessful', 'parent command', 'exit code']) { + const data = checkboxJestFixture(); + if (failure === 'interrupted') data.wasInterrupted = true; + if (failure === 'unsuccessful') data.success = false; + const input = options(data); + if (failure === 'parent command') input.unit!.executionFailure = 'Declared workspace script failed.'; + if (failure === 'exit code') input.unit!.context!.exitCode = 1; + const report = assessCheckbox(input); + assert.equal(report.unitEvidence.outcome, 'execution-failed'); + assert.equal(report.assessmentOutcome, 'execution-failed'); + assert.ok(report.requirements.every(({ outcome }) => outcome === 'insufficient-evidence')); + } + }); + + it('does not let passing unit evidence hide a bound native failure', () => { + const input = options(); + const native = JSON.stringify({ + endpoint: 'win32', + platformManifestDigest: 'b'.repeat(64), + status: 'failed', + tests: [ + { + storyId: 'components-checkbox--default', + testName: 'toggles native checked state in both directions', + status: 'failed', + durationMs: 1, + }, + ], + }); + input.report = parseNativeReport(JSON.parse(native)); + input.reportSha256 = reportDigest(native); + input.expectedBuildId = 'synthetic-native-build'; + input.context = { + schemaVersion: 1, + reportSha256: input.reportSha256, + sourceCommit, + buildId: input.expectedBuildId, + contractDigest: contract.digest, + platformManifestDigest: 'b'.repeat(64), + endpoint: 'win32', + renderer: 'paper', + targetId: 'agenticstorybook-win32', + finishedAt: '2026-10-21T20:00:02.000Z', + physicalClick: true, + }; + const report = assessCheckbox(input); + assert.equal(report.unitEvidence.outcome, 'passed'); + assert.equal(report.assessmentOutcome, 'failed'); + assert.equal(report.requirements[1].outcome, 'blocked-by-failure'); + assert.equal(report.requirements[3].outcome, 'blocked-by-failure'); + assert.equal(report.requirements[0].outcome, 'partial'); + }); + + it('records startup/runtime failure without manufacturing completed assertion timestamps', () => { + const data = checkboxJestFixture(); + Object.assign(data, { + success: false, + numRuntimeErrorTestSuites: 1, + numPassedTestSuites: 0, + numFailedTestSuites: 1, + numTotalTests: 0, + numPassedTests: 0, + }); + Object.assign(data.testResults[0], { status: 'failed', startTime: 0, endTime: 0, assertionResults: [] }); + const report = assessCheckbox(options(data)); + assert.equal(report.unitEvidence.outcome, 'execution-failed'); + assert.equal(report.unitEvidence.caseCounts.passed, 0); + assert.ok(report.requirements.every(({ outcome }) => outcome === 'insufficient-evidence')); + }); + + it('rejects invalid schemas, type-suite substitution, duplicates, contradictory counters, and oversized reports', () => { + const mutations = [ + (data: ReturnType) => { + data.testResults[0].name = 'checkbox.types.test.ts'; + }, + (data: ReturnType) => { + data.testResults.push(data.testResults[0]); + }, + (data: ReturnType) => { + data.testResults[0].assertionResults.push(data.testResults[0].assertionResults[0]); + }, + (data: ReturnType) => { + data.numPassedTests = 900; + }, + (data: ReturnType) => { + data.testResults[0].assertionResults[0].status = 'unknown'; + }, + (data: ReturnType) => { + data.testResults[0].assertionResults[0].title = 'wrong identity'; + }, + (data: ReturnType) => { + data.testResults[0].assertionResults[0].failureMessages = ['contradiction']; + }, + (data: ReturnType) => { + data.numRuntimeErrorTestSuites = 1; + }, + (data: ReturnType) => { + data.testResults[0].endTime = 1; + }, + (data: ReturnType) => { + data.numFailedTestSuites = 1; + }, + (data: ReturnType) => { + Object.assign(data.testResults[0].assertionResults[0], { invocations: -1 }); + }, + (data: ReturnType) => { + Object.assign(data.testResults[0].assertionResults[0], { retryReasons: 'invalid' }); + }, + (data: ReturnType) => { + Object.assign(data.testResults[0].assertionResults[0], { retryReasons: [null] }); + }, + (data: ReturnType) => { + data.testResults[0].status = 'focused'; + }, + (data: ReturnType) => { + data.testResults[0].status = 'skipped'; + data.numPassedTestSuites = 0; + data.numPendingTestSuites = 1; + }, + ]; + for (const mutate of mutations) { + const data = checkboxJestFixture(); + mutate(data); + assert.throws(() => parseCheckboxJest(JSON.stringify(data))); + } + assert.throws(() => parseCheckboxJest('{}')); + assert.throws(() => parseCheckboxJest(' '.repeat(1_000_001)), /size limit/); + const input = options(); + assert.throws(() => parseUnitContext({ ...input.unit!.context, platform: 'win32' }), /platform/); + assert.throws(() => parseUnitContext({ ...input.unit!.context, exitCode: -1 }), /exit code/); + assert.throws(() => parseUnitContext({ ...input.unit!.context, finishedAt: '2026-10-20T00:00:00.000Z' }), /before/); + input.unit!.report = undefined; + assert.throws(() => assessCheckbox(input), /requires a unit report/); + }); +}); + +describe('unit execution capture', () => { + it('leaves uncaptured and other-platform package execution unchanged and calls the existing runner once', async () => { + const args = ['--runTestsByPath', 'existing.test.ts']; + for (const [platform, env] of [ + ['macos', {}], + ['windows', { FURN_ASSESSMENT_UNIT_CAPTURE: 'unused' }], + ] as const) { + let invocations = 0; + const code = await runJestWithUnitCapture( + 'unused', + platform, + args, + async (forwarded) => { + invocations++; + assert.deepEqual(forwarded, args); + return 7; + }, + env, + ); + assert.equal(code, 7); + assert.equal(invocations, 1); + } + }); + + function fixture(mode = 'passed') { + const directory = mkdtempSync(path.join(tmpdir(), 'furn-unit-capture-')); + directories.push(directory); + const put = (file: string, content: string) => { + mkdirSync(path.dirname(path.join(directory, file)), { recursive: true }); + writeFileSync(path.join(directory, file), content); + }; + for (const file of [...unitMappingFiles, 'packages/agentic/components/package.json']) put(file, read(file)); + put(checkboxNativeAssertionPath, read(checkboxNativeAssertionPath)); + for (const file of [ + 'SPEC.md', + 'spec/source.json', + 'spec/interaction.md', + 'spec/accessibility.md', + 'spec/tokens.yaml', + 'spec/usage.md', + 'checkbox.stories.tsx', + ]) { + const relative = path.posix.join(path.posix.dirname(checkboxUnitTestPath), file); + put(relative, read(relative)); + } + put('package.json', '{"packageManager":"yarn@0.0.0"}'); + put('.gitignore', 'apps/storybook/artifacts/\n'); + put('source.ts', 'original\n'); + put('fixture-unit.json', JSON.stringify({ ...checkboxJestFixture(), fixtureMode: mode })); + // This synthetic package-manager fixture exercises capture, not component execution. + put( + '.yarn/releases/yarn-0.0.0.cjs', + ` +const fs = require('node:fs'); +const path = require('node:path'); +const data = JSON.parse(fs.readFileSync('fixture-unit.json', 'utf8')); +if (data.fixtureMode === 'missing') process.exit(0); +const now = Date.now(); +data.startTime = now; +data.testResults[0].startTime = now; +data.testResults[0].endTime = now; +const destination = process.argv[process.argv.indexOf('--outputFile') + 1]; +fs.writeFileSync(destination, JSON.stringify(data)); +if (data.fixtureMode === 'changed') fs.writeFileSync('source.ts', 'changed\\n'); +if (data.fixtureMode === 'failed') process.exitCode = 1; +`, + ); + const git = (...args: string[]) => execFileSync('git', args, { cwd: directory, encoding: 'utf8' }).trim(); + git('init', '-q'); + git('config', 'user.name', 'Fixture'); + git('config', 'user.email', 'fixture@example.invalid'); + git('config', 'core.autocrlf', 'false'); + git('config', 'commit.gpgsign', 'false'); + git('add', '.'); + git('commit', '-qm', 'Synthetic unit capture inputs'); + return { directory, output: path.join(directory, 'apps', 'storybook', 'artifacts', 'unit') }; + } + + it('captures exact bytes and source before execution, with a fresh output directory for each run', () => { + const f = fixture(); + const first = runCheckboxUnitTests(f.directory, f.output); + const second = runCheckboxUnitTests(f.directory, f.output); + assert.equal(first.exitCode, 0); + assert.equal(first.context!.workingTreeDirty, false); + assert.equal(first.context!.platform, 'macos'); + assert.equal(first.context!.reportSha256, reportDigest(first.report!)); + assert.notEqual(first.context!.buildId, second.context!.buildId); + assert.equal(readdirSync(f.output).length, 2); + const report = assessCheckbox({ + contract: readCheckboxContract(f.directory), + sourceCommit: first.context!.sourceCommit, + workingTreeDirty: false, + now: new Date(), + maxAgeHours: 24, + unit: { + mapping: readCheckboxUnitMappingFrom((file) => readFileSync(path.join(f.directory, file), 'utf8')), + report: first.report!, + context: first.context!, + expectedBuildId: first.context!.buildId, + }, + }); + assert.equal(report.unitEvidence.status, 'matched'); + }); + + for (const mode of ['missing', 'failed', 'changed']) { + it(`surfaces ${mode} execution instead of returning a successful capture`, () => { + const f = fixture(mode); + const result = runCheckboxUnitTests(f.directory, f.output, 'synthetic-build'); + assert.notEqual(result.exitCode, 0); + assert.ok(result.executionFailure); + if (mode === 'missing') assert.equal(result.report, null); + if (mode === 'changed') assert.equal(result.context?.workingTreeDirty, true); + if (mode === 'failed') assert.equal(result.context?.exitCode, 1); + }); + } +}); diff --git a/scripts/src/tasks/controlAssessmentUnit.ts b/scripts/src/tasks/controlAssessmentUnit.ts new file mode 100644 index 0000000000..6dfe4b6b04 --- /dev/null +++ b/scripts/src/tasks/controlAssessmentUnit.ts @@ -0,0 +1,381 @@ +import { createHash } from 'node:crypto'; + +import { hex, object, reportDigest, text, timestamp } from './controlAssessment.ts'; + +export const checkboxUnitTestPath = 'packages/agentic/components/src/components/checkbox/checkbox.test.tsx'; +const packagePath = 'packages/agentic/components/package.json'; +export const unitMappingFiles = [ + checkboxUnitTestPath, + 'packages/agentic/components/jest.config.cjs', + 'scripts/configs/jest/jest.config.cjs', + 'scripts/configs/jest/babel.config.cjs', + 'scripts/src/const.mts', + 'scripts/src/tasks/jest.ts', + 'scripts/src/tasks/controlAssessmentUnitExecution.ts', +]; +// October 29 review: appended regression cases preserve these 42 assertions; additions remain explicitly unmapped. +const reviewedUnitDigest = 'b12a44cea1897231d43ead2734d32a55e2fea2e5e12921d7e3f4c8bff086e7fa'; +const mappingVersion = 'checkbox-jest-macos-mock-v1'; +const environment = 'React Native macOS Jest mock; explicit platform tables simulate OS values, not native dispatch'; +type Outcome = 'passed' | 'failed' | 'skipped' | 'not-run' | 'unverified' | 'mismatched' | 'stale'; +type Status = 'missing' | 'unverified' | 'matched' | 'mismatched' | 'stale'; +type UnitCase = { + fullName: string; + status: 'passed' | 'failed' | 'skipped'; + reason: string | null; + invocations: number | null; + retryReasons: string[]; + retries: number; + flaky: boolean; +}; + +const cases: { fullName: string; requirements: string[]; scope: string }[] = []; +function mapped(title: string, requirements: string[], scope: string, group = 'Checkbox'): void { + cases.push({ fullName: `${group} ${title}`, requirements, scope }); +} +mapped('renders default checkbox accessibility and secondary styling', ['CBX-001', 'CBX-004', 'CBX-005'], 'Default JS props and styles.'); +mapped( + 'toggles uncontrolled status on press and forwards user handlers', + ['CBX-002'], + 'JS unchecked-to-checked press and forwarded handler.', +); +mapped('reports presses without changing state when the status is externally driven', ['CBX-002'], 'JS controlled ownership on press.'); +for (const platform of ['windows', 'win32', 'macos']) { + const group = `Checkbox ${platform} accessibility actions`; + const requirements = platform === 'win32' ? ['CBX-002', 'CBX-004'] : []; + const scope = `${platform} Platform.OS simulation under the macOS Jest mock; no native accessibility dispatch.`; + for (const status of ['unchecked', 'checked', 'indeterminate']) { + mapped(`toggles ${status} once without synthesizing a press`, requirements, scope, group); + } + for (const title of [ + 'preserves a normalized caller action list and labels', + 'deduplicates legacy and native spellings without losing the caller label or custom actions', + 'reports accessibility toggles without changing externally driven status', + 'blocks disabled accessibility toggles while forwarding the caller handler once', + 'forwards custom, wrong-case, and activate actions without a second activation path', + ]) { + mapped(title, requirements, scope, group); + } +} +mapped('advances indeterminate status to checked on press', ['CBX-002'], 'JS indeterminate-to-checked transition.'); +mapped('disables interaction and exposes disabled accessibility state', ['CBX-002', 'CBX-004'], 'JS disabled props and press suppression.'); +mapped( + 'renders checked and indeterminate indicator glyphs', + ['CBX-003', 'CBX-004'], + 'JS glyph sizes and checked/mixed accessibility props.', +); +mapped('shows secondary text and exposes it as a description', ['CBX-003', 'CBX-004'], 'JS secondary text and accessibilityHint.'); +mapped('keeps the visible label optional while preserving an accessible name', ['CBX-003', 'CBX-004'], 'JS explicit accessible name.'); +mapped('falls back to the label text when the visible label is hidden', ['CBX-003', 'CBX-004'], 'JS hidden-label name fallback.'); +mapped('resolves hover and pressed feedback for label, indicator, and status colors', ['CBX-005'], 'Resolved JS hover/pressed tokens.'); +mapped('uses native focus visuals without mounting custom rings', [], 'macOS mock focus props, not Win32 dual-ring coverage.'); +for (const variant of ['standard', 'circular']) { + mapped(`resolves the ${variant} variant`, ['CBX-005'], 'Resolved JS variant border radius.'); +} +for (const status of ['unchecked', 'checked', 'indeterminate']) { + mapped(`resolves ${status} status colors`, ['CBX-005'], 'Resolved JS status colors.'); +} +mapped('allows constrained label text to wrap', ['CBX-003'], 'JS label flexShrink, not rendered wrapping geometry.'); +mapped('applies disabled colors to the label and indicator', ['CBX-005'], 'Resolved JS disabled colors.'); +export const checkboxUnitCases: readonly (typeof cases)[number][] = cases; + +export type UnitMapping = { digest: string; current: boolean }; +export type UnitContext = { + schemaVersion: 1; + platform: 'macos'; + sourceCommit: string; + workingTreeDirty: boolean; + buildId: string; + contractDigest: string; + mappingDigest: string; + reportSha256: string; + startedAt: string; + finishedAt: string; + exitCode: number | null; +}; +export type UnitInput = { + mapping?: UnitMapping; + report?: string; + context?: UnitContext; + executionFailure?: string; + expectedBuildId?: string; +}; + +export function readCheckboxUnitMappingFrom(read: (file: string) => string): UnitMapping { + const hash = createHash('sha256'); + for (const file of unitMappingFiles) hash.update(`${file}\0${read(file).replace(/\r\n/g, '\n')}\0`); + const manifest = object(JSON.parse(read(packagePath)), 'Unit package manifest'); + const profile = object(manifest.furn, 'Unit package profile').jestPlatform; + const script = object(manifest.scripts, 'Unit package scripts').test; + hash.update(`${packagePath}\0${JSON.stringify({ profile, script })}\0`); + const digest = hash.digest('hex'); + return { digest, current: digest === reviewedUnitDigest }; +} + +function count(value: unknown, label: string): number { + if (!Number.isSafeInteger(value) || Number(value) < 0) throw new Error(`${label} must be a nonnegative integer.`); + return Number(value); +} + +export function parseUnitContext(value: unknown): UnitContext { + const data = object(value, 'Unit execution context'); + if (data.schemaVersion !== 1 || data.platform !== 'macos' || typeof data.workingTreeDirty !== 'boolean') { + throw new Error('Unsupported unit context schema, platform, or dirty flag.'); + } + const startedAt = timestamp(data.startedAt, 'Unit execution start'); + const finishedAt = timestamp(data.finishedAt, 'Unit execution finish'); + if (Date.parse(startedAt) > Date.parse(finishedAt)) throw new Error('Unit execution finished before it started.'); + return { + schemaVersion: 1, + platform: 'macos', + sourceCommit: hex(data.sourceCommit, 40, 'Unit source commit'), + workingTreeDirty: data.workingTreeDirty, + buildId: text(data.buildId, 'Unit build ID'), + contractDigest: hex(data.contractDigest, 64, 'Unit contract digest'), + mappingDigest: hex(data.mappingDigest, 64, 'Unit mapping digest'), + reportSha256: hex(data.reportSha256, 64, 'Unit report hash'), + startedAt, + finishedAt, + exitCode: data.exitCode === null ? null : count(data.exitCode, 'Unit exit code'), + }; +} + +export function parseCheckboxJest(content: string) { + if (Buffer.byteLength(content) > 1_000_000) throw new Error('Unit report exceeds its size limit.'); + const data = object(JSON.parse(content), 'Jest report'); + if (typeof data.success !== 'boolean' || typeof data.wasInterrupted !== 'boolean') throw new Error('Invalid Jest execution flags.'); + if (!Array.isArray(data.testResults) || !data.testResults.length) throw new Error('Jest report must contain test suites.'); + const startTime = count(data.startTime, 'Jest start time'); + const parsed = data.testResults.map((value) => parseJestSuite(value, startTime)); + if (new Set(parsed.map(({ name }) => name)).size !== parsed.length) throw new Error('Duplicate Jest suite identity.'); + const selected = parsed.filter(({ name }) => name === checkboxUnitTestPath || name.endsWith(`/${checkboxUnitTestPath}`)); + if (selected.length !== 1) throw new Error('Unit report must contain exactly one Checkbox runtime suite; type tests cannot substitute.'); + if (new Set(selected[0].entries.map(({ fullName }) => fullName)).size !== selected[0].entries.length) { + throw new Error('Duplicate Jest assertion identity.'); + } + const counts = { passed: 0, failed: 0, skipped: 0 }; + const suites = { passed: 0, failed: 0, skipped: 0 }; + for (const suite of parsed) { + if (suite.status === 'passed' || suite.status === 'focused') suites.passed++; + else if (suite.status === 'failed') suites.failed++; + else suites.skipped++; + for (const status of ['passed', 'failed', 'skipped'] as const) counts[status] += suite.counts[status]; + } + if ( + count(data.numTotalTestSuites, 'Jest total suites') !== parsed.length || + count(data.numPassedTestSuites, 'Jest passed suites') !== suites.passed || + count(data.numFailedTestSuites, 'Jest failed suites') !== suites.failed || + count(data.numPendingTestSuites, 'Jest pending suites') !== suites.skipped || + count(data.numTotalTests, 'Jest total') !== counts.passed + counts.failed + counts.skipped || + count(data.numPassedTests, 'Jest passed') !== counts.passed || + count(data.numFailedTests, 'Jest failed') !== counts.failed || + count(data.numPendingTests, 'Jest pending') + count(data.numTodoTests, 'Jest todo') !== counts.skipped + ) + throw new Error('Jest suite/assertion counts disagree with its summary.'); + const runtimeErrors = count(data.numRuntimeErrorTestSuites, 'Jest runtime errors'); + if (runtimeErrors > suites.failed || (data.success && (counts.failed || runtimeErrors || suites.failed))) { + throw new Error('Jest execution status contradicts its failures.'); + } + const retryCounts = (entries: UnitCase[]) => ({ + retries: entries.reduce((sum, entry) => sum + entry.retries, 0), + flaky: entries.filter(({ flaky }) => flaky).length, + }); + return { + entries: selected[0].entries, + counts: selected[0].counts, + packageCounts: counts, + packageSuites: parsed.length, + retryCounts: retryCounts(selected[0].entries), + packageRetryCounts: retryCounts(parsed.flatMap(({ entries }) => entries)), + startTime, + endTime: Math.max(...parsed.map(({ endTime }) => endTime)), + failed: !data.success || data.wasInterrupted || runtimeErrors > 0 || suites.failed > 0, + }; +} + +function parseJestSuite(value: unknown, startTime: number) { + const suite = object(value, 'Jest suite'); + const name = text(suite.name, 'Jest suite filename').replace(/\\/g, '/'); + const suiteStatus = suite.status; + if (suiteStatus !== 'passed' && suiteStatus !== 'failed' && suiteStatus !== 'skipped' && suiteStatus !== 'focused') { + throw new Error('Unsupported Jest suite status.'); + } + if (!Array.isArray(suite.assertionResults)) throw new Error('Jest suite must contain assertionResults.'); + const entries: UnitCase[] = suite.assertionResults.map((value) => { + const entry = object(value, 'Jest assertion'); + const fullName = text(entry.fullName, 'Jest full name'); + const title = text(entry.title, 'Jest title'); + if ( + !Array.isArray(entry.ancestorTitles) || + !entry.ancestorTitles.every((value) => typeof value === 'string') || + [...entry.ancestorTitles, title].join(' ') !== fullName + ) { + throw new Error('Jest assertion identity is inconsistent.'); + } + if (!['passed', 'failed', 'pending', 'todo'].includes(String(entry.status))) throw new Error('Unsupported Jest assertion status.'); + if (!Array.isArray(entry.failureMessages) || entry.failureMessages.some((message) => typeof message !== 'string')) { + throw new Error('Jest failureMessages must be a string array.'); + } + if (entry.status === 'passed' && entry.failureMessages.length) throw new Error('Passed Jest assertion contains a failure.'); + const invocations = entry.invocations === undefined ? null : count(entry.invocations, 'Jest assertion invocations'); + if ( + entry.retryReasons !== undefined && + (!Array.isArray(entry.retryReasons) || entry.retryReasons.some((reason) => typeof reason !== 'string')) + ) { + throw new Error('Jest retryReasons must be a string array.'); + } + const retryReasons: string[] = entry.retryReasons ?? []; + const retries = Math.max(invocations === null ? 0 : invocations - 1, retryReasons.length); + return { + fullName, + status: entry.status === 'passed' || entry.status === 'failed' ? entry.status : 'skipped', + reason: entry.failureMessages.length ? entry.failureMessages.join('\n').slice(0, 2000) : entry.status === 'todo' ? 'todo' : null, + invocations, + retryReasons, + retries, + flaky: entry.status === 'passed' && retries > 0, + }; + }); + const counts = { + passed: entries.filter(({ status }) => status === 'passed').length, + failed: entries.filter(({ status }) => status === 'failed').length, + skipped: entries.filter(({ status }) => status === 'skipped').length, + }; + if ( + (suiteStatus === 'passed' && counts.failed) || + (suiteStatus === 'focused' && (counts.failed || !counts.skipped)) || + (suiteStatus === 'skipped' && (counts.passed || counts.failed)) + ) { + throw new Error('Jest execution status contradicts its failures.'); + } + const suiteStart = count(suite.startTime, 'Jest suite start time'); + const endTime = count(suite.endTime, 'Jest suite end time'); + const earlyRuntimeFailure = suiteStatus === 'failed' && suiteStart === 0 && endTime === 0 && counts.passed === 0; + if (suite.status !== 'skipped' && !earlyRuntimeFailure && (suiteStart < startTime || endTime < suiteStart)) { + throw new Error('Invalid Jest execution interval.'); + } + return { + name, + status: suiteStatus, + entries, + counts, + startTime, + endTime, + }; +} + +export function assessCheckboxUnits(options: { + input?: UnitInput; + sourceCommit: string; + workingTreeDirty: boolean; + contractDigest: string; + contractMappingCurrent: boolean; + now: Date; + maxAgeHours?: number; +}) { + const { input } = options; + const report = input?.report === undefined ? undefined : parseCheckboxJest(input.report); + if (report && !input?.mapping) throw new Error('Unit report requires the current source mapping fingerprint.'); + const context = input?.context; + if (context && !report) throw new Error('Unit execution context requires a unit report.'); + const hash = input?.report === undefined ? null : reportDigest(input.report); + let status: Status = report ? 'matched' : 'missing'; + const reasons: string[] = []; + if (report) { + if ( + context && + (context.sourceCommit !== options.sourceCommit || + context.reportSha256 !== hash || + context.contractDigest !== options.contractDigest || + context.mappingDigest !== input!.mapping!.digest || + (input?.expectedBuildId !== undefined && context.buildId !== input.expectedBuildId) || + Date.parse(context.finishedAt) > options.now.getTime() || + report.startTime < Date.parse(context.startedAt) || + (report.endTime !== 0 && report.endTime > Date.parse(context.finishedAt))) + ) { + status = 'mismatched'; + reasons.push('Unit report bytes, source, contract, mapping, build, or execution interval do not match their binding.'); + } else if ( + context && + options.maxAgeHours !== undefined && + options.now.getTime() - Date.parse(context.finishedAt) > options.maxAgeHours * 3_600_000 + ) { + status = 'stale'; + reasons.push('Unit evidence exceeds the explicitly requested maximum age.'); + } else if ( + !context || + options.workingTreeDirty || + context.workingTreeDirty || + !input?.expectedBuildId || + options.maxAgeHours === undefined + ) { + status = 'unverified'; + reasons.push( + 'Unit evidence needs an execution-generated context, expected unit build, age policy, and unchanged clean source inputs.', + ); + } + } + const mappingCurrent = input?.mapping?.current ?? null; + if (mappingCurrent === false) + reasons.push('Unit assertions or mock configuration changed since review; update the reviewed mapping before accepting evidence.'); + const current = mappingCurrent === true && options.contractMappingCurrent; + const runFailed = Boolean(report?.failed || input?.executionFailure || (context && context.exitCode !== 0)); + const runRetried = Boolean(report?.packageRetryCounts.retries); + if (runRetried) reasons.push('Retried or flaky unit execution is not qualifying evidence in this pilot.'); + const checks = checkboxUnitCases.map((mapping) => { + const recorded = report?.entries.find(({ fullName }) => fullName === mapping.fullName); + let evidenceOutcome: Outcome = 'not-run'; + if (recorded) { + if (!current || status === 'stale') evidenceOutcome = 'stale'; + else if (status === 'mismatched') evidenceOutcome = 'mismatched'; + else if (status !== 'matched' || (recorded.status === 'passed' && (runFailed || runRetried))) evidenceOutcome = 'unverified'; + else evidenceOutcome = recorded.status; + } + return { + ...mapping, + recordedOutcome: recorded?.status ?? 'not-run', + evidenceOutcome, + reason: recorded?.reason ?? null, + invocations: recorded?.invocations ?? null, + retryReasons: recorded?.retryReasons ?? [], + retries: recorded?.retries ?? 0, + flaky: recorded?.flaky ?? false, + }; + }); + const outcome = checks.some(({ evidenceOutcome }) => evidenceOutcome === 'failed') + ? 'failed' + : status === 'matched' && current && runFailed + ? 'execution-failed' + : report && !current + ? 'stale' + : status !== 'matched' + ? status === 'missing' + ? 'not-run' + : status + : runRetried + ? 'retried' + : report!.counts.passed === 0 + ? 'skipped' + : 'passed'; + return { + status, + reasons, + outcome, + environment, + mappingVersion, + mappingCurrent, + mappingDigest: input?.mapping?.digest ?? null, + reportSha256: hash, + runFailed, + runRetried, + executionFailure: input?.executionFailure ?? null, + context: context ?? null, + caseCounts: report?.counts ?? { passed: 0, failed: 0, skipped: 0 }, + packageCounts: report?.packageCounts ?? null, + packageSuites: report?.packageSuites ?? null, + retryCounts: report?.retryCounts ?? { retries: 0, flaky: 0 }, + packageRetryCounts: report?.packageRetryCounts ?? null, + checks, + unmappedCases: report?.entries.filter(({ fullName }) => !checkboxUnitCases.some((mapping) => mapping.fullName === fullName)) ?? [], + }; +} diff --git a/scripts/src/tasks/controlAssessmentUnitExecution.ts b/scripts/src/tasks/controlAssessmentUnitExecution.ts new file mode 100644 index 0000000000..05c8537388 --- /dev/null +++ b/scripts/src/tasks/controlAssessmentUnitExecution.ts @@ -0,0 +1,172 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { existsSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; + +import { PLATFORM_ENV_VAR } from '../const.mts'; +import { object, readCheckboxContract, reportDigest, text, timestamp } from './controlAssessment.ts'; +import { checkboxUnitTestPath, parseUnitContext, readCheckboxUnitMappingFrom } from './controlAssessmentUnit.ts'; +import type { UnitContext } from './controlAssessmentUnit.ts'; + +export type UnitExecution = { + schemaVersion: 1; + report: string | null; + context: UnitContext | null; + executionFailure: string | null; + exitCode: number; +}; + +function sourceState(root: string) { + const sourceCommit = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).trim(); + const status = execFileSync('git', ['status', '--porcelain', '--untracked-files=all'], { cwd: root, encoding: 'utf8' }); + const diff = execFileSync('git', ['diff', '--binary', 'HEAD'], { cwd: root, maxBuffer: 32_000_000 }); + return { sourceCommit, workingTreeDirty: status.length > 0, fingerprint: reportDigest(`${status}\0${diff.toString('base64')}`) }; +} + +function beginUnitExecution(root: string, directory: string, expectedBuildId?: string) { + const initial = sourceState(root); + const read = (file: string) => readFileSync(path.join(root, file), 'utf8'); + const mapping = readCheckboxUnitMappingFrom(read); + const contract = readCheckboxContract(root); + const runId = randomUUID(); + const runDirectory = path.join(directory, `unit-${runId}`); + mkdirSync(directory, { recursive: true }); + mkdirSync(runDirectory); + const reportPath = path.join(runDirectory, 'jest.json'); + const startedAt = new Date().toISOString(); + return { + root, + initial, + mapping, + contract, + runDirectory, + reportPath, + startedAt, + buildId: expectedBuildId ?? `local-checkbox-unit/${runId}`, + }; +} + +function finishUnitExecution( + capture: ReturnType, + status: number | null, + launchError?: string, +): UnitExecution & { directory: string } { + const { root, initial, mapping, contract, reportPath, startedAt, runDirectory, buildId } = capture; + const finishedAt = new Date().toISOString(); + const final = sourceState(root); + const current = initial.sourceCommit === final.sourceCommit && initial.fingerprint === final.fingerprint; + const report = existsSync(reportPath) ? readFileSync(reportPath, 'utf8') : null; + const executionFailure = launchError + ? `Checkbox unit command could not start: ${launchError}` + : !current + ? 'Source inputs changed during Checkbox unit execution; its results cannot qualify.' + : status !== 0 + ? `Checkbox unit command failed (exit ${status ?? 'none'}).` + : report === null + ? `Checkbox unit command produced no fresh ${checkboxUnitTestPath} Jest report.` + : null; + const context = + report === null + ? null + : parseUnitContext({ + schemaVersion: 1, + platform: 'macos', + sourceCommit: initial.sourceCommit, + workingTreeDirty: initial.workingTreeDirty || final.workingTreeDirty || !current, + buildId, + contractDigest: contract.digest, + mappingDigest: mapping.digest, + reportSha256: reportDigest(report), + startedAt, + finishedAt, + exitCode: status, + }); + const execution: UnitExecution = { + schemaVersion: 1, + report, + context, + executionFailure, + exitCode: executionFailure ? status || 1 : 0, + }; + writeFileSync(path.join(runDirectory, 'execution.json'), `${JSON.stringify(execution, null, 2)}\n`); + if (context) writeFileSync(path.join(runDirectory, 'context.json'), `${JSON.stringify(context, null, 2)}\n`); + return { ...execution, directory: runDirectory }; +} + +export function runCheckboxUnitTests(root: string, directory: string, expectedBuildId?: string): UnitExecution & { directory: string } { + const manifest = object(JSON.parse(readFileSync(path.join(root, 'package.json'), 'utf8')), 'Root manifest'); + const version = /^yarn@(\d+\.\d+\.\d+)$/.exec(text(manifest.packageManager, 'Package manager'))?.[1]; + if (!version) throw new Error('Unit collection requires the repository-pinned Yarn release.'); + const yarn = path.join(root, '.yarn', 'releases', `yarn-${version}.cjs`); + if (!existsSync(yarn)) throw new Error(`Pinned Yarn entry point is missing: ${yarn}.`); + const capture = beginUnitExecution(root, directory, expectedBuildId); + const env: NodeJS.ProcessEnv = { ...process.env, [PLATFORM_ENV_VAR]: 'macos' }; + delete env.FURN_ASSESSMENT_UNIT_CAPTURE; + const result = spawnSync( + process.execPath, + [ + yarn, + 'workspace', + '@fluentui-react-native/components', + 'test', + 'src/components/checkbox/checkbox.test.tsx', + '--runInBand', + '--json', + '--outputFile', + capture.reportPath, + ], + { cwd: root, env, stdio: 'inherit' }, + ); + return finishUnitExecution(capture, result.status, result.error?.message); +} + +/** Decorates the existing package invocation; the supplied runner is called exactly once. */ +export async function runJestWithUnitCapture( + packageRoot: string, + platform: string, + args: string[], + run: (args: string[]) => Promise, + env: NodeJS.ProcessEnv = process.env, +): Promise { + const directory = env.FURN_ASSESSMENT_UNIT_CAPTURE; + if (!directory || platform !== 'macos') return run(args); + const root = execFileSync('git', ['rev-parse', '--show-toplevel'], { cwd: packageRoot, encoding: 'utf8' }).trim(); + if (path.relative(realpathSync.native(root), realpathSync.native(packageRoot)).replace(/\\/g, '/') !== 'packages/agentic/components') { + return run(args); + } + if (env[PLATFORM_ENV_VAR] !== undefined && env[PLATFORM_ENV_VAR] !== 'macos') + throw new Error('Unit capture requires the declared macOS Jest mock.'); + if (args.some((arg) => arg === '--outputFile' || arg.startsWith('--outputFile=') || arg === '--listTests')) { + throw new Error('Assessment capture cannot replace explicit output/list-only Jest options.'); + } + const destination = path.resolve(directory); + const start = object(JSON.parse(readFileSync(path.join(destination, 'unit-collection-start.json'), 'utf8')), 'Unit collection start'); + if (start.schemaVersion !== 1) throw new Error('Unsupported unit collection capture schema.'); + const identity = object(start.identity, 'Unit collection identity'); + const began = timestamp(start.startedAt, 'Unit collection start time'); + if (existsSync(path.join(destination, 'unit-execution.json'))) + throw new Error('A unit execution was already collected; refusing replacement.'); + const capture = beginUnitExecution(root, destination, text(start.buildId, 'Unit collection build ID')); + if (capture.initial.sourceCommit !== identity.testedCommit || Date.parse(capture.startedAt) < Date.parse(began)) { + throw new Error('Existing unit invocation does not match pre-execution source or interval.'); + } + let exitCode: number | null = null; + try { + exitCode = await run([...args, '--json', '--outputFile', capture.reportPath]); + return exitCode; + } finally { + const execution = finishUnitExecution(capture, exitCode); + writeFileSync( + path.join(destination, 'unit-execution.json'), + `${JSON.stringify( + { + ...execution, + identity, + collectionStartedAt: began, + }, + null, + 2, + )}\n`, + ); + } +} diff --git a/scripts/src/tasks/jest.ts b/scripts/src/tasks/jest.ts index 9acd468616..b1cc890b11 100644 --- a/scripts/src/tasks/jest.ts +++ b/scripts/src/tasks/jest.ts @@ -8,6 +8,7 @@ import { hasJestConfig } from '../preinstall/tool-versions.ts'; import { PackageContext, type PlatformTarget } from '../pkgContext.ts'; import { PLATFORM_ENV_VAR, DEFULT_WIN_PLATFORM } from '../const.mts'; import { getFileMetadata } from '../utils/fileData.ts'; +import { runJestWithUnitCapture } from './controlAssessmentUnitExecution.ts'; import { buildJestOptionTable, matchTestFiles, @@ -103,7 +104,9 @@ export class JestCommand extends Command { if (files && files.length > 0) { console.log(`Running jest for platform ${platform} with ${files.length} test files.`); const options = getPlatformOptions(platform, defaultPlatform); - const result = await runScript('jest', [...args, ...parsed.passthrough, ...files], options); + const result = await runJestWithUnitCapture(context.root, platform, [...args, ...parsed.passthrough, ...files], (jestArgs) => + runScript('jest', jestArgs, options), + ); if (result !== 0) { return result; }