From 0f4971e3869863018b048d5781e9b45e774a4cd6 Mon Sep 17 00:00:00 2001 From: Srividya Ramaswamy Date: Thu, 29 Oct 2026 13:47:40 -0700 Subject: [PATCH 1/9] Add Checkbox readiness assessor pilot using existing CI evidence Add plan.md describing the dedicated Checkbox-first pilot, reviewed mappings, evidence provenance, CI result reuse and optional trusted same-PR publication. Preserve existing source checks and disabled input capabilities; do not activate publishing or include historical generated assessments. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4b66f3ef-85ac-4405-bd35-dcc167a077b1 --- .changeset/happy-lands-sink.md | 3 + .github/workflows/pr-bundle-size-comment.yml | 17 + .github/workflows/pr-control-assessment.yml | 75 ++ .github/workflows/pr.yml | 207 +++- .oxfmtrc.json | 1 + package.json | 1 + packages/agentic/components/package.json | 1 + .../components/references/contracts.md | 12 + plan.md | 112 ++ scripts/CONTROL-ASSESSMENT.md | 646 +++++++++++ scripts/package.json | 2 +- scripts/src/cli.ts | 2 + scripts/src/tasks/assessControlTask.ts | 184 +++ scripts/src/tasks/controlAssessment.test.ts | 790 +++++++++++++ scripts/src/tasks/controlAssessment.ts | 699 +++++++++++ scripts/src/tasks/controlAssessmentCi.test.ts | 1030 +++++++++++++++++ scripts/src/tasks/controlAssessmentCi.ts | 564 +++++++++ scripts/src/tasks/controlAssessmentCiTask.ts | 133 +++ .../src/tasks/controlAssessmentFixtures.ts | 138 +++ scripts/src/tasks/controlAssessmentGithub.ts | 272 +++++ scripts/src/tasks/controlAssessmentSpec.ts | 177 +++ .../src/tasks/controlAssessmentUnit.test.ts | 495 ++++++++ scripts/src/tasks/controlAssessmentUnit.ts | 332 ++++++ .../tasks/controlAssessmentUnitExecution.ts | 172 +++ scripts/src/tasks/jest.ts | 5 +- 25 files changed, 6064 insertions(+), 6 deletions(-) create mode 100644 .changeset/happy-lands-sink.md create mode 100644 .github/workflows/pr-control-assessment.yml create mode 100644 plan.md create mode 100644 scripts/CONTROL-ASSESSMENT.md create mode 100644 scripts/src/tasks/assessControlTask.ts create mode 100644 scripts/src/tasks/controlAssessment.test.ts create mode 100644 scripts/src/tasks/controlAssessment.ts create mode 100644 scripts/src/tasks/controlAssessmentCi.test.ts create mode 100644 scripts/src/tasks/controlAssessmentCi.ts create mode 100644 scripts/src/tasks/controlAssessmentCiTask.ts create mode 100644 scripts/src/tasks/controlAssessmentFixtures.ts create mode 100644 scripts/src/tasks/controlAssessmentGithub.ts create mode 100644 scripts/src/tasks/controlAssessmentSpec.ts create mode 100644 scripts/src/tasks/controlAssessmentUnit.test.ts create mode 100644 scripts/src/tasks/controlAssessmentUnit.ts create mode 100644 scripts/src/tasks/controlAssessmentUnitExecution.ts diff --git a/.changeset/happy-lands-sink.md b/.changeset/happy-lands-sink.md new file mode 100644 index 0000000000..853d812bb3 --- /dev/null +++ b/.changeset/happy-lands-sink.md @@ -0,0 +1,3 @@ +--- + +--- diff --git a/.github/workflows/pr-bundle-size-comment.yml b/.github/workflows/pr-bundle-size-comment.yml index 349e1559d7..3449b96caf 100644 --- a/.github/workflows/pr-bundle-size-comment.yml +++ b/.github/workflows/pr-bundle-size-comment.yml @@ -31,7 +31,22 @@ jobs: with: node-version: '22' + - name: Find bundle-size artifact + id: artifact + env: + GH_TOKEN: ${{ github.token }} + SOURCE_RUN_ID: ${{ github.event.workflow_run.id }} + shell: bash + run: | + set -euo pipefail + found=$(gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/actions/runs/$SOURCE_RUN_ID/artifacts?per_page=100" --jq '[.[].artifacts[]] | any(.name == "bundle-size-analysis" and .expired == false)') + echo "found=$found" >> "$GITHUB_OUTPUT" + if [[ "$found" != "true" ]]; then + echo "No bundle-size artifact for this attempt; report-only verification runs do not rebuild bundles." >> "$GITHUB_STEP_SUMMARY" + fi + - name: Download bundle-size report + if: steps.artifact.outputs.found == 'true' uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: bundle-size-analysis @@ -41,9 +56,11 @@ jobs: run-id: ${{ github.event.workflow_run.id }} - name: Validate downloaded bundle size report + if: steps.artifact.outputs.found == 'true' run: node apps/bundle-size/scripts/validate-report.mts bundle-size-analysis/report.md - name: Publish bundle size PR comment + if: steps.artifact.outputs.found == 'true' env: EXPECTED_HEAD_REPOSITORY: ${{ github.event.workflow_run.head_repository.full_name }} EXPECTED_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} diff --git a/.github/workflows/pr-control-assessment.yml b/.github/workflows/pr-control-assessment.yml new file mode 100644 index 0000000000..968da26dac --- /dev/null +++ b/.github/workflows/pr-control-assessment.yml @@ -0,0 +1,75 @@ +name: Control Assessment PR Summary + +on: + workflow_run: + workflows: [PR] + types: [completed] + +permissions: + actions: read + contents: read + pull-requests: read + +concurrency: + group: control-assessment-${{ github.event.workflow_run.head_repository.id }}-${{ github.event.workflow_run.head_branch }} + cancel-in-progress: false + +jobs: + publish: + name: Publish trusted Checkbox assessment + if: ${{ github.event.workflow_run.event == 'pull_request' && vars.CONTROL_ASSESSMENT_PUBLISH == 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout trusted publisher + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 0 + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + + - name: Find source assessment + id: artifact + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/src/tasks/controlAssessmentCiTask.ts find-artifact + + - name: Download source assessment data + if: steps.artifact.outputs.found == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ steps.artifact.outputs.artifact-name }} + path: ${{ runner.temp }}/checkbox-source-evidence + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ github.event.workflow_run.id }} + + - name: Validate publication without write credentials + id: prepare + if: steps.artifact.outputs.found == 'true' + env: + GITHUB_TOKEN: ${{ github.token }} + BUNDLE_PATH: ${{ runner.temp }}/checkbox-source-evidence/bundle.json + run: node scripts/src/tasks/controlAssessmentCiTask.ts prepare-publication + + - name: Create narrowly scoped publisher token + id: app-token + if: steps.prepare.outputs.eligible == 'true' + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3 + with: + client-id: ${{ vars.CONTROL_ASSESSMENT_APP_CLIENT_ID }} + private-key: ${{ secrets.CONTROL_ASSESSMENT_APP_PRIVATE_KEY }} + permission-contents: write + + - name: Commit the generated summary and SPEC assessment fields to the PR branch + if: steps.prepare.outputs.eligible == 'true' + env: + GITHUB_TOKEN: ${{ github.token }} + ASSESSMENT_WRITE_TOKEN: ${{ steps.app-token.outputs.token }} + BUNDLE_PATH: ${{ runner.temp }}/checkbox-source-evidence/bundle.json + run: node scripts/src/tasks/controlAssessmentCiTask.ts publish diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 3117b5b6eb..3d3c4d70e9 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -13,10 +13,70 @@ concurrency: cancel-in-progress: true jobs: + assessment-inputs: + name: Validate assessment inputs + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: read + contents: read + outputs: + report-only: ${{ steps.verify.outputs.report-only || steps.inspect.outputs.report-only || 'false' }} + source-result: ${{ steps.verify.outputs.source-result }} + steps: + - name: Checkout trusted input validator + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.base.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + + - name: Inspect report-only candidate + id: inspect + env: + GITHUB_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + if [[ -f scripts/src/tasks/controlAssessmentCiTask.ts ]]; then + node scripts/src/tasks/controlAssessmentCiTask.ts inspect + else + echo "candidate=false" >> "$GITHUB_OUTPUT" + echo "report-only=false" >> "$GITHUB_OUTPUT" + echo "Trusted assessment validator is not on the base branch yet. Running all normal source checks." >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Download original assessment evidence + if: steps.inspect.outputs.candidate == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ steps.inspect.outputs.artifact-name }} + path: ${{ runner.temp }}/checkbox-source-evidence + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ steps.inspect.outputs.run-id }} + + - name: Verify unchanged inputs and generated report plus SPEC assessment fields + id: verify + if: steps.inspect.outputs.candidate == 'true' + env: + CANDIDATE_PATH: ${{ steps.inspect.outputs.candidate-path }} + BUNDLE_PATH: ${{ runner.temp }}/checkbox-source-evidence/bundle.json + run: node scripts/src/tasks/controlAssessmentCiTask.ts verify + js: name: JS PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: ubuntu-latest timeout-minutes: 60 + outputs: + unit-artifact: ${{ steps.unit-evidence.outputs.artifact-id }} steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -34,11 +94,28 @@ jobs: - name: Test agent permissions run: yarn test:agent-permissions + - name: Capture existing component test inputs + run: node scripts/src/tasks/controlAssessmentCiTask.ts begin-unit + - name: Build CI + env: + FURN_ASSESSMENT_UNIT_CAPTURE: ${{ github.workspace }}/apps/storybook/artifacts/assessment-unit-ci run: yarn lage buildci + - name: Upload existing component unit evidence + id: unit-evidence + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: checkbox-unit-evidence-${{ github.run_attempt }} + path: apps/storybook/artifacts/assessment-unit-ci + if-no-files-found: warn + retention-days: 14 + bundle-size: name: Bundle Size PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: windows-latest timeout-minutes: 30 permissions: @@ -108,6 +185,8 @@ jobs: android: name: Android PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -132,6 +211,8 @@ jobs: macos: name: macOS PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: macos-26 timeout-minutes: 60 steps: @@ -180,6 +261,8 @@ jobs: macos-storybook: name: macOS Storybook PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: macos-26 timeout-minutes: 40 steps: @@ -233,6 +316,8 @@ jobs: macos-paper-storybook: name: macOS Paper Storybook PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: macos-26 timeout-minutes: 40 steps: @@ -284,6 +369,8 @@ jobs: ios: name: iOS PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: macos-26 timeout-minutes: 60 steps: @@ -321,6 +408,8 @@ jobs: windows: name: Windows PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: windows-latest timeout-minutes: 60 steps: @@ -397,6 +486,8 @@ jobs: windows-storybook: name: Windows Storybook PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: windows-latest timeout-minutes: 60 steps: @@ -463,6 +554,8 @@ jobs: win32: name: Win32 PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: windows-latest timeout-minutes: 60 steps: @@ -511,17 +604,23 @@ jobs: win32-storybook: name: Win32 Storybook PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: windows-latest timeout-minutes: 60 + outputs: + native-artifact: ${{ steps.native-evidence.outputs.artifact-id }} steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 - name: Set up toolchain uses: microsoft/react-native-test-app/.github/actions/setup-toolchain@6046bdd4914c7ceb35ec83b92ae3211a03ddccef # 5.4.14 with: platform: windows - node-version: 22 + node-version: 24 - name: Install dependencies run: yarn @@ -536,10 +635,32 @@ jobs: platform: win32 disabled-input-features: keyboard,physicalClick,wheel + - name: Capture Checkbox execution inputs + id: checkbox-inputs + run: node scripts/src/tasks/controlAssessmentCiTask.ts begin + - name: Smoke test Win32 Storybook + id: smoke + if: ${{ !cancelled() && steps.checkbox-inputs.outcome == 'success' }} run: yarn storybook smoke --win32 --mode stories-and-tests working-directory: apps/storybook + - name: Collect Checkbox native evidence + if: ${{ always() && !cancelled() }} + env: + SMOKE_OUTCOME: ${{ steps.smoke.outcome }} + run: node scripts/src/tasks/controlAssessmentCiTask.ts finish + + - name: Upload Checkbox native evidence + id: native-evidence + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: checkbox-native-evidence-${{ github.run_attempt }} + path: apps/storybook/artifacts/win32/assessment-ci + if-no-files-found: error + retention-days: 14 + - name: Upload Win32 Storybook artifacts if: ${{ always() }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -553,11 +674,59 @@ jobs: ${{ steps.desktop-driver.outputs.cache-root }}/v1/diagnostics/** if-no-files-found: warn + control-assessment: + name: Assess Checkbox evidence + needs: [assessment-inputs, js, win32-storybook] + if: ${{ always() && !cancelled() && needs.assessment-inputs.result == 'success' && needs.assessment-inputs.outputs.report-only != 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout assessed source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + + - name: Download native evidence + if: needs.win32-storybook.outputs.native-artifact != '' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: checkbox-native-evidence-${{ github.run_attempt }} + path: apps/storybook/artifacts/win32/assessment-ci + + - name: Download existing unit evidence + if: needs.js.outputs.unit-artifact != '' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: checkbox-unit-evidence-${{ github.run_attempt }} + path: apps/storybook/artifacts/assessment-unit-ci + + - name: Assess existing results without rerunning tests + env: + UNIT_OUTCOME: ${{ needs.js.result }} + NATIVE_OUTCOME: ${{ needs.win32-storybook.result }} + run: node scripts/src/tasks/controlAssessmentCiTask.ts combine + + - name: Upload Checkbox assessment + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: checkbox-win32-assessment-${{ github.run_attempt }} + path: apps/storybook/artifacts/win32/assessment-ci + if-no-files-found: error + retention-days: 14 + check-changesets: name: Check for Changesets + needs: assessment-inputs runs-on: ubuntu-latest # Skip this check for the version bump PR created by changesets - if: ${{ github.head_ref != 'changeset-release/main' }} + if: ${{ needs.assessment-inputs.outputs.report-only != 'true' && github.head_ref != 'changeset-release/main' }} steps: - name: Checkout @@ -583,6 +752,8 @@ jobs: publish-dry-run-pack: name: NPM Publish Dry Run — Pack + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: ubuntu-latest timeout-minutes: 60 outputs: @@ -624,6 +795,8 @@ jobs: test-links: name: Test repo links + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -641,11 +814,12 @@ jobs: - name: Test markdown links run: yarn lage test-links --verbose --grouped - pr: - name: PR + pr-source: + name: PR source validation if: always() runs-on: ubuntu-latest needs: + - assessment-inputs - js - android - macos @@ -656,6 +830,11 @@ jobs: - publish-dry-run-pack - test-links steps: + - name: Preserve reused source result + if: ${{ needs.assessment-inputs.outputs.report-only == 'true' && needs.assessment-inputs.outputs.source-result != 'success' }} + run: | + echo "The original source validation did not pass. A report-only commit cannot turn it green." + exit 1 - name: Check for failures or cancellations if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} run: | @@ -663,3 +842,23 @@ jobs: exit 1 - name: All required jobs passed run: echo "All required jobs completed successfully." + + pr: + name: PR + if: always() + runs-on: ubuntu-latest + needs: [assessment-inputs, pr-source] + steps: + - name: Require successful source validation + if: ${{ needs.assessment-inputs.result != 'success' || needs.pr-source.result != 'success' }} + run: | + echo "Source validation or assessment-input validation failed." + exit 1 + - name: Require same-PR assessment when publishing is enabled + if: ${{ vars.CONTROL_ASSESSMENT_PUBLISH == 'true' && github.event.pull_request.head.repo.full_name == github.repository && github.base_ref == github.event.repository.default_branch && needs.assessment-inputs.outputs.report-only != 'true' }} + run: | + echo "Source CI completed. Waiting for the trusted publisher to commit the Checkbox assessment to this PR." + echo "The report-only follow-up run must verify that report before this required check can pass." + exit 1 + - name: Required PR validation passed + run: echo "Source checks and the applicable assessment-publication requirements passed." diff --git a/.oxfmtrc.json b/.oxfmtrc.json index 4750ea7dac..831c5e11bb 100644 --- a/.oxfmtrc.json +++ b/.oxfmtrc.json @@ -10,6 +10,7 @@ "**/*.generated.*", "**/lib-commonjs/**", "**/dist/**", + "packages/agentic/components/src/components/checkbox/assessment/ASSESSMENT.md", "packages/agentic/native-core/windows/NativeCore/codegen/**", "**/CHANGELOG.*", "**/CODE_OF_CONDUCT.md", diff --git a/package.json b/package.json index 2c4c791c32..064a6dea28 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "build:clean": "yarn root-build --clean && yarn build", "bundle-size": "yarn workspace @fluentui-react-native/bundle-size measure", "bundle-size:update": "yarn workspace @fluentui-react-native/bundle-size update-baseline", + "assess-control": "node ./scripts/src/cli.ts assess-control", "docs": "yarn workspace fluent-rn-website start", "bundle:repo": "lage bundle", "clean": "tsc -b --clean && lage clean", diff --git a/packages/agentic/components/package.json b/packages/agentic/components/package.json index cf84c88241..8cfc317870 100644 --- a/packages/agentic/components/package.json +++ b/packages/agentic/components/package.json @@ -21,6 +21,7 @@ "spec-source-lock.json", "spec-source-report.json", "src/", + "!src/components/checkbox/assessment/**", "tsconfig.json", "tsconfig.stories.json" ], diff --git a/packages/agentic/components/references/contracts.md b/packages/agentic/components/references/contracts.md index 10da0530d0..822c0e7f10 100644 --- a/packages/agentic/components/references/contracts.md +++ b/packages/agentic/components/references/contracts.md @@ -28,6 +28,18 @@ discard the main benefit of YAML comments. Keep this provenance in JSON. `spec/tokens.yaml` remains YAML because it is concise, human-authored token mapping data rather than a generated identity record. +The optional Checkbox assessor snapshot is separate from contract provenance. +It uses a strictly managed `assessment-*` block in the agentic Checkbox +`SPEC.md` front matter; full evidence, digests, and workflow identity remain +in the assessment JSON/report. It never changes `status`, `spec/source.json`, +requirements, or review approval. Only a matched clean execution may populate +its tested commit; local unverified snapshots leave that field null. The +repository-only `@fluentui-react-native/scripts/CONTROL-ASSESSMENT.md` guide +defines generation, fingerprinting, and trusted two-file publication. These +assessment fields do not apply to legacy CheckboxV1. +Generated files in `src/components/checkbox/assessment/` are source-repository +snapshots, not package contents; they are excluded from the npm inventory. + ## Schema version 2 Every source document has the following document-level shape: diff --git a/plan.md b/plan.md new file mode 100644 index 0000000000..71cbcec221 --- /dev/null +++ b/plan.md @@ -0,0 +1,112 @@ +# Control readiness assessor: Checkbox pilot + +## Purpose and scope + +Make the evidence behind a control's readiness review visible and repeatable. +Start with the **agentic Checkbox on Win32/Paper**, using tests FURN already +runs. This is an advisory assessor, not a new test framework, release gate, +or automatic maturity promotion. + +The longer-term goal is an agent that evaluates changed controls against their +reviewed contracts, reports gaps, and supports human decisions about lifecycle +stages such as Backlog, Experimental, Beta, and Production. The pilot deliberately +does not assign those stages: the evidence requirements and promotion policy +still need agreement. + +## Design + +```text +Reviewed Checkbox SPEC + requirement metadata + reviewed assertion mappings + | +Existing JS PR Jest results --+-- Existing Win32 Storybook native results + | + Validate execution provenance + | + Assess each mapped requirement + | + JSON + Markdown + proposed SPEC snapshot + | + Optional trusted publisher -> same PR -> human review +``` + +1. **Use the contract as the inventory.** Read CBX-001 through CBX-006 from + `packages/agentic/components/src/components/checkbox/SPEC.md` and its + `spec/source.json`. A filename listed as evidence is not an executed test. +2. **Use reviewed mappings, not guessed coverage.** Map exact test identities to + the particular assertions they establish. Fingerprints detect changes to the + contract, test sources, and relevant harness inputs. Drift requires review; + the pilot does not automatically rewrite its mappings. +3. **Reuse existing execution.** Capture JSON and context from the component + package's existing Jest invocation in **JS PR**, without changing its test + selection or running Checkbox again. Independently collect the canonical + `desktop-test` report and Storybook companion from **Win32 Storybook PR**. + The subsequent **Assess Checkbox evidence** job combines results; it runs no + tests and does not serialize the JS and native workloads. +4. **Bind evidence to what actually ran.** Validate source/build identity, + report hashes, target, execution intervals, dirty state, and mapping + fingerprints. Preserve failures, skips, missing execution, retries, flaky + results, and stale or mismatched evidence rather than turning them into a + pass. Other controls' results are not Checkbox coverage. +5. **Publish an advisory explanation.** Emit machine-readable JSON and readable + Markdown with requirement-level observations, gaps, and provenance. Upload + attempt-specific artifacts and write a job summary. An opt-in publisher can + later commit the generated Markdown and managed SPEC assessment fields + atomically to the same PR; it does not change the human-authored contract. + +Implementation details, commands, schemas, and setup: +[scripts/CONTROL-ASSESSMENT.md](scripts/CONTROL-ASSESSMENT.md). + +## What the initial mappings establish + +| Requirement | Appropriate evidence | Initial pilot scope | +| ---------------------------- | ---------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | +| CBX-001: defaults and API | Runtime unit assertions and compiler/type assertions | Selected existing unit assertions; compiler-result ingestion is not implemented. | +| CBX-002: state and callbacks | Unit assertions for ownership/transitions; native execution for integration claims | Selected existing units plus partial binary-toggle native coverage when physical input is available. | +| CBX-003: anatomy and text | Unit assertions for structure/props; native layout for rendered geometry | Selected existing units; no blanket claim about rendered wrapping. | +| CBX-004: accessibility | Unit assertions for resolved semantics; native assertions for actual platform exposure/actions | Selected existing units plus the existing native callback's checkbox-role assertion. | +| CBX-005: styling and tokens | Unit assertions for resolved styles; visual evidence for visual claims | Selected existing units; no visual qualification. | +| CBX-006: Win32 focus | Native keyboard/focus and geometry evidence | No mapped executed evidence in the initial slice. | + +A requirement does **not** inherently need both unit and native tests. Choose +the evidence layer that proves the claim. Jest's macOS mock and simulated +`Platform.OS` values do not prove native Win32 behavior. The normal compiler +already includes Checkbox type assertions, but the pilot does not yet ingest +that build result. + +The initial mapping intentionally remains bounded to 42 previously reviewed +runtime cases and the existing default-story callback, +`toggles native checked state in both directions`. Subsequent Checkbox tests +already on `main` remain existing tests; unmapped results are not silently +credited as requirement coverage. Expanding those mappings is a follow-up. +This PR adds tooling tests, not new Checkbox behavior tests. + +## Pilot rollout and acceptance + +1. **Review this draft.** Agree on the architecture, partial mappings, report + vocabulary, and CI impact before enabling any automated branch writes. +2. **Validate collection in CI.** Confirm the ordinary Jest invocation executes + once, reports retain package-wide failures, and the assessor combines only + matching source/run/attempt evidence. Exercise missing and skipped evidence + as well as successful execution. +3. **Review the Checkbox report.** Confirm each credited assertion is supported + and missing claims remain explicit. Hosted Win32 Storybook CI disables + `keyboard`, `physicalClick`, and `wheel`; the mapped physical-toggle test + therefore remains skipped there. A green CI job is not physical-input + qualification or full Checkbox readiness. +4. **Enable optional same-PR publication separately.** After trusted tooling is + on the default branch, a maintainer may configure the narrowly scoped GitHub + App and opt-in variable described in the usage guide. The publisher executes + trusted default-branch code, treats PR artifacts as data, and permits only + the generated Checkbox summary and managed SPEC block. Verified report-only + follow-ups may reuse unchanged source validation; ordinary changes still run + normal checks. Fork PRs do not receive automated branch writes. +5. **Expand only after the pilot is useful.** Review mappings for the remaining + existing Checkbox tests, ingest existing compiler results, define readiness + policy with owners, then introduce another control/platform through explicit + adapters rather than generalizing from Checkbox automatically. + +Acceptance means honest, reproducible evidence reporting with preserved source +validation, not a Production rating. No hosted publisher activation, physical +input enablement, repository-rule changes, or additional control rollout is +part of submitting this draft. No historical local assessment is checked in as +current CI evidence. diff --git a/scripts/CONTROL-ASSESSMENT.md b/scripts/CONTROL-ASSESSMENT.md new file mode 100644 index 0000000000..2a81afb2cd --- /dev/null +++ b/scripts/CONTROL-ASSESSMENT.md @@ -0,0 +1,646 @@ +# Advisory control assessment: Checkbox pilot + +This repository-only tool reads existing native and unit test reports and the reviewed +Checkbox contract. By default it does **not** run tests; `--run-unit-tests` explicitly +runs the existing Checkbox runtime tests. It does not change controls, edit lifecycle +metadata, or approve a release. It currently supports only the agentic Checkbox +in Win32/Paper Storybook. CI collection and an opt-in same-PR publisher are +implemented alongside the local command; they require deployment and the +maintainer setup below before hosted operation can be verified. + +For the design, pilot acceptance criteria, and rollout boundaries, start with +[`plan.md`](../plan.md). This draft does not enable the publisher or check in +historical local output as current CI evidence. + +From the repository root: + +```powershell +yarn assess-control --component checkbox --evidence "apps\storybook\artifacts\desktop\\run.json" +``` + +Reports default to the ignored +`apps\storybook\artifacts\win32\assessment\assessment.json` and `assessment.md`. +Use `--output "path\to\report-directory"` to retain separate assessments. +Omit both evidence inputs to produce an inventory with explicitly missing execution evidence. +At most one native and one unit execution are accepted per assessment. Their +source/build/freshness bindings are checked independently; do not combine different +local/CI revisions into apparent current coverage. + +## Unit evidence and choosing the right test + +**A requirement does not need both unit and native tests by default.** Choose +evidence that establishes the particular claim: + +| Claim | Suitable evidence | +| ------------------------------------------------------------------------------- | --------------------------------------------------------------------------------- | +| Defaults, state transitions, callbacks, resolved tokens/styles | Complete unit assertions can suffice; no native duplicate is inherently required. | +| Public TypeScript constraints, including `children: never` | TypeScript build/type-check evidence, not Jest's stripped types. | +| Rendered layout or focus-ring geometry | Layout/visual evidence where those properties are claimed. | +| Actual native accessibility dispatch, keyboard focus/navigation, physical input | Native execution for that integration claim; mocked events do not establish it. | + +The initial reviewed unit mapping selects **42 existing runtime cases** in +`checkbox.test.tsx`. It maps applicable JS assertions to parts of CBX-001 through +CBX-005; its remaining gaps identify missing assertions rather than requiring +native duplication of all logic/style tests. It authors no new Checkbox tests. +The mappings remain deliberately partial; full per-requirement completion and +maturity gates have not yet been defined by this pilot. + +Additional existing regression cases merged after the initial mapping remain +visible as `unmappedCases`, not credited requirement coverage. The October 29 +mapping review confirmed the original 42 assertions and default native callback +are unchanged. Newly added defaults, forwarding, state, anatomy, styling, +native naming, and wrapping tests can be mapped in a follow-up; a reported +mapping gap does not necessarily mean a test is absent. + +FURN's existing **JS PR** job already runs package unit tests through +`yarn lage buildci`. The pilot adds assessment and evidence collection, not a new +unit-testing framework. It **does not rerun Checkbox in CI**. An opt-in capture +decorates the existing component package's Jest invocation, preserving its full +file selection, platform, arguments, and exit status while adding JSON output. +The runner is invoked exactly once. JS and native jobs remain parallel; a later +**Assess Checkbox evidence** job validates and combines their artifacts without +running tests or installing dependencies. + +The original full-package JSON is retained. The parser selects only +`checkbox.test.tsx` for requirement mapping; it does not count +`checkbox.types.test.ts` as runtime or compiler qualification. Package-wide +counters and failures remain visible. Missing/cached execution without an +exported report remains missing, never a pass. A failure elsewhere in the Lage +aggregate does not erase a separately captured successful component command; +original source-validation failure still stays failed. + +The component package's `furn.jestPlatform` is `macos`. CI capture records that +**React Native macOS Jest mock** and rejects conflicting inherited platform +overrides. The explicit local runner selects that same profile. +Eight cases simulate Win32 accessibility action naming/handlers via +`Platform.OS`; this is not native UIA dispatch. The 16 Windows/macOS action cases +and the macOS-only system-focus case remain recorded but are not Win32 requirement +coverage. In particular, that focus case does not establish Win32's custom dual-ring +path. An evidence label describes what ran, not a demand for a second test. + +For optional explicit local execution (not used by CI): + +```powershell +yarn assess-control --component checkbox --run-unit-tests --max-age-hours 24 +``` + +This invokes the workspace's declared `test` script with the exact runtime-file +selector, `--runInBand --json --outputFile`. Its contract/story prechecks still +run. It does not select `checkbox.types.test.ts` or turn a Jest pass into type +qualification. A fresh UUID-named directory under +`apps\storybook\artifacts\win32\assessment-units\` retains `jest.json`, +`context.json`, and `execution.json`; the assessment JSON links those inputs. +The native input remains optional and is never synthesized. + +To replay a report without running anything: + +```powershell +yarn assess-control --component checkbox --unit-evidence "path\to\jest.json" +``` + +Replay without its original context remains unverified. To evaluate a genuine +execution binding, also supply `--unit-context "path\to\context.json"` +`--unit-build-id "" --max-age-hours 24`. The sidecar binds the +exact JSON bytes, source SHA, dirty flag, contract and unit mapping fingerprints, +mock platform, start/finish times, build ID, and command exit code. Do not create +one retroactively. A dirty checkout remains unverified even after a fresh run; +recorded results remain visible. Native and unit provenance are separate. + +The reviewed unit fingerprint covers the runtime test file, package/shared Jest +configuration, Babel configuration, platform constant, and package test script/mock +profile, plus the Jest/capture harness source. Changed assertions or mock inputs invalidate the reviewed mapping even +when test names are unchanged. CI additionally binds the complete tracked tree. +The parser accepts a complete package report but requires exactly one Checkbox +runtime suite. It rejects type-only substitution, duplicate suites/inconsistent +case identities, invalid statuses, contradictory counts, and oversized JSON. +Skipped/todo/missing cases never become passes. Enclosing unit-command failure +prevents its recorded passes from qualifying; mapped failures remain visible. + +## Assessment snapshot in the agentic SPEC + +The pilot targets +`packages/agentic/components/src/components/checkbox/SPEC.md`, **not** +the legacy `packages/components/Checkbox/SPEC.md`. Its existing +`status: implemented` is contract lifecycle, not a maturity rating. + +The default local command does not modify tracked inputs. Explicitly add +`--update-spec` to save a generated front-matter snapshot and its linked local +`assessment/ASSESSMENT.md`. The block has flat, canonical `assessment-*` fields +between `# furn-assessment:begin` and `# furn-assessment:end`. It records the +source manifest version, assessed source commit, verified tested commit or +`null`, dirty flag, Win32/Paper Storybook target, outcome, assessment health, +undetermined stage, evidence binding, generation time, and report pointer. + +`assessment-source-commit` identifies the assessed checkout/merge, **not proof +that its native tests ran**. `assessment-tested-commit` is populated only for +matched, clean **native** execution evidence. Unit-only evidence does not populate +this native-tested identity or change `assessment-provenance`, which also remains +the native binding. The full report contains separate unit binding and outcomes. +The existing SPEC schema stays v1; its overall outcome/health incorporate unit +failures or stale/mismatched mappings. A dirty or unbound local run keeps the tested commit `null`; +its recorded pass is still visible in the full report. Package version is +descriptive source identity, not qualification of a published npm artifact. +No Backlog/Experimental/Beta/Production promotion is performed. + +The publisher is the separate, opt-in task after the **whole source PR +workflow completes**, not a second assessor or a new test runner. It reconstructs +the report and SPEC fields from original evidence using trusted tooling, +then commits both to that PR atomically. It preserves the PR head's contract +bytes, takes version and assessment identity from the tested merge, and never +labels the generated commit as tested. CI collection writes a proposed +`SPEC.md` only in its artifact; it does not edit the tracked SPEC during tests. +Local snapshots explicitly say they are not bound CI publications. +Generated `checkbox/assessment/` report files are excluded from the component's +npm pack inventory; the linked report is browsed in the source repository. + +After CI deployment, the **Assess Checkbox evidence** job combines the existing +**JS PR** unit report and **Win32 Storybook PR** native bundle and produces files in +`apps/storybook/artifacts/win32/assessment-ci/`. Open the PR's associated +Actions run, select **Summary**, and download +`checkbox-win32-assessment-` under **Artifacts**. It contains +`ASSESSMENT.md`, `assessment.json`, `bundle.json`, proposed `SPEC.md`, and +execution capture data and original package Jest JSON/unit context. The source +artifacts are `checkbox-unit-evidence-` and +`checkbox-native-evidence-`, downloaded to separate directories to +avoid overwrites. All three are retained for 14 days. The readable +report is also written to the job summary. These are workflow artifacts, +not PR file changes; only subsequent publisher commits appear in **Files +changed**. No hosted artifact is implied while this pilot remains undeployed. + +### Historical local validation, October 26, 2026 + +These dated results document the earlier local checkpoint, not validation of +the current draft PR. See the PR description for its current checks and limits. + +The full tooling suite passed **228 tests across 18 suites**, with no failures +or skips. Scripts build/lint, root TypeScript build/repository lint, +instruction/publishing checks, component contracts, and all three workflow +checks passed. The existing normal component command passed **561 tests across +66 suites and 8 snapshots in one Jest invocation**. Replaying its original JSON +selected **42 Checkbox runtime passes** without launching tests or treating +other-component/type suites as Checkbox evidence. Local replay has no original +execution context, so it remains unverified and assigns no maturity stage. + +The required clean-cache plain aggregate reported 95 successful and 1 failed +Lage task before stopping. The continuation completed with **82 successful, +89 skipped, and 4 failed tasks**. The aggregate is **not green**: the unchanged +Windows package-content launcher fails with `spawnSync yarn ENOENT`, two +unchanged codemod newline fixtures fail, and unchanged Desktop Driver and +Storybook Desktop tests fail under parallel load. All timeout-affected files +passed isolated serial retry with original assertions/deadlines: Storybook +Desktop **10 tests/4 suites**, Desktop Driver **7 tests/1 suite**. These retries +do not erase aggregate failures or establish native qualification. + +At that checkpoint, the base pilot was committed locally and the unit extension +was uncommitted and unpushed. Hosted collection/publication had not been activated. The following +SPEC-only checkpoint is historical, not the current unit-extension result. + +### Prior SPEC snapshot validation, October 26, 2026 + +The final tooling suite passed **189 tests across 16 suites**, with no failures +or skips. Scripts build/lint, root TypeScript build/repository lint, all 24 +component contract checks, instruction/publishing checks, and formatting passed. +All 561 component tests across 66 suites passed in the aggregate. Actual +`yarn pack --dry-run --json` confirmed that SPEC is included and generated +assessment reports are excluded. The real local SPEC snapshot preserves every +original contract byte and the reviewed mapping digest. All 20 retained native +run files and eight older evidence/report hashes remain unchanged. + +The clean-cache aggregate and continuation were run; the final continuation +reported 83 successful, 88 skipped, and 4 failed Lage tasks. It is **not a +green aggregate**. The Windows package-check launcher still fails with +`spawnSync yarn ENOENT`, and two unchanged codemod newline fixtures still fail. +The other failures were in unchanged desktop packages under aggregate load; +all their affected suites passed isolated serial retry with original assertions +and deadlines: Storybook Desktop 11 tests/5 suites, Desktop Driver 7 tests/1 +suite. These retries do not erase the original aggregate failures or establish +new native qualification. Hosted activation remains pending. + +The generated block is a snapshot, not a live claim about a subsequently +changed contract. Do not edit it manually. Only its strictly validated, +bounded schema is normalized out of fingerprints. Unknown/duplicate fields, +invalid identities, malformed markers, and content injected into the block +are errors. All surrounding front matter and the full human-authored body +remain fingerprinted. Trusted follow-up verification compares **both** files +byte-for-byte with the original evidence rendering, so plausible but altered +metadata cannot authorize reuse. + +## What the pilot evaluates + +All six `CBX-*` requirements are loaded from Checkbox's `SPEC.md` and +`spec/source.json`. Metadata evidence filenames remain **declared references**, +not passing results. + +The single existing named native case, +`components-checkbox--default` / `toggles native checked state in both directions`, +provides partial mappings for CBX-002 (binary uncontrolled transitions) and CBX-004 +(native checkbox role). Neither complete requirement is covered by that case. +A multi-assertion case failure blocks both mappings; it does not identify exactly +which individual requirement is defective. + +The remaining gaps are explicit. The unit adapter above supplements the native +case; type/build/visual/manual adapters remain unimplemented. The pilot authors +no new Checkbox tests. A passing native +case therefore **cannot establish complete readiness**. + +The current default is the schema-v1 **desktop-test canonical report**, introduced +by PR #4389 (merged October 23, 2026), plus its declared Storybook smoke companion: + +```text +apps\storybook\artifacts\desktop\\run.json +apps\storybook\artifacts\desktop\\evidence\win32\static\run.json +``` + +Pass the canonical `run.json`; keep the complete run directory together. The +assessor loads its companion automatically. It checks supervisor failures, +unfinished work, attempt history, counts, timestamps, and callback identity/status +against the smoke results. Navigation and portable/static checks are not +Checkbox requirement coverage. Retried/flaky runs and cleanup/evidence failures +cannot qualify a passing case in this conservative pilot. + +Historical direct Desktop Driver `tests` reports and schema-v1 Storybook smoke +`wdio` reports remain supported for replay. Smoke `tests` entries are portable/legacy evidence and are not +mistaken for native WDIO results. Enclosing `"status": "passed"` never converts a +named skipped case into a pass. Invalid JSON/schema, duplicate identities, +contradictory statuses, and unreadable files are errors. + +## Provenance and policy + +Existing direct reports have no source commit, build ID, or timestamp. Smoke +reports add timestamps but still lack full provenance. The assessor records their +outcomes as observations, not current qualification. File modification times, +current checkout identity, and manifest hashes alone do not establish execution +identity. + +For local runner integration, an optional **execution-generated** sidecar can +bind provenance claims to the exact UTF-8 evidence bytes. For canonical input, +these are a JSON envelope containing `schemaVersion: 1`, `format: "desktop-test"`, +and the exact original canonical and smoke JSON strings as `desktopRun` and +`storyReport`. `reportSha256` hashes that envelope as returned by +`readNativeEvidence()`, not either file alone. Historical direct/smoke input still +hashes the original report bytes. Do not construct a sidecar +retroactively from the current checkout to make old results appear current. +This is correlation metadata, not a cryptographic execution attestation. + +```json +{ + "schemaVersion": 1, + "reportSha256": "<64 lowercase hex characters: SHA-256 of the accepted evidence bytes>", + "sourceCommit": "<40 lowercase hex characters: execution source commit>", + "buildId": "", + "contractDigest": "", + "platformManifestDigest": "<64 lowercase hex characters from the report>", + "endpoint": "win32", + "renderer": "paper", + "targetId": "", + "finishedAt": "2026-10-21T20:00:00.000Z", + "physicalClick": true +} +``` + +Use `--context "path\to\context.json" --build-id "" +--max-age-hours ""` when real runner context is +available. For the current harness, also supply +`--target-id "win32-<16-hex-instance-id>"` from the runner's pre-execution identity; +the context and report must both match it. New IDs derive from the real app root, +endpoint, and renderer, so they differ between checkouts. The old fixed +`agenticstorybook-win32` identity remains accepted for historical report formats, +not as a replacement for the new instance binding. CI captures the expected +Win32/Paper Storybook ID before smoke and rejects a different instance. +The hash protocol has a regression check against the actual upstream resolver. +No default age or stage threshold is invented. Context must match the +report, current source revision, contract inputs, expected build, native target, +and manifest; its time cannot be in the future or contradict a smoke report. +A dirty target working tree remains unverified because its commit does not +identify the changes. Supplied context is still only a claim, and full readiness +remains unsupported in this pilot. + +The reviewed mapping fingerprint covers, in order, `SPEC.md`, `spec/source.json`, +`spec/interaction.md`, `spec/accessibility.md`, `spec/tokens.yaml`, `spec/usage.md`, +and `checkbox.stories.tsx`. Hash input is each relative filename, a NUL, its +UTF-8 content normalized to LF, and a NUL, after removing only the validated +generated SPEC assessment block. Any other changed input invalidates the +mapping. Review the contract and exact assertions before deliberately updating +the pinned digest in `src/tasks/controlAssessment.ts`; do not merely refresh it +to silence a stale result. + +## Reading the output + +`assessmentHealth` describes whether the assessment is complete, stale, or blocked. +`assessmentOutcome` distinguishes insufficient evidence, a qualifying native/unit +case failure, and runner failure. Native `check` and unit `unitChecks` remain +separate under each requirement. Each mapped check separates `recordedOutcome` +from `evidenceOutcome`; only matched current evidence can qualify. + +`declaredStatus`, `evidenceSupportedStatus`, and `approvedStatus` are null. +The contract's `implemented` / `reviewed` metadata is preserved separately and +is **not** converted to Backlog, Experimental, Beta, or Production. Maturity gates +and approvals require a separate agreed policy. + +Exit code **0 means the advisory reports were generated**, not that the control +is ready. With `--run-unit-tests`, a failed executed unit command also returns +nonzero after retaining available assessment evidence. Invalid inputs or failed +report writes return nonzero. This is not a +release gate. The CI integration checks report integrity and publication, not +Experimental/Beta/Production qualification. + +## Same-PR CI pilot (Option 2) + +The hand-authored integration is in `.github/workflows/pr.yml`, +`.github/workflows/pr-control-assessment.yml`, and +`scripts/src/tasks/controlAssessmentCi*.ts` plus `controlAssessmentGithub.ts`. +No additional native framework or input-capable runner is provisioned. + +1. Ordinary source PRs run the existing jobs. The JS job captures source before + its unchanged `yarn lage buildci`; the existing component Jest invocation + exports a full-package JSON report and execution context exactly once. + The independent Win32 Storybook job uses Node 24, retains hosted + `keyboard,physicalClick,wheel` disabling and captures source before native + execution. Its capture inventories existing + `artifacts/desktop` run directories; collection accepts exactly one new run + and checks its ID and execution time. It never deletes/reuses an old run or + selects an arbitrary newest report. Missing/unfinished output is explicitly + unavailable, and multiple fresh runs are an ambiguity error. Unit/native jobs + do not depend on one another. No `continue-on-error`, test rerun, or + permission/input-capability bypass is used. +2. The **Assess Checkbox evidence** job waits for JS and Win32 results and runs + even after an ordinary source-job failure. It downloads separate same-attempt + artifacts, validates matching repository/PR/head/base/tested SHA/tree/run/attempt, + report hashes and each source's own execution interval, and combines them. + Unit execution may precede native capture; cross-job evidence is not falsely + required to fall inside the native execution interval. + It never selects JSON by modification time or executes tests. Missing source + artifacts are reported as unavailable. Failed original source checks remain failed. + Collection writes `bundle.json`, `assessment.json`, `ASSESSMENT.md`, and a proposed `SPEC.md` in + ignored `apps/storybook/artifacts/win32/assessment-ci`. The readable report + also appears in the job summary. Artifact name: + `checkbox-win32-assessment-`, retained for 14 days. + New assessment JSON and CI bundles use schema v2. The trusted reader explicitly + accepts native-only v1 bundles, but rejects unit fields smuggled into v1. + Trusted reconstruction reads reviewed unit inputs from the tested Git tree, + not PR-supplied assessment output, and executes no PR unit code with write credentials. +3. An opt-in, default-branch `workflow_run` publisher validates that data and + recomputes the advisory assessment using trusted tooling. It does not check + out, install dependencies from, or execute PR code with a write credential. +4. For an eligible same-repository PR targeting the default branch, a narrowly + scoped GitHub App atomically commits **only** + `packages/agentic/components/src/components/checkbox/assessment/ASSESSMENT.md` + and the generated assessment fields in the adjacent `SPEC.md` + to the existing PR source branch. It preserves every other SPEC byte and + never writes to main or merges a PR. + The API's `expectedHeadOid` rejects a branch race; newer source results must + be used instead of force-pushing or appending stale evidence. +5. The report commit triggers a follow-up PR run. Trusted input validation + downloads the original attempt's bundle, verifies the exact rendered report and SPEC, + checks all tracked inputs, and retrieves the original source-validation + result from GitHub. Only then may expensive jobs be skipped. A failed or + unrun source-validation job cannot become successful through a report commit. +6. Report-only verification runs do not create native assessment artifacts, so + the publisher makes no further report commit. The bundle-size commenter also + handles the absence of its artifact on those runs. + +The existing required check is still named **PR**. Its source-check aggregation +is now separately visible as **PR source validation**. With publishing enabled +for an eligible PR, the initial **PR** check deliberately fails with a +"waiting for the trusted publisher" message after source validation. This +prevents merging before publication. The report-only follow-up replaces it with +a successful **PR** check only if both the source result and report verification +pass. This temporary publication failure is not a control-readiness rating. + +Without opt-in, CI still collects reports and the required check preserves normal +source-validation behavior. Fork PRs and non-default-target PRs retain CI +reports, but the pilot does not write their branches or require automatic +publication. No fork token, maintainer-edit assumption, approval bypass, or +branch-protection exception is used. + +### Tested source, report commits, and freshness + +`sourceCommit` remains the actual tested PR **merge revision**, not the later +report commit. The summary separately records PR head, tested base, source +workflow run and attempt, generation time, and the input digest. + +The CI digest hashes Git tree entries, including blob IDs and file modes, for +**every tracked file except the exact generated summary path**, with only the +validated SPEC assessment block removed before recomputing that SPEC's blob ID. +The original mode and path remain inputs. A SPEC without the generated block +retains its previous digest unchanged. Tests, +specifications, code, all dependency files, workflows, formatting policy, and +other configuration are inputs. This deliberately conservative pilot does not +attempt dependency-graph impact analysis. Another control change can therefore +invalidate reuse. + +Reuse requires a single two-file report/SPEC commit directly after the tested PR head, +the same complete input digest in the new merge tree, the matching original +workflow attempt and PR, and byte-for-byte equality of both files with trusted renderings. +It does not use the cumulative PR diff or merely inspect the latest commit. +The explicit CI reuse limit is **168 hours**; this is an operational evidence-age +limit, not a maturity-stage policy. Older references cause a full source run. +Deleted or expired original artifacts cannot authorize a report-only fast path. +The original local CLI still requires an explicit `--max-age-hours` value. + +Legacy summary-only publications fall back to full source CI rather than +receiving an unchecked exemption. Malformed SPEC assessment metadata fails +explicitly; valid but altered generated fields fail exact reconstruction. + +The summary says **AUTO-GENERATED. DO NOT EDIT**. Its sole formatter exclusion +prevents a source formatter from changing evidence-bound output. Do not exclude +the entire control, its specifications, or its tests. The SPEC renderer uses +the repository's single-quoted YAML style and does not require a SPEC-wide +formatting exemption. + +### Maintainer activation and first hosted trial + +Deployment is not implied by local validation: + +1. Review and merge the collector, trusted validator, publisher workflow, and + tooling to main **with publication disabled**. `workflow_run` needs its + workflow on the default branch. The bootstrap PR runs full CI if the trusted + validator is not yet on its base branch. +2. Install a dedicated GitHub App on **only this repository**, granting + `Contents: write`. It needs no Actions-write, Workflows-write, + Administration, review-approval, or protection-bypass permission. +3. Configure repository variable `CONTROL_ASSESSMENT_APP_CLIENT_ID` and secret + `CONTROL_ASSESSMENT_APP_PRIVATE_KEY`. Keep the private key out of source and + logs. The action requests only `contents:write`, scopes the token to the + current repository, and revokes it when its job finishes. +4. Have a maintainer verify organizational Actions execution policies, the App's + permitted branch writes, and fork-run approval behavior. No policy is + weakened by this pilot. +5. Set repository variable `CONTROL_ASSESSMENT_PUBLISH=true` only after setup. + Open a same-repository pilot PR, then observe source CI, the single report + commit, trusted report-only validation, and the final required **PR** check. + Verify one failed source check stays failed, a newer source push rejects stale + publication, and concurrent Checkbox PRs regenerate reports after conflicts. + +If the publisher is misconfigured, the eligible PR remains blocked rather than +merging without its report. Disabling the opt-in restores the ordinary artifact +flow; changing a variable may require rerunning the PR workflow. This does not +change the repository's branch rules. + +### Merge conflicts and concurrent changes + +When two Checkbox PRs conflict, resolve actual source conflicts first. Keep +main's generated report and assessment block temporarily rather than merging assessment prose, +update the PR branch, and rerun source CI. The publisher regenerates the summary and SPEC snapshot +for the updated inputs. GitHub's ordinary `pull_request` workflow will not start +while conflicts remain. + +After a newer source commit, closed PR, unavailable merge revision, different +merged inputs, or unsupported fork, publication explicitly reports why no commit +was made. Permission and API failures are errors, not successful empty reports. +There is no force push, automatic source-conflict resolver, auto-approval, or +automatic merge. + +### FURN rules verified October 21, 2026 (Pacific) + +Read-only API inspection found active main ruleset **12778298**: + +- One approving review; stale-review dismissal **off**, latest-push approval + requirement **off**, code-owner review requirement **off**. +- Review-thread resolution required; only squash merges allowed. +- Required status context **PR**, GitHub Actions integration **15368**. +- Strict up-to-date checking **off**. The report identifies the tested + integration tree; it does not promise compatibility with every later main + revision. +- Extra approval for unattributed Copilot PRs is enabled. This is not a blanket + exemption for generated reports; applicable repository rules still govern. + +The inspecting account had read-only repository access. Actions settings +endpoints returned **403**, so organization execution policy, default workflow +permissions, and fork-contributor approval configuration were **not verified**. +The legacy branch-protection endpoint returned **404**; that does not negate +the active ruleset returned by the effective branch-rules API. + +The published rules permit the Option 2 design; they do not establish that an +App is installed, that secrets exist, or that hosted publication has run. + +## Historical October 26, 2026 migration checkpoint + +Local main was fast-forwarded to +`01ab9fee6dbf40de9cec40926f8aa8d68193d446`, including Jason's PR #4389. +The reviewed Checkbox mapping digest changed only after checking the upstream +focus-document link relocations and unchanged mapped assertions. No new control +test, maturity policy, publisher permission, or native input capability was added. +The legacy desktop Appium and mobile lanes were not removed by #4389. + +A replay of PR #4389's actual Win32 artifact (Actions run `39868561037`, attempt +1, desktop run `bfcbec99-2712-4449-9722-81dac06a02cd`) successfully parsed the +canonical result and companion: supervisor passed; Checkbox interaction skipped +because physical pointer input was disabled. This is historical artifact replay, +not a new native test or current qualification. + +Scripts build/lint and all **174 tests passed** (15 suites, no skips). Tests +include canonical reporting, incomplete/ambiguous/missing evidence, retries, +target-ID conformance against the real upstream resolver, target mismatch, +actual CLI entrypoints, publication, and report-only verification. +Actionlint 1.7.12 passed for all three pilot workflows. Formatting checks and +`git diff --check` passed. These are local tooling checks, not native qualification. + +**Immutable dependency restoration now passed.** The corporate registry supplied +`ip-address@10.7.4` and `zod@4.6.7`. The final missing package, +`storybook@10.6.3`, was recovered from public CDN files over verified HTTPS. +Its reconstructed Yarn cache archive matched the **entire original lockfile +SHA-512 checksum**, before any recovered package code was executed. Offline +`yarn install --immutable`, with checksum behavior `throw`, then succeeded with +peer-dependency/linker warnings. The lockfile and package versions are unchanged; +no certificate validation, permission, or integrity check was bypassed. +Registry/auth overrides were process-local. This repairs the local cache, not +the normal registry TLS route; a future cache miss may still need that route fixed. + +The Storybook update from 10.6.0 to 10.6.3 came from Dependabot PR #4387 on +October 23, not Jason's #4389. The app already declared `storybook: ^10.4.0`; +the lockfile selects the exact version. This is an existing npm development +dependency, not another desktop application or a binary to commit into Git. + +The required clean-cache aggregate now reaches actual validation. Root +TypeScript build, repository lint, agent-instruction checks, and publishing +checks passed. It is **not entirely green**: the unchanged upstream +`checkPackageContentsTask.ts` invokes `execFileSync('yarn', ...)`, which fails +with `ENOENT` on this Windows setup for all eight guided packages. Directly +spawning the available `.cmd` shim also reproduces `EINVAL`. That task came +from upstream PR #4393 and was not changed by this pilot. This is not the old +stale-pnpm-map failure or proof that the entire pre-sync baseline persists. + +The continuation completed with **82 successful, 88 skipped, and 5 failed Lage +tasks**; those are task statuses, not native test-case counts. The scripts +regression suite passed again after restoration, as did the neutral +`desktop-test` suite. Besides the package-content check, both unchanged codemod +fixture tests failed with newline-only output differences. Three other +workspace tasks failed on timeouts during the parallel aggregate. All their +affected suites passed on isolated serial retry with unchanged assertions and +time limits: Storybook Desktop 11 tests, Desktop Driver 7, and agentic components +73 (**91 tests across 11 suites**). This is evidence of load-sensitive failures, +not a clean aggregate pass or native qualification. The upstream packages and +their test sources are unchanged by this pilot. + +Post-restore CLI replay of the real canonical artifact also passed, preserving +insufficient evidence and null maturity statuses. +**Fresh local native verification now passed.** After generating the current +manifest, bundling Win32, explicitly building the helper, and passing its +prebuilt-only `doctor`, the current harness completed desktop run +`740c6879-528f-46f2-818f-3eb36b5fbd34` on October 26, 20:19:33-20:21:33 UTC. +It recorded 166 story-navigation passes and the one Checkbox callback pass: +unchecked -> checked -> unchecked through physical clicks. All 167 selected +checks passed, with no skips, failures, retries, flaky results, incomplete work, +or harness failures. The story filter selected the Checkbox callback; it did +not reduce the harness's full story traversal. + +Run app-owned commands from `apps/storybook`, or select its workspace explicitly +from the repository root, for example: + +```powershell +yarn workspace @fluentui-react-native/agentic-components-storybook storybook bundle --win32 +yarn workspace @fluentui-react-native/agentic-components-storybook storybook build-driver --win32 +yarn workspace @fluentui-react-native/agentic-components-storybook desktop-driver doctor --platform win32 +$env:STORYBOOK_SMOKE_STORY = 'components-checkbox--default' +$env:FURN_DESKTOP_DRIVER_BUILD_POLICY = 'never' +yarn workspace @fluentui-react-native/agentic-components-storybook storybook smoke --win32 --mode stories-and-tests +``` + +The environment overrides above are process-local; clear the story filter +before selecting other controls. Win32 uses the prebuilt REX Paper host: the +bundle and native helper are built, not a new native Win32 app host. + +The assessor consumed that actual canonical run and its companion. It preserved +one passed case, zero skipped cases, and the current reviewed mapping, while +correctly reporting `insufficient-evidence` and null maturity statuses. Local +execution has no bound CI provenance and the checkout is dirty; no context or +attestation was fabricated. The native case still covers only parts of CBX-002 +and CBX-004. Navigation passes do not become requirement coverage. +The full run's 20 files were retained byte-for-byte with a SHA-256 inventory in +the user's FURN evidence folder, with a separate assessment and local provenance +note; historical evidence and CI skips were preserved. + +No hosted collector, GitHub App publication, or required-check transition was +run. A fully green repository aggregate is not claimed. + +At this checkpoint, the original pre-sync pilot was retained in a named Git stash +and migration changes remained local, uncommitted, and unpushed. + +## Historical pre-sync validation (October 21-22, 2026) + +The owning scripts workspace build, lint, and all **112 tests passed**. +Tests cover actual CLI entrypoints and disposable Git histories, including +report-only reuse, changed inputs, failed source checks, stale publication, +branch races, fork rejection, and concurrent generated-report conflicts followed +by regeneration. Write API calls use test doubles; these are not hosted trials. +Actionlint 1.7.12 passed for all three changed workflows. + +The required clean-cache `yarn lage buildci` run failed on the same 11 Knip +unused dependencies previously reproduced on the unchanged baseline. A +continuation run completed with 84 successful tasks, 85 cached/skipped tasks, +no pending or aborted tasks, and two failed tasks: repository lint and the +legacy codemod test task (two failing tests). The root TypeScript build, +agent-instruction checks, repository formatting, and scripts tests passed. +These baseline failures have not been hidden or changed. + +No new native execution was performed for this CI implementation. Hosted +collection, GitHub App publication, and required-check transitions were +**not run** at that checkpoint, pending maintainer deployment and configuration. +None of these historical results constitute hosted activation. + +Run the owning workspace tests with: + +```powershell +yarn workspace @fluentui-react-native/scripts test +``` diff --git a/scripts/package.json b/scripts/package.json index 2ed46c544d..4f5bc2b86d 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -30,7 +30,7 @@ "bundlesize": "bundlesize --debug", "format": "node ./src/cli.ts format", "lint": "node ./src/cli.ts lint", - "test": "node --test src/tasks/agentInstructions.test.ts src/tasks/clean.test.mjs src/utils/jestSelection.test.ts src/utils/runScript.test.ts" + "test": "node --test \"$@\" src/tasks/agentInstructions.test.ts src/tasks/clean.test.mjs src/utils/jestSelection.test.ts src/utils/runScript.test.ts src/tasks/controlAssessment.test.ts src/tasks/controlAssessmentCi.test.ts src/tasks/controlAssessmentUnit.test.ts" }, "dependencies": { "@babel/core": "catalog:", diff --git a/scripts/src/cli.ts b/scripts/src/cli.ts index 2d0b17c244..e78fcf5f38 100644 --- a/scripts/src/cli.ts +++ b/scripts/src/cli.ts @@ -8,6 +8,7 @@ import { JestCommand } from './tasks/jest.ts'; import { CheckPublishingCommand } from './tasks/checkPublishingTask.ts'; import { CheckAgentInstructionsCommand } from './tasks/checkAgentInstructionsTask.ts'; import { CheckPackageContentsCommand } from './tasks/checkPackageContentsTask.ts'; +import { AssessControlCommand } from './tasks/assessControlTask.ts'; const cli = new Cli({ binaryLabel: 'fluentui-scripts CLI', @@ -22,6 +23,7 @@ cli.register(JestCommand); cli.register(CheckPublishingCommand); cli.register(CheckAgentInstructionsCommand); cli.register(CheckPackageContentsCommand); +cli.register(AssessControlCommand); cli.register(Builtins.DefinitionsCommand); cli.register(Builtins.HelpCommand); diff --git a/scripts/src/tasks/assessControlTask.ts b/scripts/src/tasks/assessControlTask.ts new file mode 100644 index 0000000000..78088ed263 --- /dev/null +++ b/scripts/src/tasks/assessControlTask.ts @@ -0,0 +1,184 @@ +import { execFileSync } from 'node:child_process'; +import { existsSync, lstatSync, mkdirSync, readFileSync, realpathSync, statSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { Command, Option } from 'clipanion'; +import { findGitRoot } from 'workspace-tools'; + +import { + assessCheckbox, + formatAssessmentMarkdown, + object, + parseEvidenceContext, + parseNativeReport, + readCheckboxContract, + readNativeEvidence, + reportDigest, + text, +} from './controlAssessment.ts'; +import { assessmentPackagePath, assessmentSpecPath, specAssessmentFor, updateSpecAssessment } from './controlAssessmentSpec.ts'; +import { parseUnitContext, readCheckboxUnitMappingFrom } from './controlAssessmentUnit.ts'; +import { runCheckboxUnitTests } from './controlAssessmentUnitExecution.ts'; + +function sameFile(first: string, second: string): boolean { + if (path.relative(first, second) === '') return true; + if (!existsSync(first) || !existsSync(second)) return false; + const firstInfo = statSync(first, { bigint: true }); + const secondInfo = statSync(second, { bigint: true }); + return firstInfo.dev === secondInfo.dev && firstInfo.ino === secondInfo.ino; +} + +export class AssessControlCommand extends Command { + static override paths = [['assess-control']]; + + static override usage = Command.Usage({ + description: 'Produce an advisory Checkbox/Win32 assessment from native and JavaScript unit evidence', + details: + 'Reads existing native/unit reports, or explicitly runs Checkbox runtime tests with --run-unit-tests. Writes assessment.json and assessment.md, preserving skips, provenance, and evidence gaps. --update-spec records a generated SPEC snapshot and linked report. No maturity stage is assigned. See scripts/CONTROL-ASSESSMENT.md.', + examples: [ + [ + 'Assess an existing native report', + '$0 assess-control --component checkbox --evidence apps/storybook/artifacts/desktop//run.json', + ], + ], + }); + + component = Option.String('--component', { required: true, description: 'Pilot component: checkbox' }); + evidence = Option.String('--evidence', { + description: 'Canonical desktop-test run.json with its Storybook companion, or a historical WDIO/smoke report', + }); + contextFile = Option.String('--context', { + description: 'Optional execution-generated provenance sidecar, never synthesized from the current checkout', + }); + unitEvidence = Option.String('--unit-evidence', { description: 'Jest JSON selecting only checkbox.test.tsx, not type tests' }); + unitContextFile = Option.String('--unit-context', { description: 'Execution-generated unit provenance sidecar' }); + unitBuildId = Option.String('--unit-build-id', { description: 'Expected unit build ID for replayed evidence' }); + runUnitTests = Option.Boolean('--run-unit-tests', false, { + description: 'Run the declared Checkbox runtime test script and capture fresh unit evidence', + }); + buildId = Option.String('--build-id', { description: 'Expected test build ID; required to accept bound evidence' }); + targetId = Option.String('--target-id', { description: 'Expected native target instance; required for bound desktop-test evidence' }); + maxAgeHours = Option.String('--max-age-hours', { description: 'Explicit positive evidence-age limit; no default readiness policy' }); + output = Option.String('--output', { description: 'Report directory; defaults to apps/storybook/artifacts/win32/assessment' }); + updateSpec = Option.Boolean('--update-spec', false, { + description: 'Also write the generated SPEC assessment snapshot and local report', + }); + + async execute() { + if (this.component !== 'checkbox') throw new Error('This pilot supports only --component checkbox on Win32/Paper.'); + if (this.runUnitTests && (this.unitEvidence !== undefined || this.unitContextFile !== undefined || this.unitBuildId !== undefined)) { + throw new Error('--run-unit-tests cannot be combined with replayed unit evidence, context, or build ID.'); + } + if (this.unitContextFile !== undefined && this.unitEvidence === undefined) throw new Error('--unit-context requires --unit-evidence.'); + if (this.unitBuildId !== undefined && this.unitEvidence === undefined) throw new Error('--unit-build-id requires --unit-evidence.'); + const root = findGitRoot(process.cwd()); + const unitRun = this.runUnitTests + ? runCheckboxUnitTests(root, path.join(root, 'apps', 'storybook', 'artifacts', 'win32', 'assessment-units')) + : undefined; + const sourceCommit = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).trim(); + const workingTreeDirty = + execFileSync('git', ['status', '--porcelain', '--untracked-files=all'], { cwd: root, encoding: 'utf8' }).length > 0; + const input = this.evidence === undefined ? undefined : path.resolve(this.evidence); + const contextFile = this.contextFile === undefined ? undefined : path.resolve(this.contextFile); + const unitInput = unitRun?.report + ? path.join(unitRun.directory, 'jest.json') + : this.unitEvidence === undefined + ? undefined + : path.resolve(this.unitEvidence); + const unitContextFile = unitRun?.context + ? path.join(unitRun.directory, 'context.json') + : this.unitContextFile === undefined + ? undefined + : path.resolve(this.unitContextFile); + const content = input === undefined ? undefined : readNativeEvidence(input); + const specPath = path.join(root, assessmentSpecPath); + const originalSpec = readFileSync(specPath, 'utf8'); + const report = assessCheckbox({ + contract: readCheckboxContract(root), + sourceCommit, + workingTreeDirty, + now: new Date(), + report: content === undefined ? undefined : parseNativeReport(JSON.parse(content)), + reportSha256: content === undefined ? undefined : reportDigest(content), + context: contextFile === undefined ? undefined : parseEvidenceContext(JSON.parse(readFileSync(contextFile, 'utf8'))), + expectedBuildId: this.buildId, + expectedTargetId: this.targetId, + maxAgeHours: this.maxAgeHours === undefined ? undefined : Number(this.maxAgeHours), + unit: + unitRun || unitInput !== undefined + ? { + mapping: readCheckboxUnitMappingFrom((file) => readFileSync(path.join(root, file), 'utf8')), + report: unitRun?.report ?? (unitInput === undefined ? undefined : readFileSync(unitInput, 'utf8')), + context: + unitRun?.context ?? + (unitContextFile === undefined ? undefined : parseUnitContext(JSON.parse(readFileSync(unitContextFile, 'utf8')))), + executionFailure: unitRun?.executionFailure ?? undefined, + expectedBuildId: unitRun?.context?.buildId ?? this.unitBuildId, + } + : undefined, + }); + const directory = + this.output === undefined ? path.join(root, 'apps', 'storybook', 'artifacts', 'win32', 'assessment') : path.resolve(this.output); + const jsonPath = path.join(directory, 'assessment.json'); + const markdownPath = path.join(directory, 'assessment.md'); + const localSummary = path.join(path.dirname(specPath), 'assessment', 'ASSESSMENT.md'); + const outputs = this.updateSpec ? [jsonPath, markdownPath, specPath, localSummary] : [jsonPath, markdownPath]; + for (const output of outputs) { + if ([input, contextFile, unitInput, unitContextFile].some((file) => file !== undefined && sameFile(file, output))) { + throw new Error('Assessment output must not overwrite an input report or context.'); + } + } + if (sameFile(jsonPath, markdownPath)) throw new Error('Assessment JSON and Markdown must be separate files.'); + for (const output of [jsonPath, markdownPath, ...(this.updateSpec ? [localSummary] : [])]) { + if (sameFile(output, specPath)) throw new Error('Assessment output must not overwrite the specification.'); + } + if (this.updateSpec && [jsonPath, markdownPath].some((output) => sameFile(output, localSummary))) { + throw new Error('Local summary must be separate from the artifact reports.'); + } + let updatedSpec: string | undefined; + if (this.updateSpec) { + for (const file of [specPath, localSummary]) { + if (existsSync(file) && (!lstatSync(file).isFile() || lstatSync(file).nlink !== 1)) { + throw new Error('SPEC snapshot destinations must be regular, unlinked files.'); + } + } + const manifest = object(JSON.parse(readFileSync(path.join(root, assessmentPackagePath), 'utf8')), 'Component package manifest'); + updatedSpec = updateSpecAssessment(originalSpec, specAssessmentFor(report, text(manifest.version, 'Component package version'))); + mkdirSync(path.dirname(localSummary), { recursive: true }); + if (path.relative(realpathSync(path.dirname(specPath)), realpathSync(path.dirname(localSummary))) !== 'assessment') { + throw new Error('Local assessment directory must not redirect outside its component.'); + } + } + mkdirSync(directory, { recursive: true }); + writeFileSync( + jsonPath, + `${JSON.stringify( + { + ...report, + inputs: { + report: input ?? null, + context: contextFile ?? null, + unitReport: unitInput ?? null, + unitContext: unitContextFile ?? null, + unitExecuted: this.runUnitTests, + }, + }, + null, + 2, + )}\n`, + ); + writeFileSync(markdownPath, formatAssessmentMarkdown(report)); + if (updatedSpec !== undefined) { + if (readFileSync(specPath, 'utf8') !== originalSpec) + throw new Error('SPEC changed during assessment; refusing concurrent overwrite.'); + writeFileSync( + localSummary, + `\n\n${formatAssessmentMarkdown(report)}`, + ); + writeFileSync(specPath, updatedSpec); + } + this.context.stdout.write( + `Assessment report generated: ${report.assessmentOutcome}; readiness not established (advisory).\n${jsonPath}\n${markdownPath}\n`, + ); + return unitRun?.exitCode ?? 0; + } +} diff --git a/scripts/src/tasks/controlAssessment.test.ts b/scripts/src/tasks/controlAssessment.test.ts new file mode 100644 index 0000000000..d38228073a --- /dev/null +++ b/scripts/src/tasks/controlAssessment.test.ts @@ -0,0 +1,790 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { copyFileSync, linkSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, it } from 'node:test'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +import { + assessCheckbox, + formatAssessmentMarkdown, + parseEvidenceContext, + parseNativeReport, + readCheckboxContract, + readCheckboxContractFrom, + readNativeEvidence, + reportDigest, + win32StorybookTargetId, +} from './controlAssessment.ts'; +import type { AssessmentOptions, EvidenceContext } from './controlAssessment.ts'; +import { checkboxJestFixture, desktopEvidenceFixture } from './controlAssessmentFixtures.ts'; +import { + assessmentPackagePath, + assessmentSpecPath, + specAssessmentFor, + splitSpecAssessment, + updateSpecAssessment, +} from './controlAssessmentSpec.ts'; + +const root = fileURLToPath(new URL('../../../', import.meta.url)); +const contract = readCheckboxContract(root); +const sourceCommit = 'a'.repeat(40); +const platformManifestDigest = 'b'.repeat(64); +const now = new Date('2026-10-21T21:00:00.000Z'); +const nativeCase = { + storyId: 'components-checkbox--default', + testName: 'toggles native checked state in both directions', + status: 'passed', + durationMs: 3687, +}; +const direct = { endpoint: 'win32', platformManifestDigest, status: 'passed', tests: [nativeCase] }; +const aggregate = { + schemaVersion: 1, + endpoint: 'win32', + platformName: 'windows', + targetId: 'agenticstorybook-win32', + runId: 'synthetic-run', + startedAt: '2026-10-21T19:59:00.000Z', + finishedAt: '2026-10-21T20:00:00.000Z', + manifest: { platform: platformManifestDigest, portable: 'c'.repeat(64) }, + status: 'passed', + tests: [{ storyId: 'ignored-portable-test', status: 'passed' }], + wdio: [{ ...nativeCase, status: 'skipped', skipReason: 'This test requires physical pointer input.' }], +}; +const raw = JSON.stringify(direct); +const context: EvidenceContext = { + schemaVersion: 1, + reportSha256: reportDigest(raw), + sourceCommit, + buildId: 'synthetic-build', + contractDigest: contract.digest, + platformManifestDigest, + endpoint: 'win32', + renderer: 'paper', + targetId: 'agenticstorybook-win32', + finishedAt: '2026-10-21T20:00:00.000Z', + physicalClick: true, +}; + +function options(overrides: Partial = {}): AssessmentOptions { + return { + contract, + sourceCommit, + workingTreeDirty: false, + now, + report: parseNativeReport(direct), + reportSha256: reportDigest(raw), + context, + expectedBuildId: context.buildId, + maxAgeHours: 24, + ...overrides, + }; +} + +const fixtures: string[] = []; +function fixture(): string { + const directory = mkdtempSync(path.join(tmpdir(), 'furn-assessment-')); + fixtures.push(directory); + return directory; +} +afterEach(() => { + for (const directory of fixtures.splice(0)) rmSync(directory, { recursive: true, force: true }); +}); + +describe('Checkbox advisory assessment', () => { + it('loads six reviewed requirements without converting implemented/reviewed into a maturity stage', () => { + assert.equal(contract.mappingCurrent, true); + assert.deepEqual( + contract.requirements.map(({ id }) => id), + ['CBX-001', 'CBX-002', 'CBX-003', 'CBX-004', 'CBX-005', 'CBX-006'], + ); + const report = assessCheckbox(options()); + assert.equal(report.contract.lifecycle, 'implemented'); + assert.equal(report.contract.conformance, 'reviewed'); + assert.equal(report.declaredStatus, null); + assert.equal(report.approvedStatus, null); + assert.equal(report.evidenceSupportedStatus, null); + assert.equal(report.policyStatus, 'pending'); + assert.equal(report.assessmentOutcome, 'insufficient-evidence'); + assert.deepEqual( + report.requirements.map(({ outcome }) => outcome), + ['insufficient-evidence', 'partial', 'insufficient-evidence', 'partial', 'insufficient-evidence', 'insufficient-evidence'], + ); + assert.equal(report.evidence.caseCounts.passed, 1); + assert.equal(report.requirements.filter(({ check }) => check).length, 2); + assert.ok(report.requirements.every(({ remainingCoverage }) => remainingCoverage.length > 0)); + }); + + describe('desktop-test canonical evidence', () => { + const smoke = JSON.stringify(aggregate); + it('computes the same Win32/Paper target identity as the actual upstream configuration resolver', () => { + const directory = fixture(); + const appRoot = path.join(directory, 'apps', 'storybook'); + mkdirSync(appRoot, { recursive: true }); + const configPath = path.join(appRoot, 'desktop.config.mts'); + writeFileSync(configPath, 'export default { schemaVersion: 1, targets: { win32: { host: { kind: "rex", renderer: "paper" } } } };'); + const resolver = pathToFileURL(path.join(root, 'packages', 'agentic', 'desktop-test', 'src', 'config.ts')).href; + const result = spawnSync( + process.execPath, + [ + '--input-type=module', + '-e', + 'const { loadDesktopConfig } = await import(process.argv[1]); const { targets } = await loadDesktopConfig(process.argv[2]); console.log(targets.win32.id);', + resolver, + configPath, + ], + { encoding: 'utf8' }, + ); + assert.equal(result.status, 0, result.stderr); + assert.equal(win32StorybookTargetId(directory), result.stdout.trim()); + }); + + it('binds generated target IDs without treating an arbitrary Win32 instance as the expected host', () => { + const targetId = `win32-${'1'.repeat(16)}`; + const { envelope } = desktopEvidenceFixture(JSON.stringify({ ...aggregate, targetId })); + const report = parseNativeReport(JSON.parse(envelope)); + const bound = parseEvidenceContext({ ...context, reportSha256: reportDigest(envelope), targetId }); + const input = options({ report, reportSha256: reportDigest(envelope), context: bound, expectedTargetId: targetId }); + assert.equal(assessCheckbox(input).evidence.status, 'matched'); + assert.equal(assessCheckbox({ ...input, expectedTargetId: undefined }).evidence.status, 'unverified'); + assert.equal(assessCheckbox({ ...input, expectedTargetId: `win32-${'2'.repeat(16)}` }).evidence.status, 'mismatched'); + assert.equal(assessCheckbox({ ...input, context: { ...bound, targetId: 'agenticstorybook-win32' } }).evidence.status, 'mismatched'); + assert.equal(assessCheckbox({ ...input, context: undefined, expectedTargetId: undefined }).evidence.status, 'unverified'); + }); + + it('combines canonical supervisor and smoke evidence without counting navigation as requirement coverage', () => { + const result = parseNativeReport(JSON.parse(desktopEvidenceFixture(smoke).envelope)); + assert.equal(result.format, 'desktop-test'); + assert.equal(result.harness?.status, 'passed'); + assert.equal(result.cases.length, 1); + assert.equal(result.cases[0].status, 'skipped'); + assert.equal(result.harness?.retries, 0); + }); + + it('loads a canonical run with its declared companion, preserving both sets of bytes', () => { + const directory = fixture(); + const { canonical } = desktopEvidenceFixture(smoke); + const evidence = path.join(directory, 'evidence', 'win32', 'static'); + mkdirSync(evidence, { recursive: true }); + writeFileSync(path.join(directory, 'run.json'), canonical); + writeFileSync(path.join(evidence, 'run.json'), smoke); + const content = readNativeEvidence(path.join(directory, 'run.json')); + assert.equal(JSON.parse(content).desktopRun, canonical); + assert.equal(JSON.parse(content).storyReport, smoke); + assert.equal(parseNativeReport(JSON.parse(content)).format, 'desktop-test'); + }); + + it('rejects a missing companion instead of silently accepting supervisor success', () => { + const directory = fixture(); + writeFileSync(path.join(directory, 'run.json'), desktopEvidenceFixture(smoke).canonical); + assert.throws(() => readNativeEvidence(path.join(directory, 'run.json')), /ENOENT/); + }); + + it('retains cleanup failure even when the Storybook evidence passed', () => { + const { envelope } = desktopEvidenceFixture(smoke, { + status: 'failed', + failures: [{ failureId: 'cleanup', kind: 'cleanup', error: { name: 'Error', message: 'session cleanup failed' } }], + }); + const result = parseNativeReport(JSON.parse(envelope)); + assert.equal(result.status, 'failed'); + assert.equal(result.harness?.failureCount, 1); + assert.match(result.error ?? '', /supervisor status: failed/); + }); + + it('preserves unfinished supervisor execution rather than qualifying its recorded cases', () => { + const result = parseNativeReport(JSON.parse(desktopEvidenceFixture(smoke, { status: 'incomplete' }).envelope)); + assert.equal(result.status, 'failed'); + assert.equal(result.harness?.status, 'incomplete'); + }); + + it('does not accept retry/flaky passes as a clean qualifying execution', () => { + const passedSmoke = JSON.stringify({ ...aggregate, wdio: [nativeCase] }); + const { run } = desktopEvidenceFixture(passedSmoke); + const retried = run.tests[1]; + retried.attempts.unshift({ ...retried.attempts[0] }); + retried.attempts[1].attempt = 1; + run.counts.retries = 1; + const result = parseNativeReport({ + schemaVersion: 1, + format: 'desktop-test', + desktopRun: JSON.stringify(run), + storyReport: passedSmoke, + }); + assert.equal(result.harness?.retries, 1); + assert.equal(result.status, 'failed'); + assert.match(result.error ?? '', /Retried or flaky/); + }); + + for (const [name, mutate] of [ + [ + 'counts', + (run: Record) => { + run.counts = {}; + }, + ], + [ + 'duplicated artifact', + (run: Record) => { + run.artifacts = [{ path: 'evidence/win32/static/run.json' }, { path: 'evidence\\win32\\static\\run.json' }]; + }, + ], + [ + 'wrong run kind', + (run: Record) => { + run.kind = 'lifecycle'; + }, + ], + [ + 'outside timestamp', + (run: Record) => { + run.startedAt = '2026-10-22T00:00:00.000Z'; + }, + ], + [ + 'invalid schema', + (run: Record) => { + run.schemaVersion = 2; + }, + ], + ] as const) { + it(`rejects inconsistent ${name}`, () => { + const run: Record = JSON.parse(desktopEvidenceFixture(smoke).canonical); + mutate(run); + assert.throws(() => + parseNativeReport({ schemaVersion: 1, format: 'desktop-test', desktopRun: JSON.stringify(run), storyReport: smoke }), + ); + }); + } + + it('rejects a callback status that contradicts the canonical result', () => { + const { canonical } = desktopEvidenceFixture(smoke); + assert.throws( + () => + parseNativeReport({ + schemaVersion: 1, + format: 'desktop-test', + desktopRun: canonical, + storyReport: JSON.stringify({ ...aggregate, wdio: [nativeCase] }), + }), + /does not match/, + ); + }); + + it('rejects a nested evidence envelope instead of recursively trusting it as a companion', () => { + const { canonical, envelope } = desktopEvidenceFixture(smoke); + assert.throws( + () => + parseNativeReport({ + schemaVersion: 1, + format: 'desktop-test', + desktopRun: canonical, + storyReport: envelope, + }), + /companion/, + ); + }); + + it('rejects successful supervisor status when selected work is unfinished', () => { + const { envelope } = desktopEvidenceFixture(smoke, { + work: [{ workId: 'unfinished', label: 'selected test cohort', status: 'not-run' }], + }); + assert.throws(() => parseNativeReport(JSON.parse(envelope)), /incomplete execution/); + }); + + it('rejects a native identity that is inconsistent with its canonical test ID', () => { + const { run } = desktopEvidenceFixture(smoke); + run.tests[1].testId = `csf-${'0'.repeat(64)}`; + assert.throws( + () => + parseNativeReport({ + schemaVersion: 1, + format: 'desktop-test', + desktopRun: JSON.stringify(run), + storyReport: smoke, + }), + /authored callback identity/, + ); + }); + + it('accepts the new direct optional-cohort skipped status without calling it a pass', () => { + const result = parseNativeReport({ + ...direct, + status: 'skipped', + tests: [{ ...nativeCase, status: 'skipped', skipReason: 'physical input disabled' }], + }); + assert.equal(result.cases[0].status, 'skipped'); + assert.equal(result.status, 'failed'); + assert.match(result.error ?? '', /skipped/); + }); + }); + + it('records a raw historical pass without accepting it as current qualification', () => { + const report = assessCheckbox(options({ context: undefined, expectedBuildId: undefined, maxAgeHours: undefined })); + assert.equal(report.evidence.status, 'unverified'); + assert.equal(report.requirements[1].check?.recordedOutcome, 'passed'); + assert.equal(report.requirements[1].check?.evidenceOutcome, 'unverified'); + assert.equal(report.requirements[1].outcome, 'insufficient-evidence'); + }); + + it('uses aggregate wdio results, not enclosing success or portable tests', () => { + const report = assessCheckbox(options({ report: parseNativeReport(aggregate), context: undefined })); + assert.equal(report.evidence.runStatus, 'passed'); + assert.equal(report.evidence.caseCounts.skipped, 1); + assert.equal(report.evidence.caseCounts.passed, 0); + assert.equal(report.requirements[1].check?.recordedOutcome, 'skipped'); + assert.match(report.requirements[1].check?.reason ?? '', /physical pointer/); + assert.equal(report.assessmentOutcome, 'insufficient-evidence'); + }); + + it('keeps a matched skip as a skip and never counts it as a pass', () => { + const report = assessCheckbox( + options({ + report: parseNativeReport(aggregate), + context: { ...context, physicalClick: false }, + }), + ); + assert.equal(report.evidence.status, 'matched'); + assert.equal(report.requirements[1].check?.evidenceOutcome, 'skipped'); + assert.equal(report.requirements[1].outcome, 'insufficient-evidence'); + }); + + it('blocks mapped evidence on a case failure without attributing the precise broken requirement', () => { + const report = assessCheckbox( + options({ + report: parseNativeReport({ ...direct, status: 'failed', tests: [{ ...nativeCase, status: 'failed', error: 'assertion failed' }] }), + }), + ); + assert.equal(report.assessmentOutcome, 'failed'); + assert.equal(report.requirements[1].outcome, 'blocked-by-failure'); + assert.equal(report.requirements[3].outcome, 'blocked-by-failure'); + assert.equal(report.requirements[1].check?.reason, 'assertion failed'); + }); + + it('separates runner errors from product assertion failures and does not accept partial-run passes', () => { + const report = assessCheckbox(options({ report: parseNativeReport({ ...direct, status: 'failed', error: 'driver disconnected' }) })); + assert.equal(report.assessmentOutcome, 'execution-failed'); + assert.equal(report.evidence.runError, 'driver disconnected'); + assert.equal(report.requirements[1].check?.recordedOutcome, 'passed'); + assert.equal(report.requirements[1].check?.evidenceOutcome, 'unverified'); + }); + + for (const report of [ + undefined, + parseNativeReport({ ...direct, tests: [] }), + parseNativeReport({ ...direct, tests: [{ ...nativeCase, storyId: 'components-button--default' }] }), + ]) { + it(`reports missing coverage for ${report === undefined ? 'absent report' : report.cases.length ? 'unrelated cases' : 'empty cases'}`, () => { + const result = assessCheckbox(options({ report, context: undefined })); + assert.equal(result.requirements[1].check?.recordedOutcome, 'not-run'); + assert.equal(result.requirements[1].outcome, 'insufficient-evidence'); + }); + } + + it('detects endpoint and native-host mismatches even without provenance context', () => { + const report = assessCheckbox(options({ report: parseNativeReport({ ...direct, endpoint: 'windows' }), context: undefined })); + assert.equal(report.assessmentHealth, 'blocked'); + assert.equal(report.evidence.status, 'mismatched'); + const host = assessCheckbox(options({ report: parseNativeReport({ ...aggregate, targetId: 'different-host' }), context: undefined })); + assert.equal(host.evidence.status, 'mismatched'); + }); + + for (const [field, value] of Object.entries({ + reportSha256: '1'.repeat(64), + sourceCommit: '2'.repeat(40), + contractDigest: '3'.repeat(64), + platformManifestDigest: '4'.repeat(64), + buildId: 'another-build', + finishedAt: '2026-10-22T00:00:00.000Z', + physicalClick: false, + })) { + it(`rejects mismatched ${field} provenance`, () => { + const report = assessCheckbox(options({ context: parseEvidenceContext({ ...context, [field]: value }) })); + assert.equal(report.evidence.status, 'mismatched'); + assert.equal(report.requirements[1].check?.evidenceOutcome, 'mismatched'); + assert.equal(report.evidenceSupportedStatus, null); + }); + } + + it('does not accept a context timestamp that contradicts the aggregate', () => { + const report = assessCheckbox( + options({ report: parseNativeReport(aggregate), context: { ...context, finishedAt: '2026-10-21T20:01:00.000Z' } }), + ); + assert.equal(report.evidence.status, 'mismatched'); + }); + + it('requires explicit freshness/build policy and a commit-identifiable working tree', () => { + for (const missing of [{ maxAgeHours: undefined }, { expectedBuildId: undefined }, { workingTreeDirty: true }]) { + assert.equal(assessCheckbox(options(missing)).evidence.status, 'unverified'); + } + assert.equal(assessCheckbox(options({ maxAgeHours: 1 })).evidence.status, 'matched'); + const stale = assessCheckbox(options({ maxAgeHours: 0.5 })); + assert.equal(stale.assessmentHealth, 'stale'); + assert.equal(stale.requirements[1].check?.evidenceOutcome, 'stale'); + assert.equal(stale.evidenceSupportedStatus, null); + }); + + it('invalidates changed reviewed inputs and never promotes an unmapped requirement', () => { + const report = assessCheckbox( + options({ + contract: { + ...contract, + mappingCurrent: false, + requirements: [...contract.requirements, { id: 'CBX-007', summary: 'New contract requirement', declaredEvidence: [] }], + }, + }), + ); + assert.equal(report.assessmentHealth, 'stale'); + assert.equal(report.requirements[1].check?.evidenceOutcome, 'stale'); + assert.equal(report.requirements[6].check, null); + assert.equal(report.requirements[6].outcome, 'insufficient-evidence'); + }); + + it('fingerprints normalized contract/story content and detects a changed source file', () => { + const directory = fixture(); + const component = 'packages/agentic/components/src/components/checkbox'; + for (const file of [ + 'SPEC.md', + 'spec/source.json', + 'spec/interaction.md', + 'spec/accessibility.md', + 'spec/tokens.yaml', + 'spec/usage.md', + 'checkbox.stories.tsx', + ]) { + const destination = path.join(directory, component, file); + mkdirSync(path.dirname(destination), { recursive: true }); + copyFileSync(path.join(root, component, file), destination); + writeFileSync(destination, readFileSync(destination, 'utf8').replace(/\r?\n/g, '\r\n')); + } + assert.equal(readCheckboxContract(directory).mappingCurrent, true); + writeFileSync(path.join(directory, component, 'checkbox.stories.tsx'), '// changed test\n'); + assert.equal(readCheckboxContract(directory).mappingCurrent, false); + }); + + it('produces deterministic decisions and escapes untrusted Markdown', () => { + assert.deepEqual(assessCheckbox(options()), assessCheckbox(options())); + const report = assessCheckbox( + options({ + report: parseNativeReport({ + ...direct, + tests: [{ ...nativeCase, status: 'skipped', skipReason: '