From 9b2644f6b2e82daf1428efaccd191b45d41054ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 28 Oct 2026 23:05:05 +0100 Subject: [PATCH 1/3] docs(capabilities): drop the version stamp, name each table once The capabilities page said it covered the unreleased in-tree version (0.21.34-dev) and the day it was generated, so the published page named a version users cannot install and changed on every release. It now carries neither and changes only when the facts or annotations do; the drift guard compares it whole. Each table heading names its section and target group, so search lists five distinct entries instead of five "Hosted provider sessions". Co-Authored-By: Claude Opus 5.5 --- scripts/capability-matrix/generate.ts | 47 +++++----------------- test/repo-guards/capability-matrix.test.ts | 44 +++++--------------- website/docs/docs/capabilities.md | 41 +++++++++---------- 3 files changed, 39 insertions(+), 93 deletions(-) diff --git a/scripts/capability-matrix/generate.ts b/scripts/capability-matrix/generate.ts index 4b91efbc9b..c9983f5c4c 100644 --- a/scripts/capability-matrix/generate.ts +++ b/scripts/capability-matrix/generate.ts @@ -2,8 +2,9 @@ // `website/docs/docs/capabilities.md` from the collected matrix. // // Table cells, column sets, and the partial-support notes all come from -// `collect.ts`; this file adds the labels and links from `annotations.ts` and the -// header's version stamp. Columns whose command sets are identical print once. +// `collect.ts`; this file adds the labels and links from `annotations.ts`. Columns +// whose command sets are identical print once. The page carries no version or date, +// so it changes only when the facts or the annotations do. // Without `--write` the script compares the page on disk against the generated // page and fails when they differ, which is what the drift guard runs. @@ -25,21 +26,14 @@ function targetById(id: string): Target { } export type MatrixData = Readonly<{ - version: string; - generatedAt: string; commands: readonly string[]; support: ReadonlyMap>; operations: ReadonlyMap>; }>; async function collectPageData(): Promise { - const packageJson = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'package.json'), 'utf8')) as { - version: string; - }; const { commands, support, operations } = await collectMatrix(); return { - version: packageJson.version, - generatedAt: new Date().toISOString().slice(0, 10), commands, support: new Map(Object.entries(support).map(([id, list]) => [id, new Set(list)])), operations: new Map( @@ -271,7 +265,6 @@ export function renderPage(data: MatrixData): string { '---', '', 'The target tables below are generated from the runtime command-admission facts that back the `capabilities` command: a command that leaves the code leaves this page. Labels, limits, and integration rows are maintained beside the generator, and command availability does not establish support for every operation, subcommand, or device model.', - `It covers agent-device ${data.version}, generated ${data.generatedAt}.`, '', '**Yes** means the facts admit every command in that row on the target and every operation the row advertises. **No** means at least one is refused. A qualified cell names the operations that answer there. Targets that share the same available operations print as one column.', '', @@ -294,7 +287,9 @@ export function renderPage(data: MatrixData): string { lines.push(`## ${section.title}`, ''); for (const { caption, group } of GROUPS) { const groupColumns = columns.get(group) as Column[]; - lines.push(`### ${caption}`, ''); + // Section and group in one heading, so search results and anchors stay distinct. + const groupLabel = caption.replace(/^[A-Z][a-z]/, (word) => word.toLowerCase()); + lines.push(`### ${section.title}: ${groupLabel}`, ''); lines.push(...tableFor(section.rows, groupColumns, data), ''); } const limits = rowLimits(section.rows); @@ -333,39 +328,15 @@ function firstDifference(committed: string, generated: string): string { return '(none)'; } -/** - * The generation date in the page header's provenance line; see - * {@link withNeutralProvenance}. Matching the full line structure keeps every - * other word in the comparison: a change to the sentence, a different version, - * or text appended after the date no longer matches and shows as drift. - */ -const PROVENANCE_LINE = /^(It covers agent-device [0-9][^,\n]*, generated )\d{4}-\d{2}-\d{2}(\.)$/m; - -/** - * The header's generation date is provenance, not a fact: it moves with every - * new day the page is written. Drift comparison masks that one date so the - * facts can be re-collected without a diff, while the version stays compared: - * a page stamped with an older version than package.json is stale and fails - * until it is regenerated. The pattern demands the exact shape the renderer - * writes, so edited or appended text around the stamp still shows as drift. - */ -function withNeutralProvenance(page: string): string { - return page.replace(PROVENANCE_LINE, '$1$2'); -} - /** * The drift report between the committed page and the generated one, or null - * when they agree. Both are date-masked: the header's generation date moves - * with every regeneration, while the version stamp and every table cell stay - * compared exactly. + * when they agree. The page carries no date or version, so it is compared whole. */ export function driftReport(committed: string, generated: string): string | null { - const a = withNeutralProvenance(committed); - const b = withNeutralProvenance(generated); - if (a === b) return null; + if (committed === generated) return null; return ( `${PAGE_PATH} is stale. Run \`pnpm generate:capabilities\`.\nFirst difference:\n` + - firstDifference(a, b) + firstDifference(committed, generated) ); } diff --git a/test/repo-guards/capability-matrix.test.ts b/test/repo-guards/capability-matrix.test.ts index e4c0d3486f..b3ade7e35f 100644 --- a/test/repo-guards/capability-matrix.test.ts +++ b/test/repo-guards/capability-matrix.test.ts @@ -25,10 +25,8 @@ import type { CollectedMatrix } from '../../scripts/capability-matrix/run.ts'; // exactly the commands the registry derives from device facts, and every // qualified cell still names facts that hold. All three are checked, so a // capability cannot be added, renamed, or removed without an edit under -// scripts/capability-matrix/ and a regenerated page. The page's generation-date -// is provenance rather than a fact, so it is masked before the comparison — and -// negative tests pin what the mask does and does not hide: the version is a -// fact about the tree and stays compared. +// scripts/capability-matrix/ and a regenerated page. The page carries no version +// or date, so the comparison is exact. // // The facts are collected in a child process on purpose: the provider runtimes // need `node:module` loader hooks to keep their third-party SDKs out of the run, @@ -62,14 +60,7 @@ function toSets( const support = toSets(matrix.support); const operations = toSets(matrix.operations); const committed = fs.readFileSync(path.join(REPO_ROOT, PAGE_PATH), 'utf8'); -// The real package version, so this guard also proves the committed page is -// stamped with the version the tree builds; the generated page must agree. -const packageVersion = ( - JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'package.json'), 'utf8')) as { version: string } -).version; const data: MatrixData = { - version: packageVersion, - generatedAt: '2000-01-01', commands: matrix.commands, support, operations, @@ -119,30 +110,15 @@ describe('capabilities page matches runtime facts', () => { }); test('drift detection proves itself', () => { - // The committed page is current, so every mutation below must report drift — - // including one inside the masked provenance line's surroundings. + // The committed page is current, so every mutation below must report drift. assert.notEqual( driftReport(committed.replace(/Yes \| Yes \| No/, 'Yes | No | No'), committed), null, ); - const dated = committed.replace(/generated \d{4}-\d{2}-\d{2}\./, 'generated 2000-01-01.'); - assert.equal(driftReport(dated, committed), null, 'the generation date is provenance'); - const versioned = committed.replace( - /It covers agent-device [^,]+,/, - 'It covers agent-device 9.9.9,', - ); - assert.notEqual( - driftReport(versioned, committed), - null, - 'a page stamped with another version than the tree must read as stale', - ); assert.notEqual( - driftReport( - committed.replace(/generated [^.]+\./, 'generated 2000-01-01 and hand edited.'), - committed, - ), + driftReport(committed.replace(/tables below/, 'tables here'), committed), null, - 'text appended to the provenance sentence must not be masked', + 'any edit to the prose reads as drift', ); assert.notEqual( driftReport(committed.replace(/\n## Inspect and verify the UI/, ''), committed), @@ -315,10 +291,10 @@ describe('capabilities page matches runtime facts', () => { `Target "${target.id}" (label "${target.label}") is not named in any column header on the page.`, ); } - // The committed page carries the real stamp, not the mask. - assert.match( - committed, - /^It covers agent-device \d+\.\d+\.\d+[^,\n]*, generated \d{4}-\d{2}-\d{2}\.$/m, - ); + // No version or date: the page changes only with the facts. + assert.doesNotMatch(committed, /It covers agent-device|generated \d{4}-\d{2}-\d{2}/); + // Every heading is distinct, so search results and anchors name one table each. + const headings = committed.split('\n').filter((line) => line.startsWith('#')); + assert.deepEqual(headings, [...new Set(headings)], 'the page repeats a heading'); }); }); diff --git a/website/docs/docs/capabilities.md b/website/docs/docs/capabilities.md index 8ec2111857..89b880b0fa 100644 --- a/website/docs/docs/capabilities.md +++ b/website/docs/docs/capabilities.md @@ -4,7 +4,6 @@ description: Which commands each target supports, read from the runtime code tha --- The target tables below are generated from the runtime command-admission facts that back the `capabilities` command: a command that leaves the code leaves this page. Labels, limits, and integration rows are maintained beside the generator, and command availability does not establish support for every operation, subcommand, or device model. -It covers agent-device 0.21.34-dev, generated 2026-10-27. **Yes** means the facts admit every command in that row on the target and every operation the row advertises. **No** means at least one is refused. A qualified cell names the operations that answer there. Targets that share the same available operations print as one column. @@ -23,7 +22,7 @@ A provider column answers for that provider in its default configuration. Provid ## Inspect and verify the UI -### Mobile targets +### Inspect and verify the UI: mobile targets | Capability and reference | iOS Simulator | iOS device (CoreDevice) | Android emulator | Android device | HarmonyOS emulator | HarmonyOS device | | --- | --- | --- | --- | --- | --- | --- | @@ -32,7 +31,7 @@ A provider column answers for that provider in its default configuration. Provid | [Screenshots](/docs/commands#media-and-logs) | Yes | Yes | Yes | Yes | Yes | Yes | | [Wait for text, selectors, and timeouts](/docs/commands#wait-and-alerts) | Yes | Yes | Yes | Yes | Yes | Yes | -### TV targets +### Inspect and verify the UI: TV targets | Capability and reference | Android TV emulator | Android TV / Fire OS device | tvOS Simulator | tvOS device | Vega OS (VVD) | | --- | --- | --- | --- | --- | --- | @@ -41,7 +40,7 @@ A provider column answers for that provider in its default configuration. Provid | [Screenshots](/docs/commands#media-and-logs) | Yes | Yes | Yes | Yes | No | | [Wait for text, selectors, and timeouts](/docs/commands#wait-and-alerts) | Yes | Yes | Yes | Yes | No | -### Desktop and web targets +### Inspect and verify the UI: desktop and web targets | Capability and reference | macOS desktop | macOS app | Linux desktop | Web browser | | --- | --- | --- | --- | --- | @@ -50,7 +49,7 @@ A provider column answers for that provider in its default configuration. Provid | [Screenshots](/docs/commands#media-and-logs) | Yes | Yes | Yes | Yes | | [Wait for text, selectors, and timeouts](/docs/commands#wait-and-alerts) | Yes | Yes | Yes | Yes | -### Hosted provider sessions +### Inspect and verify the UI: hosted provider sessions | Capability and reference | BrowserStack Android / TestMu AI Android | BrowserStack iOS / TestMu AI iOS | AWS Device Farm Android | AWS Device Farm iOS | Limrun iOS | Limrun Android | | --- | --- | --- | --- | --- | --- | --- | @@ -65,7 +64,7 @@ What a **Yes** does not claim: ## Interact with the app -### Mobile targets +### Interact with the app: mobile targets | Capability and reference | iOS Simulator | iOS device (CoreDevice) | Android emulator | Android device | HarmonyOS emulator | HarmonyOS device | | --- | --- | --- | --- | --- | --- | --- | @@ -78,7 +77,7 @@ What a **Yes** does not claim: | [Keyboard dismiss, enter, and status](/docs/commands#keyboard) | Dismiss and enter; status unsupported | Dismiss and enter; status unsupported | Yes | Yes | Dismiss and enter; status unsupported | Dismiss and enter; status unsupported | | [React Native overlay dismissal](/docs/commands#react-native-overlays) | Yes | Yes | Yes | Yes | Yes | Yes | -### TV targets +### Interact with the app: TV targets | Capability and reference | Android TV emulator | Android TV / Fire OS device | tvOS Simulator | tvOS device | Vega OS (VVD) | | --- | --- | --- | --- | --- | --- | @@ -91,7 +90,7 @@ What a **Yes** does not claim: | [Keyboard dismiss, enter, and status](/docs/commands#keyboard) | Yes | Yes | No | No | No | | [React Native overlay dismissal](/docs/commands#react-native-overlays) | Yes | Yes | Yes | Yes | No | -### Desktop and web targets +### Interact with the app: desktop and web targets | Capability and reference | macOS desktop | macOS app | Linux desktop | Web browser | | --- | --- | --- | --- | --- | @@ -104,7 +103,7 @@ What a **Yes** does not claim: | [Keyboard dismiss, enter, and status](/docs/commands#keyboard) | No | No | No | No | | [React Native overlay dismissal](/docs/commands#react-native-overlays) | Yes | Yes | Yes | Yes | -### Hosted provider sessions +### Interact with the app: hosted provider sessions | Capability and reference | BrowserStack Android / TestMu AI Android | BrowserStack iOS / TestMu AI iOS | AWS Device Farm Android | AWS Device Farm iOS | Limrun iOS | Limrun Android | | --- | --- | --- | --- | --- | --- | --- | @@ -125,7 +124,7 @@ What a **Yes** does not claim: ## Apps and devices -### Mobile targets +### Apps and devices: mobile targets | Capability and reference | iOS Simulator | iOS device (CoreDevice) | Android emulator | Android device | HarmonyOS emulator | HarmonyOS device | | --- | --- | --- | --- | --- | --- | --- | @@ -140,7 +139,7 @@ What a **Yes** does not claim: | [Shut down a simulator or emulator](/docs/commands#navigation) | Yes | No | Yes | No | No | No | | [Prepare the Apple runner](/docs/commands#prepare-apple-runner) | Yes | Yes | No | No | No | No | -### TV targets +### Apps and devices: TV targets | Capability and reference | Android TV emulator | Android TV / Fire OS device | tvOS Simulator | tvOS device | Vega OS (VVD) | | --- | --- | --- | --- | --- | --- | @@ -155,7 +154,7 @@ What a **Yes** does not claim: | [Shut down a simulator or emulator](/docs/commands#navigation) | Yes | No | Yes | No | No | | [Prepare the Apple runner](/docs/commands#prepare-apple-runner) | No | No | Yes | Yes | No | -### Desktop and web targets +### Apps and devices: desktop and web targets | Capability and reference | macOS desktop | macOS app | Linux desktop | Web browser | | --- | --- | --- | --- | --- | @@ -170,7 +169,7 @@ What a **Yes** does not claim: | [Shut down a simulator or emulator](/docs/commands#navigation) | No | No | No | No | | [Prepare the Apple runner](/docs/commands#prepare-apple-runner) | Yes | Yes | No | No | -### Hosted provider sessions +### Apps and devices: hosted provider sessions | Capability and reference | BrowserStack Android / TestMu AI Android | BrowserStack iOS / TestMu AI iOS | AWS Device Farm Android | AWS Device Farm iOS | Limrun iOS | Limrun Android | | --- | --- | --- | --- | --- | --- | --- | @@ -192,7 +191,7 @@ What a **Yes** does not claim: ## Evidence and diagnostics -### Mobile targets +### Evidence and diagnostics: mobile targets | Capability and reference | iOS Simulator | iOS device (CoreDevice) | Android emulator | Android device | HarmonyOS emulator | HarmonyOS device | | --- | --- | --- | --- | --- | --- | --- | @@ -202,7 +201,7 @@ What a **Yes** does not claim: | [Performance samples (frames, memory, CPU profiles)](/docs/debugging-profiling#performance-snapshots) | Yes | Frames, memory, and CPU profiles; memory-snapshot artifact reports unavailable | Yes | Yes | Process RSS memory sample only; frames, memory-snapshot artifacts, CPU profiles, and native captures report unavailable | Process RSS memory sample only; frames, memory-snapshot artifacts, CPU profiles, and native captures report unavailable | | [Audio probe](/docs/debugging-profiling#audio-probes) | Yes | No | Yes | No | No | No | -### TV targets +### Evidence and diagnostics: TV targets | Capability and reference | Android TV emulator | Android TV / Fire OS device | tvOS Simulator | tvOS device | Vega OS (VVD) | | --- | --- | --- | --- | --- | --- | @@ -212,7 +211,7 @@ What a **Yes** does not claim: | [Performance samples (frames, memory, CPU profiles)](/docs/debugging-profiling#performance-snapshots) | Yes | Yes | Yes | Yes | No | | [Audio probe](/docs/debugging-profiling#audio-probes) | Yes | No | Yes | No | No | -### Desktop and web targets +### Evidence and diagnostics: desktop and web targets | Capability and reference | macOS desktop | macOS app | Linux desktop | Web browser | | --- | --- | --- | --- | --- | @@ -222,7 +221,7 @@ What a **Yes** does not claim: | [Performance samples (frames, memory, CPU profiles)](/docs/debugging-profiling#performance-snapshots) | Yes | Yes | No | No | | [Audio probe](/docs/debugging-profiling#audio-probes) | Yes | No | No | Page media probe only; host capture unsupported | -### Hosted provider sessions +### Evidence and diagnostics: hosted provider sessions | Capability and reference | BrowserStack Android / TestMu AI Android | BrowserStack iOS / TestMu AI iOS | AWS Device Farm Android | AWS Device Farm iOS | Limrun iOS | Limrun Android | | --- | --- | --- | --- | --- | --- | --- | @@ -240,7 +239,7 @@ What a **Yes** does not claim: ## System and hardware -### Mobile targets +### System and hardware: mobile targets | Capability and reference | iOS Simulator | iOS device (CoreDevice) | Android emulator | Android device | HarmonyOS emulator | HarmonyOS device | | --- | --- | --- | --- | --- | --- | --- | @@ -257,7 +256,7 @@ What a **Yes** does not claim: | [App event triggers through the app hook](/docs/commands#app-event-triggers-app-hook) | Yes | Yes | Yes | Yes | No | No | | [Call and SMS simulation](/docs/commands#navigation) | No | No | Yes | No | No | No | -### TV targets +### System and hardware: TV targets | Capability and reference | Android TV emulator | Android TV / Fire OS device | tvOS Simulator | tvOS device | Vega OS (VVD) | | --- | --- | --- | --- | --- | --- | @@ -274,7 +273,7 @@ What a **Yes** does not claim: | [App event triggers through the app hook](/docs/commands#app-event-triggers-app-hook) | Yes | Yes | Yes | Yes | No | | [Call and SMS simulation](/docs/commands#navigation) | Yes | No | No | No | No | -### Desktop and web targets +### System and hardware: desktop and web targets | Capability and reference | macOS desktop | macOS app | Linux desktop | Web browser | | --- | --- | --- | --- | --- | @@ -291,7 +290,7 @@ What a **Yes** does not claim: | [App event triggers through the app hook](/docs/commands#app-event-triggers-app-hook) | Yes | Yes | No | No | | [Call and SMS simulation](/docs/commands#navigation) | No | No | No | No | -### Hosted provider sessions +### System and hardware: hosted provider sessions | Capability and reference | BrowserStack Android / TestMu AI Android | BrowserStack iOS / TestMu AI iOS | AWS Device Farm Android | AWS Device Farm iOS | Limrun iOS | Limrun Android | | --- | --- | --- | --- | --- | --- | --- | From a50785d3bc180a40eca8445fadf4f5a5e3aa4e7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 28 Oct 2026 23:29:11 +0100 Subject: [PATCH 2/3] chore(gates): stop restamping the capabilities page after a release The page no longer carries a version, so the mark-released job's generate:capabilities run would only re-collect unchanged facts. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 556c57b2ba..b05fb40080 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -251,11 +251,6 @@ jobs: # main must never carry a published version. Retried because merges can # land on main while the release runs. - # - # The capabilities page stamps the version it covers and the drift check - # compares that stamp to package.json, so the version bump restamps the - # page in the same commit. The collector reads runtime sources and needs - # no device. - name: Commit the next development version env: VERSION: ${{ needs.plan.outputs.version }} @@ -267,7 +262,6 @@ jobs: git fetch origin main git reset --hard origin/main next="$(node scripts/release-version.mjs advance "$VERSION")" - pnpm generate:capabilities if git diff --quiet; then echo "main already builds towards $next." exit 0 From bce95a99b5b86842e5f7541654e3228269aba96f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 29 Oct 2026 07:48:08 +0100 Subject: [PATCH 3/3] refactor(capabilities): store the group captions as they render The headings lowercased each caption's first letter with a regex; the captions are now written as they appear, so the regex goes. The page is unchanged. Co-Authored-By: Claude Opus 5.5 --- scripts/capability-matrix/generate.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/capability-matrix/generate.ts b/scripts/capability-matrix/generate.ts index c9983f5c4c..b8b0011497 100644 --- a/scripts/capability-matrix/generate.ts +++ b/scripts/capability-matrix/generate.ts @@ -223,10 +223,10 @@ function rowLimits(rows: readonly FactRow[]): string[] { } const GROUPS: ReadonlyArray<{ readonly group: TargetGroup; readonly caption: string }> = [ - { group: 'local-mobile', caption: 'Mobile targets' }, + { group: 'local-mobile', caption: 'mobile targets' }, { group: 'local-tv', caption: 'TV targets' }, - { group: 'local-desktop-web', caption: 'Desktop and web targets' }, - { group: 'provider', caption: 'Hosted provider sessions' }, + { group: 'local-desktop-web', caption: 'desktop and web targets' }, + { group: 'provider', caption: 'hosted provider sessions' }, ]; export function renderPage(data: MatrixData): string { @@ -288,8 +288,7 @@ export function renderPage(data: MatrixData): string { for (const { caption, group } of GROUPS) { const groupColumns = columns.get(group) as Column[]; // Section and group in one heading, so search results and anchors stay distinct. - const groupLabel = caption.replace(/^[A-Z][a-z]/, (word) => word.toLowerCase()); - lines.push(`### ${section.title}: ${groupLabel}`, ''); + lines.push(`### ${section.title}: ${caption}`, ''); lines.push(...tableFor(section.rows, groupColumns, data), ''); } const limits = rowLimits(section.rows);