From 893cfef267ad3accd91daafbce7bc30e4e926ab4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 13 Sep 2026 19:58:11 +0200 Subject: [PATCH 1/4] fix(android): report the clip an Android recording really captured Android `screenrecord` encodes a frame only when the screen changes, so a window that ends on an unchanged screen returns a video far shorter than the requested duration, and `record stop` had nothing to say about it: the reported `durationMs` is host wall clock from `record start` until the export finished, which is not the length of the file that was just pulled. `record stop` now measures the pulled MP4 timelines and reports them as `capturedDurationMs`, and warns with the clip length against the window when the video is two or more seconds short. The window is measured on the device's own elapsed clock, read before the stop signal and at launch, because host wall clock drifts against the clock the encoder timestamps frames with; an unreadable clock or a chunk that answers no duration costs the caller the claim, never the recording. Measuring the timeline needed an ISO-BMFF box walk, which now lives in `@agent-device/capture-kit/recording-mp4-duration` and replaces the private top-level atom scan that MP4 container detection was doing. Stop replay through daemon recovery carries the field too, so a completion read back from the session resource reports the same numbers it did live. --- packages/capture-kit/package.json | 8 ++ .../src/recording/mp4-atoms.test.ts | 79 +++++++++++++++ .../capture-kit/src/recording/mp4-atoms.ts | 99 +++++++++++++++++++ .../src/recording/mp4-duration.test.ts | 72 ++++++++++++++ .../capture-kit/src/recording/mp4-duration.ts | 56 +++++++++++ .../capture-kit/src/recording/mp4.fixtures.ts | 39 ++++++++ .../capture-kit/src/recording/video.test.ts | 34 ++++++- packages/capture-kit/src/recording/video.ts | 49 ++------- packages/contracts/src/recording.ts | 1 + .../src/screen-recording-runtime-host.ts | 7 ++ .../contracts/src/screen-recording-runtime.ts | 7 ++ .../src/recording/captured-window.test.ts | 58 +++++++++++ .../src/recording/captured-window.ts | 46 +++++++++ .../src/recording/completion.test.ts | 62 ++++++++++++ .../src/recording/completion.ts | 57 ++++++----- .../src/recording/device-clock.test.ts | 40 ++++++++ .../src/recording/device-clock.ts | 28 ++++++ .../src/recording/finalize.test.ts | 62 ++++++++++++ .../src/recording/finalize.ts | 22 +++-- .../src/recording/fixtures.ts | 2 + .../platform-android/src/recording/launch.ts | 11 ++- .../src/recording/manifest-validation.test.ts | 38 +++++++ .../src/recording/manifest-validation.ts | 3 +- .../platform-android/src/recording/runtime.ts | 1 + src/commands/recording/index.ts | 2 +- .../screen-recording-stop-recovery.test.ts | 2 + .../__tests__/record-runtime-response.test.ts | 18 ++++ .../handlers/record-runtime-response.ts | 3 + .../screen-recording-session-resource.ts | 3 + src/daemon/screen-recording-stop-recovery.ts | 1 + src/mcp/command-output-schemas.ts | 1 + ...time-screen-recording-android-host.test.ts | 46 +++++++++ ...m-runtime-screen-recording-android-host.ts | 14 ++- website/docs/docs/commands.md | 1 + 34 files changed, 893 insertions(+), 79 deletions(-) create mode 100644 packages/capture-kit/src/recording/mp4-atoms.test.ts create mode 100644 packages/capture-kit/src/recording/mp4-atoms.ts create mode 100644 packages/capture-kit/src/recording/mp4-duration.test.ts create mode 100644 packages/capture-kit/src/recording/mp4-duration.ts create mode 100644 packages/capture-kit/src/recording/mp4.fixtures.ts create mode 100644 packages/platform-android/src/recording/captured-window.test.ts create mode 100644 packages/platform-android/src/recording/captured-window.ts create mode 100644 packages/platform-android/src/recording/completion.test.ts create mode 100644 packages/platform-android/src/recording/device-clock.test.ts create mode 100644 packages/platform-android/src/recording/device-clock.ts diff --git a/packages/capture-kit/package.json b/packages/capture-kit/package.json index 4a997111cf..76acc5357d 100644 --- a/packages/capture-kit/package.json +++ b/packages/capture-kit/package.json @@ -82,6 +82,14 @@ "types": "./src/react-native-overlay.ts", "default": "./src/react-native-overlay.ts" }, + "./recording-mp4-fixtures": { + "types": "./src/recording/mp4.fixtures.ts", + "default": "./src/recording/mp4.fixtures.ts" + }, + "./recording-mp4-duration": { + "types": "./src/recording/mp4-duration.ts", + "default": "./src/recording/mp4-duration.ts" + }, "./recording-output-path": { "types": "./src/recording/output-path.ts", "default": "./src/recording/output-path.ts" diff --git a/packages/capture-kit/src/recording/mp4-atoms.test.ts b/packages/capture-kit/src/recording/mp4-atoms.test.ts new file mode 100644 index 0000000000..580722ef48 --- /dev/null +++ b/packages/capture-kit/src/recording/mp4-atoms.test.ts @@ -0,0 +1,79 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { describe, expect, test } from 'vitest'; +import { mkdtempForTestSync } from '../tmp-dir.fixtures.ts'; +import { findMp4Atom } from './mp4-atoms.ts'; +import { mp4Atom, mp4AtomWithExtendedSize } from './mp4.fixtures.ts'; + +const directory = mkdtempForTestSync('agent-device-mp4-atoms-'); + +function write(name: string, contents: Buffer): string { + const filePath = path.join(directory, name); + fs.writeFileSync(filePath, contents); + return filePath; +} + +describe('findMp4Atom', () => { + test('locates a nested box and reports where its payload starts', () => { + const movieHeader = mp4Atom('mvhd', Buffer.alloc(100)); + const file = write( + 'nested.mp4', + Buffer.concat([mp4Atom('ftyp', Buffer.alloc(8)), mp4Atom('moov', movieHeader)]), + ); + const atom = findMp4Atom(file, ['moov', 'mvhd']); + expect(atom?.type).toBe('mvhd'); + expect(atom?.offset).toBe(24); + expect(atom?.size).toBe(108); + expect(findMp4Atom(file, ['moov', 'udta'])).toBeUndefined(); + }); + + test('walks past a 64-bit sized box', () => { + const file = write( + 'extended.mp4', + Buffer.concat([ + mp4AtomWithExtendedSize('mdat', Buffer.alloc(64)), + mp4Atom('moov', Buffer.alloc(16)), + ]), + ); + expect(findMp4Atom(file, ['moov'])?.offset).toBe(16 + 64); + }); + + test('accepts a box whose declared size runs to end of file', () => { + const streamingMdat = Buffer.concat([ + Buffer.from([0, 0, 0, 0]), + Buffer.from('mdat', 'latin1'), + Buffer.alloc(32), + ]); + const file = write( + 'streaming.mp4', + Buffer.concat([mp4Atom('ftyp', Buffer.alloc(8)), streamingMdat]), + ); + const atom = findMp4Atom(file, ['mdat']); + expect(atom?.offset).toBe(16); + expect(atom?.size).toBe(streamingMdat.length); + }); + + test('reports a box that overruns its container and stops looking past it', () => { + const overrunning = Buffer.concat([ + Buffer.from([0, 0, 0, 200]), + Buffer.from('wide', 'latin1'), + Buffer.alloc(40), + ]); + const file = write( + 'overrun.mp4', + Buffer.concat([ + mp4Atom('ftyp', Buffer.alloc(8)), + overrunning, + mp4Atom('moov', Buffer.alloc(8)), + ]), + ); + expect(findMp4Atom(file, ['wide'])?.size).toBe(200); + expect(findMp4Atom(file, ['moov'])).toBeUndefined(); + }); + + test('answers undefined for an absent file, an unreadable path, and a short box', () => { + expect(findMp4Atom(path.join(directory, 'absent.mp4'), ['moov'])).toBeUndefined(); + expect(findMp4Atom(directory, ['moov'])).toBeUndefined(); + expect(findMp4Atom(write('short.mp4', Buffer.from([0, 0])), ['moov'])).toBeUndefined(); + }); +}); diff --git a/packages/capture-kit/src/recording/mp4-atoms.ts b/packages/capture-kit/src/recording/mp4-atoms.ts new file mode 100644 index 0000000000..09b89623fc --- /dev/null +++ b/packages/capture-kit/src/recording/mp4-atoms.ts @@ -0,0 +1,99 @@ +import fs from 'node:fs'; + +// Bounded so a corrupt container cannot widen the walk; a movie header past the budget simply +// reads as absent. +const MAX_SIBLINGS_PER_LEVEL = 32; +const ATOM_HEADER_BYTES = 8; +const EXTENDED_SIZE_BYTES = 8; + +export type Mp4Atom = Readonly<{ + /** Four-character box type, for example `moov`. */ + type: string; + /** Byte offset where the box header starts. */ + offset: number; + headerSize: number; + /** Total box size in bytes, including the header. */ + size: number; +}>; + +/** Locates a box by its ancestor-to-leaf type path, for example `['moov', 'mvhd']`. */ +export function findMp4Atom(filePath: string, names: readonly string[]): Mp4Atom | undefined { + if (names.length === 0) return undefined; + let fd: number | undefined; + try { + fd = fs.openSync(filePath, 'r'); + const root: Mp4Atom = { type: '', offset: 0, headerSize: 0, size: fs.fstatSync(fd).size }; + return findDescendant(fd, root, names, 0); + } catch { + return undefined; + } finally { + if (fd !== undefined) fs.closeSync(fd); + } +} + +function findDescendant( + fd: number, + container: Mp4Atom, + names: readonly string[], + depth: number, +): Mp4Atom | undefined { + for (const atom of childAtoms(fd, container)) { + if (atom.type !== names[depth]) continue; + if (depth + 1 === names.length) return atom; + const match = findDescendant(fd, atom, names, depth + 1); + if (match) return match; + } + return undefined; +} + +function* childAtoms(fd: number, container: Mp4Atom): Generator { + const end = container.offset + container.size; + let offset = container.offset + container.headerSize; + for ( + let seen = 0; + offset + ATOM_HEADER_BYTES <= end && seen < MAX_SIBLINGS_PER_LEVEL; + seen += 1 + ) { + const atom = readAtomHeader(fd, offset, end); + if (!atom) return; + yield atom; + // A box that claims more room than its container holds, or an unrepresentable 64-bit size, + // ends this level: nothing past it can be located. + if ( + !Number.isSafeInteger(atom.size) || + atom.size < atom.headerSize || + atom.offset + atom.size > end + ) + return; + offset += atom.size; + } +} + +function readAtomHeader(fd: number, offset: number, end: number): Mp4Atom | undefined { + const header = readBytes(fd, offset, ATOM_HEADER_BYTES); + if (!header) return undefined; + let size = header.readUInt32BE(0); + let headerSize = ATOM_HEADER_BYTES; + if (size === 1) { + const extended = readBytes(fd, offset + ATOM_HEADER_BYTES, EXTENDED_SIZE_BYTES); + if (!extended) return undefined; + size = Number(extended.readBigUInt64BE(0)); + headerSize += EXTENDED_SIZE_BYTES; + } + // A declared size of 0 means the box runs to the end of its container. + return { + type: header.toString('latin1', 4, 8), + offset, + headerSize, + size: size === 0 ? end - offset : size, + }; +} + +function readBytes(fd: number, offset: number, length: number): Buffer | undefined { + const buffer = Buffer.alloc(length); + try { + return fs.readSync(fd, buffer, 0, length, offset) === length ? buffer : undefined; + } catch { + return undefined; + } +} diff --git a/packages/capture-kit/src/recording/mp4-duration.test.ts b/packages/capture-kit/src/recording/mp4-duration.test.ts new file mode 100644 index 0000000000..00b60e67d4 --- /dev/null +++ b/packages/capture-kit/src/recording/mp4-duration.test.ts @@ -0,0 +1,72 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { describe, expect, test } from 'vitest'; +import { mkdtempForTestSync } from '../tmp-dir.fixtures.ts'; +import { readMp4DurationMs } from './mp4-duration.ts'; +import { mp4Atom, mp4MovieHeader } from './mp4.fixtures.ts'; + +const directory = mkdtempForTestSync('agent-device-mp4-duration-'); + +function recording(name: string, movieHeaderPayload: Buffer): string { + const filePath = path.join(directory, `${name}.mp4`); + fs.writeFileSync( + filePath, + Buffer.concat([ + mp4Atom('ftyp', Buffer.alloc(24)), + mp4Atom('mdat', Buffer.alloc(16)), + mp4Atom('moov', mp4Atom('mvhd', movieHeaderPayload)), + ]), + ); + return filePath; +} + +describe('readMp4DurationMs', () => { + test('reads a version 0 movie header in its own timescale', () => { + const file = recording( + 'v0', + mp4MovieHeader({ version: 0, timescale: 90_000, duration: 630_000 }), + ); + expect(readMp4DurationMs(file)).toBe(7_000); + }); + + test('reports a timeline of zero for a clip of a screen that never changed', () => { + const file = recording('static', mp4MovieHeader({ version: 0, timescale: 1_000, duration: 0 })); + expect(readMp4DurationMs(file)).toBe(0); + }); + + test('reads the 64-bit duration of a version 1 movie header', () => { + const file = recording('v1', mp4MovieHeader({ version: 1, timescale: 1_000, duration: 9_500 })); + expect(readMp4DurationMs(file)).toBe(9_500); + }); + + test('rounds a duration that does not divide evenly into milliseconds', () => { + const file = recording('odd', mp4MovieHeader({ version: 0, timescale: 3, duration: 2 })); + expect(readMp4DurationMs(file)).toBe(667); + }); + + test('answers undefined when the timeline cannot be trusted', () => { + expect( + readMp4DurationMs( + recording('idle', mp4MovieHeader({ version: 0, timescale: 1_000, duration: 0xffff_ffff })), + ), + ).toBeUndefined(); + expect( + readMp4DurationMs( + recording('no-timescale', mp4MovieHeader({ version: 0, timescale: 0, duration: 5_000 })), + ), + ).toBeUndefined(); + const unsupportedVersion = mp4MovieHeader({ version: 0, timescale: 1_000, duration: 5_000 }); + unsupportedVersion.writeUInt8(2, 0); + expect(readMp4DurationMs(recording('version-2', unsupportedVersion))).toBeUndefined(); + }); + + test('answers undefined for a file with no movie header or a truncated one', () => { + const withoutMovieHeader = path.join(directory, 'no-moov.mp4'); + fs.writeFileSync(withoutMovieHeader, mp4Atom('mdat', Buffer.alloc(16))); + expect(readMp4DurationMs(withoutMovieHeader)).toBeUndefined(); + + const truncated = path.join(directory, 'truncated.mp4'); + fs.writeFileSync(truncated, mp4Atom('moov', mp4Atom('mvhd', Buffer.alloc(24)))); + expect(readMp4DurationMs(truncated)).toBeUndefined(); + }); +}); diff --git a/packages/capture-kit/src/recording/mp4-duration.ts b/packages/capture-kit/src/recording/mp4-duration.ts new file mode 100644 index 0000000000..2ecd91c1f2 --- /dev/null +++ b/packages/capture-kit/src/recording/mp4-duration.ts @@ -0,0 +1,56 @@ +import fs from 'node:fs'; +import { findMp4Atom } from './mp4-atoms.ts'; + +const MVHD = ['moov', 'mvhd'] as const; +const MVHD_TIMESCALE_VERSION_0 = 12; +const MVHD_DURATION_VERSION_0 = 16; +const MVHD_TIMESCALE_VERSION_1 = 20; +const MVHD_DURATION_VERSION_1 = 24; +const MVHD_FIXED_LAYOUT_BYTES = 32; +const UNKNOWABLE_32_BIT_DURATION = 0xffffffff; + +/** + * Duration the MP4 timeline actually covers, in milliseconds, or `undefined` when the container + * cannot answer. A screen-recording timeline can be shorter than the window it was captured in: + * `screenrecord` encodes a frame only when the screen changes. + */ +export function readMp4DurationMs(filePath: string): number | undefined { + const header = readMovieHeader(filePath); + if (!header) return undefined; + const version = header[0]; + if (version === 1) { + return movieDurationMs( + header.readUInt32BE(MVHD_TIMESCALE_VERSION_1), + Number(header.readBigUInt64BE(MVHD_DURATION_VERSION_1)), + ); + } + if (version !== 0) return undefined; + const duration = header.readUInt32BE(MVHD_DURATION_VERSION_0); + if (duration === UNKNOWABLE_32_BIT_DURATION) return undefined; + return movieDurationMs(header.readUInt32BE(MVHD_TIMESCALE_VERSION_0), duration); +} + +function movieDurationMs(timescale: number, duration: number): number | undefined { + // Zero is a real timeline: a clip of a screen that never changed holds one frame. + if (timescale <= 0 || duration < 0 || !Number.isSafeInteger(duration)) return undefined; + return Math.round((duration * 1000) / timescale); +} + +function readMovieHeader(filePath: string): Buffer | undefined { + const movieHeader = findMp4Atom(filePath, MVHD); + if (!movieHeader || !Number.isSafeInteger(movieHeader.size)) return undefined; + const payloadOffset = movieHeader.offset + movieHeader.headerSize; + if (movieHeader.size - movieHeader.headerSize < MVHD_FIXED_LAYOUT_BYTES) return undefined; + const buffer = Buffer.alloc(MVHD_FIXED_LAYOUT_BYTES); + let fd: number | undefined; + try { + fd = fs.openSync(filePath, 'r'); + return fs.readSync(fd, buffer, 0, buffer.length, payloadOffset) === buffer.length + ? buffer + : undefined; + } catch { + return undefined; + } finally { + if (fd !== undefined) fs.closeSync(fd); + } +} diff --git a/packages/capture-kit/src/recording/mp4.fixtures.ts b/packages/capture-kit/src/recording/mp4.fixtures.ts new file mode 100644 index 0000000000..b7fc625311 --- /dev/null +++ b/packages/capture-kit/src/recording/mp4.fixtures.ts @@ -0,0 +1,39 @@ +const ATOM_HEADER_BYTES = 8; +const EXTENDED_SIZE_BYTES = 8; +const MOVIE_HEADER_V0_PAYLOAD_BYTES = 100; +const MOVIE_HEADER_V1_PAYLOAD_BYTES = 112; + +export function mp4Atom(type: string, payload: Buffer): Buffer { + const header = Buffer.alloc(ATOM_HEADER_BYTES); + header.writeUInt32BE(ATOM_HEADER_BYTES + payload.length, 0); + header.write(type, 4, 'latin1'); + return Buffer.concat([header, payload]); +} + +export function mp4AtomWithExtendedSize(type: string, payload: Buffer): Buffer { + const header = Buffer.alloc(ATOM_HEADER_BYTES + EXTENDED_SIZE_BYTES); + header.writeUInt32BE(1, 0); + header.write(type, 4, 'latin1'); + header.writeBigUInt64BE(BigInt(header.length + payload.length), ATOM_HEADER_BYTES); + return Buffer.concat([header, payload]); +} + +export function mp4MovieHeader(params: { + version: 0 | 1; + timescale: number; + duration: number; +}): Buffer { + const versioned = params.version === 1; + const payload = Buffer.alloc( + versioned ? MOVIE_HEADER_V1_PAYLOAD_BYTES : MOVIE_HEADER_V0_PAYLOAD_BYTES, + ); + payload.writeUInt8(params.version, 0); + if (versioned) { + payload.writeUInt32BE(params.timescale, 20); + payload.writeBigUInt64BE(BigInt(params.duration), 24); + } else { + payload.writeUInt32BE(params.timescale, 12); + payload.writeUInt32BE(params.duration, 16); + } + return payload; +} diff --git a/packages/capture-kit/src/recording/video.test.ts b/packages/capture-kit/src/recording/video.test.ts index d075ec0d7a..f334608259 100644 --- a/packages/capture-kit/src/recording/video.test.ts +++ b/packages/capture-kit/src/recording/video.test.ts @@ -1,10 +1,24 @@ import fs from 'node:fs'; import path from 'node:path'; -import { expect, test } from 'vitest'; +import { beforeEach, expect, test, vi } from 'vitest'; +import { runCmd } from '@agent-device/host-kit/command'; import { likelyPlayableWebmContainer } from '../__tests__/test-utils/video-fixtures.ts'; import { mkdtempForTestSync } from '../tmp-dir.fixtures.ts'; +import { mp4Atom, mp4MovieHeader } from './mp4.fixtures.ts'; import { isPlayableVideo } from './video.ts'; +vi.mock('@agent-device/host-kit/command', () => ({ + runCmd: vi.fn(async () => ({ exitCode: 0, stdout: '', stderr: '' })), +})); +vi.mock('./swift-cache.ts', () => ({ + buildSwiftToolEnv: () => ({}), + compileSwiftSourceText: async () => '/bin/true', +})); + +beforeEach(() => { + vi.mocked(runCmd).mockClear(); +}); + const directory = mkdtempForTestSync('agent-device-video-webm-'); const playableWebm = likelyPlayableWebmContainer(); @@ -21,3 +35,21 @@ function writeFixture(name: string, bytes: Buffer): string { fs.writeFileSync(filePath, bytes); return filePath; } + +const ftyp = mp4Atom('ftyp', Buffer.from('isom', 'latin1')); +const moov = mp4Atom( + 'moov', + mp4Atom('mvhd', mp4MovieHeader({ version: 0, timescale: 1_000, duration: 2_000 })), +); + +test('reaches the semantic validator with an MP4 container whose last box runs past the file', async () => { + const truncated = Buffer.concat([ftyp, moov]).subarray(0, ftyp.length + moov.length - 4); + await expect(isPlayableVideo(writeFixture('truncated.mp4', truncated))).resolves.toBe(true); + expect(runCmd).toHaveBeenCalledTimes(1); +}); + +test('refuses an MP4 container that never declares a movie header', async () => { + const container = Buffer.concat([ftyp, mp4Atom('mdat', Buffer.alloc(8))]); + await expect(isPlayableVideo(writeFixture('no-moov.mp4', container))).resolves.toBe(false); + expect(runCmd).not.toHaveBeenCalled(); +}); diff --git a/packages/capture-kit/src/recording/video.ts b/packages/capture-kit/src/recording/video.ts index c3634b2bcd..ba75579e9a 100644 --- a/packages/capture-kit/src/recording/video.ts +++ b/packages/capture-kit/src/recording/video.ts @@ -4,6 +4,7 @@ import { runCmd } from '@agent-device/host-kit/command'; import { sleep } from '@agent-device/host-kit/retry'; import { buildSwiftToolEnv, compileSwiftSourceText } from './swift-cache.ts'; +import { findMp4Atom } from './mp4-atoms.ts'; import { hasPlayableWebmStructure } from './video-webm.ts'; // Duration zero must pass: a recording of a fully static screen legitimately contains a single @@ -154,49 +155,11 @@ function likelyPlayableVideoContainer(filePath: string): 'mp4' | 'webm' | undefi if (filePath.toLowerCase().endsWith('.webm')) { return hasPlayableWebmStructure(filePath) ? 'webm' : undefined; } - const atoms = inspectTopLevelAtoms(filePath); - return atoms.includes('ftyp') && atoms.includes('moov') ? 'mp4' : undefined; + return isMp4Container(filePath) ? 'mp4' : undefined; } -function inspectTopLevelAtoms(filePath: string): string[] { - try { - const fd = fs.openSync(filePath, 'r'); - try { - const size = fs.fstatSync(fd).size; - let offset = 0; - const atoms: string[] = []; - while (offset + 8 <= size && atoms.length < 16) { - const header = Buffer.alloc(8); - const bytesRead = fs.readSync(fd, header, 0, 8, offset); - if (bytesRead < 8) { - break; - } - - let atomSize = header.readUInt32BE(0); - const atomType = header.toString('latin1', 4, 8); - atoms.push(atomType); - - if (atomSize === 1) { - const extended = Buffer.alloc(8); - const extendedRead = fs.readSync(fd, extended, 0, 8, offset + 8); - if (extendedRead < 8) { - break; - } - atomSize = Number(extended.readBigUInt64BE(0)); - } - - // A top-level MP4 atom size of 0 extends to EOF. We stop here because there is no - // next sibling atom to inspect, and advancing by 0 would loop forever. - if (!Number.isFinite(atomSize) || atomSize <= 0) { - break; - } - offset += atomSize; - } - return atoms; - } finally { - fs.closeSync(fd); - } - } catch { - return []; - } +function isMp4Container(filePath: string): boolean { + return ( + findMp4Atom(filePath, ['ftyp']) !== undefined && findMp4Atom(filePath, ['moov']) !== undefined + ); } diff --git a/packages/contracts/src/recording.ts b/packages/contracts/src/recording.ts index 401a2e3701..92104d976f 100644 --- a/packages/contracts/src/recording.ts +++ b/packages/contracts/src/recording.ts @@ -27,6 +27,7 @@ export type RecordingStopCommandResult = { recordOnlySession?: boolean; activeSessionApp?: RecordingAppIdentity; durationMs: number; + capturedDurationMs?: number; showTouches: boolean; warning?: string; overlayWarning?: string; diff --git a/packages/contracts/src/screen-recording-runtime-host.ts b/packages/contracts/src/screen-recording-runtime-host.ts index 5d89d2b3c3..0d8d6a22ca 100644 --- a/packages/contracts/src/screen-recording-runtime-host.ts +++ b/packages/contracts/src/screen-recording-runtime-host.ts @@ -178,6 +178,13 @@ export type AndroidScreenRecordingTransport = Readonly<{ ): Promise; exists(remotePath: string, signal?: AbortSignal): Promise; size(remotePath: string, signal?: AbortSignal): Promise; + /** + * Milliseconds the device has been running, counting any time it spent suspended. A clip's media + * timeline is measured from the moment its recorder started, so a recording window has to be + * measured against the same elapsed-device-time clock rather than a host wall clock, which drifts + * against it. Reads may fail; `undefined` costs the caller its duration claim, not the recording. + */ + elapsedUptimeMs(signal?: AbortSignal): Promise; probeRunningWriters( remotePath: string, signal?: AbortSignal, diff --git a/packages/contracts/src/screen-recording-runtime.ts b/packages/contracts/src/screen-recording-runtime.ts index 3b151c0245..f692d867b3 100644 --- a/packages/contracts/src/screen-recording-runtime.ts +++ b/packages/contracts/src/screen-recording-runtime.ts @@ -84,6 +84,13 @@ export type ScreenRecordingCompletion = Readonly<{ clientOutPath?: string; startedAt: number; completedAt: number; + /** + * Duration the finished video timelines actually cover, when the backend can measure them. A + * backend that encodes only on screen changes can capture less video than the `startedAt` to + * `completedAt` window reports as the recording duration; a chunked recording sums its chunk + * timelines, which excludes whatever a chunk handover cost. + */ + capturedDurationMs?: number; scope: RecordingScope; showTouches: boolean; recordOnlySession: boolean; diff --git a/packages/platform-android/src/recording/captured-window.test.ts b/packages/platform-android/src/recording/captured-window.test.ts new file mode 100644 index 0000000000..c15e60c2b1 --- /dev/null +++ b/packages/platform-android/src/recording/captured-window.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, test, vi } from 'vitest'; +import { readMp4DurationMs } from '@agent-device/capture-kit/recording-mp4-duration'; +import { measureCapturedWindow } from './captured-window.ts'; + +vi.mock('@agent-device/capture-kit/recording-mp4-duration', () => ({ readMp4DurationMs: vi.fn() })); + +const measured = vi.mocked(readMp4DurationMs); + +function capture(durations: readonly (number | undefined)[], windowMs: number | undefined) { + let index = 0; + measured.mockImplementation(() => durations[index++]); + return measureCapturedWindow({ + chunkPaths: durations.map((_, offset) => + offset === 0 ? '/tmp/capture.mp4' : `/tmp/capture.part-${offset + 1}.mp4`, + ), + windowMs, + }); +} + +describe('measureCapturedWindow', () => { + test('names the clip length and the window length it fell short of', () => { + expect(capture([7_000], 16_000)).toEqual({ + capturedDurationMs: 7_000, + idleTailWarning: + 'Android screenrecord encodes a frame only when the screen changes, so this video ends at ' + + 'the last frame it encoded: it covers 7.0s of the 16.0s recording window.', + }); + }); + + test('warns for a window the recorder never drew on', () => { + expect(capture([0], 13_000)).toEqual({ + capturedDurationMs: 0, + idleTailWarning: + 'Android screenrecord encodes a frame only when the screen changes, so this video ends at ' + + 'the last frame it encoded: it covers 0.0s of the 13.0s recording window.', + }); + }); + + test('reports the clip length without a warning when the tail is export latency', () => { + expect(capture([6_998], 7_000)).toEqual({ capturedDurationMs: 6_998 }); + }); + + test('reports the clip length without a window measured on the recorder clock', () => { + expect(capture([7_000], undefined)).toEqual({ capturedDurationMs: 7_000 }); + }); + + test('sums the video of a chunked capture', () => { + expect(capture([170_000, 9_000], 180_500)).toEqual({ capturedDurationMs: 179_000 }); + }); + + test('leaves a capture longer than its window unexplained rather than inventing a tail', () => { + expect(capture([180_000], 60_000)).toEqual({ capturedDurationMs: 180_000 }); + }); + + test('stays silent when a chunk cannot answer with a duration', () => { + expect(capture([7_000, undefined], 16_000)).toEqual({}); + }); +}); diff --git a/packages/platform-android/src/recording/captured-window.ts b/packages/platform-android/src/recording/captured-window.ts new file mode 100644 index 0000000000..25aed5d8f6 --- /dev/null +++ b/packages/platform-android/src/recording/captured-window.ts @@ -0,0 +1,46 @@ +import { readMp4DurationMs } from '@agent-device/capture-kit/recording-mp4-duration'; + +/** Below this, a clip that ends before `record stop` is normal request-and-export latency. */ +const IDLE_TAIL_WARNING_MS = 2_000; + +/** + * Measures how much video really reached the pulled chunks. Android's `screenrecord` encodes a + * frame only when the screen changes, so a clip ends at the frame it encoded last rather than at + * `record stop`, and the caller needs to know which of the two lengths they are holding. + * + * `windowMs` has to be elapsed device time, the span a clip's media timeline is measured from; a + * host wall-clock window drifts against it and invents a tail that never happened. An unreadable + * chunk or window costs the caller the measurement, never the recording. + */ +export function measureCapturedWindow(params: { + chunkPaths: readonly string[]; + windowMs: number | undefined; +}): Readonly<{ capturedDurationMs?: number; idleTailWarning?: string }> { + const capturedDurationMs = sumCapturedDurationMs(params.chunkPaths); + if (capturedDurationMs === undefined) return {}; + const windowMs = params.windowMs; + if (windowMs === undefined) return { capturedDurationMs }; + const idleTailMs = windowMs - capturedDurationMs; + if (idleTailMs < IDLE_TAIL_WARNING_MS) return { capturedDurationMs }; + return { + capturedDurationMs, + idleTailWarning: + 'Android screenrecord encodes a frame only when the screen changes, so this video ends at ' + + 'the last frame it encoded: it covers ' + + `${formatSeconds(capturedDurationMs)}s of the ${formatSeconds(windowMs)}s recording window.`, + }; +} + +function sumCapturedDurationMs(chunkPaths: readonly string[]): number | undefined { + let total = 0; + for (const chunkPath of chunkPaths) { + const durationMs = readMp4DurationMs(chunkPath); + if (durationMs === undefined) return undefined; + total += durationMs; + } + return total; +} + +function formatSeconds(ms: number): string { + return (ms / 1000).toFixed(1); +} diff --git a/packages/platform-android/src/recording/completion.test.ts b/packages/platform-android/src/recording/completion.test.ts new file mode 100644 index 0000000000..3569610d9e --- /dev/null +++ b/packages/platform-android/src/recording/completion.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, test, vi } from 'vitest'; +import { readMp4DurationMs } from '@agent-device/capture-kit/recording-mp4-duration'; +import { completed, snapshot } from './completion.ts'; +import { recordingHost, recordingInput } from './fixtures.ts'; + +vi.mock('@agent-device/capture-kit/recording-mp4-duration', () => ({ readMp4DurationMs: vi.fn() })); + +async function capture(params: { + clipMs: number; + windowMs: number | undefined; + chunks?: number; + reachedLimit?: boolean; + finalization?: Record; +}) { + vi.mocked(readMp4DurationMs).mockReturnValue(params.clipMs); + return await completed({ + host: recordingHost({ finalize: { complete: async () => params.finalization ?? {} } }), + recording: snapshot(recordingInput(), 1), + chunks: Array.from({ length: params.chunks ?? 1 }, (_, offset) => ({ + index: offset + 1, + path: offset === 0 ? '/tmp/capture.mp4' : `/tmp/capture.part-${offset + 1}.mp4`, + })), + targetLabel: 'Android recording', + reachedLimit: params.reachedLimit ?? false, + windowMs: params.windowMs, + }); +} + +describe('completed', () => { + test('reports the clip the recorder really captured beside the requested window', async () => { + const outcome = await capture({ + clipMs: 7_000, + windowMs: 16_000, + finalization: { telemetryPath: '/tmp/capture.gesture-telemetry.json' }, + }); + expect(outcome.result).toMatchObject({ + telemetryPath: '/tmp/capture.gesture-telemetry.json', + capturedDurationMs: 7_000, + warning: + 'Android screenrecord encodes a frame only when the screen changes, so this video ends at ' + + 'the last frame it encoded: it covers 7.0s of the 16.0s recording window.', + }); + }); + + test('appends every recording warning behind the finalizer warning', async () => { + const outcome = await capture({ + clipMs: 180_000, + windowMs: 400_000, + chunks: 2, + reachedLimit: true, + finalization: { warning: 'recording was exported without touch overlays' }, + }); + const warning = outcome.result.warning ?? ''; + expect(warning).toMatch( + /^recording was exported without touch overlays Android adb screenrecord stopped before record stop/, + ); + expect(warning.indexOf('is capped at 180s')).toBeLessThan( + warning.indexOf('encodes a frame only'), + ); + expect(outcome.result.capturedDurationMs).toBe(360_000); + }); +}); diff --git a/packages/platform-android/src/recording/completion.ts b/packages/platform-android/src/recording/completion.ts index 83952dd8be..a18b6b3c3c 100644 --- a/packages/platform-android/src/recording/completion.ts +++ b/packages/platform-android/src/recording/completion.ts @@ -6,6 +6,16 @@ import type { ScreenRecordingLiveSnapshot, ScreenRecordingStartInput, } from '@agent-device/contracts/screen-recording-runtime'; +import { measureCapturedWindow } from './captured-window.ts'; + +const PLATFORM_LIMIT_WARNING = + 'Android adb screenrecord stopped before record stop, likely after reaching the 180s platform ' + + 'limit. The MP4 may be truncated; final interactions after the limit are not in the video.'; +const CHUNKED_WARNING = + 'Android adb screenrecord is capped at 180s, so this recording was split into multiple MP4 chunks.'; +const CHUNKED_OVERLAY_WARNING = + 'touch overlay burn-in is skipped for chunked Android recordings; returning raw chunks plus ' + + 'gesture telemetry'; export function snapshot( input: ScreenRecordingStartInput, @@ -25,13 +35,15 @@ export function snapshot( }); } -export async function completed( - host: PlatformRuntimeHost, - recording: ScreenRecordingLiveSnapshot, - chunks: readonly ScreenRecordingChunk[], - targetLabel: string, - reachedLimit = false, -): Promise> { +export async function completed(params: { + host: PlatformRuntimeHost; + recording: ScreenRecordingLiveSnapshot; + chunks: readonly ScreenRecordingChunk[]; + targetLabel: string; + reachedLimit: boolean; + windowMs: number | undefined; +}): Promise> { + const { host, recording, chunks, targetLabel, reachedLimit, windowMs } = params; const chunked = chunks.length > 1; const finalization = await host.screenRecording.finalize.complete({ outputPath: recording.outPath, @@ -40,18 +52,17 @@ export async function completed( exportQuality: recording.exportQuality ?? 'medium', targetLabel, }); + const captured = measureCapturedWindow({ + chunkPaths: chunks.map((chunk) => chunk.path), + windowMs, + }); const warnings = [ - ...(reachedLimit - ? [ - 'Android adb screenrecord stopped before record stop, likely after reaching the 180s platform limit. The MP4 may be truncated; final interactions after the limit are not in the video.', - ] - : []), - ...(chunked - ? [ - 'Android adb screenrecord is capped at 180s, so this recording was split into multiple MP4 chunks.', - ] - : []), + ...(finalization.warning ? [finalization.warning] : []), + ...(reachedLimit ? [PLATFORM_LIMIT_WARNING] : []), + ...(chunked ? [CHUNKED_WARNING] : []), + ...(captured.idleTailWarning ? [captured.idleTailWarning] : []), ]; + const completedAt = Date.now(); return { status: 'completed', result: { @@ -59,19 +70,19 @@ export async function completed( outPath: recording.outPath, ...(recording.clientOutPath ? { clientOutPath: recording.clientOutPath } : {}), startedAt: recording.startedAt, - completedAt: Date.now(), + completedAt, scope: recording.scope, showTouches: recording.showTouches, recordOnlySession: recording.recordOnlySession, ...(recording.activeSessionApp ? { activeSessionApp: recording.activeSessionApp } : {}), ...(chunked ? { chunks } : {}), - ...(warnings.length ? { warning: warnings.join(' ') } : {}), ...finalization, + ...(captured.capturedDurationMs === undefined + ? {} + : { capturedDurationMs: captured.capturedDurationMs }), + ...(warnings.length ? { warning: warnings.join(' ') } : {}), ...(chunked && recording.showTouches && recording.gestureEvents.length > 0 - ? { - overlayWarning: - 'touch overlay burn-in is skipped for chunked Android recordings; returning raw chunks plus gesture telemetry', - } + ? { overlayWarning: CHUNKED_OVERLAY_WARNING } : {}), }, }; diff --git a/packages/platform-android/src/recording/device-clock.test.ts b/packages/platform-android/src/recording/device-clock.test.ts new file mode 100644 index 0000000000..0a24cc6893 --- /dev/null +++ b/packages/platform-android/src/recording/device-clock.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, test } from 'vitest'; +import { recordingHost, androidRecordingDevice } from './fixtures.ts'; +import { readElapsedUptimeMs, recordingWindowMs } from './device-clock.ts'; + +describe('readElapsedUptimeMs', () => { + test('reads the recorder clock from the device', async () => { + const host = recordingHost({ elapsedUptimeMs: async () => 12_345 }); + const transport = await host.screenRecording.android.resolve(androidRecordingDevice); + await expect(readElapsedUptimeMs(transport)).resolves.toBe(12_345); + }); + + test('gives up on a device that cannot answer', async () => { + const host = recordingHost({ + elapsedUptimeMs: async () => { + throw new Error('adb gone'); + }, + }); + const transport = await host.screenRecording.android.resolve(androidRecordingDevice); + await expect(readElapsedUptimeMs(transport)).resolves.toBeUndefined(); + }); +}); + +describe('recordingWindowMs', () => { + test('measures recorder time between two reads of the device clock', () => { + expect(recordingWindowMs({ stoppedUptimeMs: 20_000, startedUptimeMs: 12_345 })).toBe(7_655); + }); + + test('never reads a negative window out of a clock that went backwards', () => { + expect(recordingWindowMs({ stoppedUptimeMs: 10_000, startedUptimeMs: 12_345 })).toBe(0); + }); + + test('stays unanswered when either read failed', () => { + expect( + recordingWindowMs({ stoppedUptimeMs: undefined, startedUptimeMs: 12_345 }), + ).toBeUndefined(); + expect( + recordingWindowMs({ stoppedUptimeMs: 20_000, startedUptimeMs: undefined }), + ).toBeUndefined(); + }); +}); diff --git a/packages/platform-android/src/recording/device-clock.ts b/packages/platform-android/src/recording/device-clock.ts new file mode 100644 index 0000000000..b1ff55623a --- /dev/null +++ b/packages/platform-android/src/recording/device-clock.ts @@ -0,0 +1,28 @@ +import type { AndroidScreenRecordingTransport } from '@agent-device/contracts/screen-recording-runtime-host'; + +type RecorderClock = Pick; + +/** + * Reads how long the device has been running: the elapsed recorder time a clip's media timeline is + * measured from. A failed read costs the caller its duration claim, never the recording. + */ +export async function readElapsedUptimeMs( + recorderClock: RecorderClock, + signal?: AbortSignal, +): Promise { + try { + return await recorderClock.elapsedUptimeMs(signal); + } catch { + return undefined; + } +} + +/** Recorder time between two uptime reads, or `undefined` when either read failed. */ +export function recordingWindowMs(params: { + stoppedUptimeMs: number | undefined; + startedUptimeMs: number | undefined; +}): number | undefined { + const { stoppedUptimeMs, startedUptimeMs } = params; + if (stoppedUptimeMs === undefined || startedUptimeMs === undefined) return undefined; + return Math.max(0, stoppedUptimeMs - startedUptimeMs); +} diff --git a/packages/platform-android/src/recording/finalize.test.ts b/packages/platform-android/src/recording/finalize.test.ts index eb58845441..dd9064d7ef 100644 --- a/packages/platform-android/src/recording/finalize.test.ts +++ b/packages/platform-android/src/recording/finalize.test.ts @@ -1,4 +1,8 @@ +import fs from 'node:fs'; +import path from 'node:path'; import { expect, test } from 'vitest'; +import { mp4Atom, mp4MovieHeader } from '@agent-device/capture-kit/recording-mp4-fixtures'; +import { mkdtempForTestSync } from '../__tests__/test-utils/tmp-dir.ts'; import { finalizeAndroidRecording } from './finalize.ts'; import { androidRecordingDevice, recordingHost, recordingInput } from './fixtures.ts'; import { createNativeManifest } from './manifest.ts'; @@ -43,3 +47,61 @@ test('writes terminal coordinates before removing a fenced Android artifact', as ).resolves.toMatchObject({ status: 'completed' }); expect(calls).toEqual(['completed', 'remove:/sdcard/agent-device-recording-1.mp4']); }); + +test('measures the window on the recorder clock it reads before stopping the recorder', async () => { + const directory = mkdtempForTestSync('agent-device-android-finalize-'); + const calls: string[] = []; + const host = recordingHost({ + elapsedUptimeMs: async () => { + calls.push('uptime'); + return 26_000; + }, + stop: async () => { + calls.push('stop'); + return 'stopped' as const; + }, + isRunning: async () => false, + pullPlayable: async ({ outputPath }: { outputPath: string }) => { + calls.push('pull'); + fs.writeFileSync( + outputPath, + Buffer.concat([ + mp4Atom('mdat', Buffer.alloc(8)), + mp4Atom( + 'moov', + mp4Atom('mvhd', mp4MovieHeader({ version: 0, timescale: 1_000, duration: 7_000 })), + ), + ]), + ); + return { stdout: '', stderr: '', exitCode: 0, playable: true }; + }, + }); + const input = { ...recordingInput(), outputPath: path.join(directory, 'capture.mp4') }; + const transport = await host.screenRecording.android.resolve(androidRecordingDevice); + const outcome = await finalizeAndroidRecording({ + host, + transport, + evidence: createNativeManifest( + androidRecordingDevice, + input, + 1, + [ + { + index: 1, + remotePath: '/sdcard/agent-device-recording-1.mp4', + remotePid: '41', + remoteStartTime: '7', + }, + ], + undefined, + 'local', + ), + manifestPath: '/sdcard/agent-device-recording-active.json', + recording: snapshot(input, 1), + startedUptimeMs: 10_000, + }); + + expect(calls.slice(0, 2)).toEqual(['uptime', 'stop']); + expect(outcome.result.capturedDurationMs).toBe(7_000); + expect(outcome.result.warning).toContain('it covers 7.0s of the 16.0s recording window.'); +}); diff --git a/packages/platform-android/src/recording/finalize.ts b/packages/platform-android/src/recording/finalize.ts index bdc51af5ca..bb43f4e2fa 100644 --- a/packages/platform-android/src/recording/finalize.ts +++ b/packages/platform-android/src/recording/finalize.ts @@ -1,6 +1,7 @@ import type { PlatformRuntimeHost } from '@agent-device/contracts/platform-runtime-operations'; import type { ScreenRecordingLiveSnapshot } from '@agent-device/contracts/screen-recording-runtime'; import { cleanupChunks, pullChunks, stopOwnedChunks, waitForStableArtifacts } from './chunks.ts'; +import { readElapsedUptimeMs, recordingWindowMs } from './device-clock.ts'; import { completed } from './completion.ts'; import { createCompletedNativeManifest, type NativeManifest } from './manifest.ts'; import { persistNativeManifest } from './manifest-store.ts'; @@ -14,6 +15,8 @@ export async function finalizeAndroidRecording(params: { evidence: NativeManifest; manifestPath: string; recording: ScreenRecordingLiveSnapshot; + /** Recorder clock at launch. Without it the clip length is still measured, just never compared. */ + startedUptimeMs?: number; reachedLimit?: boolean; }): Promise< Readonly<{ @@ -21,6 +24,9 @@ export async function finalizeAndroidRecording(params: { result: import('@agent-device/contracts/screen-recording-runtime').ScreenRecordingCompletion; }> > { + // Read the recorder's clock before the signal below: everything after that signal is this + // tool's own export latency rather than time the screen sat unchanged. + const stoppedUptimeMs = await readElapsedUptimeMs(params.transport); const reachedLimit = (await stopOwnedChunks(params.transport, params.evidence.chunks)) || params.reachedLimit === true; @@ -31,13 +37,17 @@ export async function finalizeAndroidRecording(params: { params.recording.outPath, params.recording.clientOutPath, ); - const outcome = await completed( - params.host, - params.recording, - outputChunks, - 'Android recording', + const outcome = await completed({ + host: params.host, + recording: params.recording, + chunks: outputChunks, + targetLabel: 'Android recording', reachedLimit, - ); + windowMs: recordingWindowMs({ + stoppedUptimeMs, + startedUptimeMs: params.startedUptimeMs, + }), + }); await persistNativeManifest( params.transport, params.manifestPath, diff --git a/packages/platform-android/src/recording/fixtures.ts b/packages/platform-android/src/recording/fixtures.ts index 6ea8de1834..ebcfb2e84b 100644 --- a/packages/platform-android/src/recording/fixtures.ts +++ b/packages/platform-android/src/recording/fixtures.ts @@ -36,6 +36,8 @@ export function recordingHost(overrides: Record): PlatformRunti }, exists: async (remotePath: string) => (legacy.exists ? await legacy.exists(remotePath) : true), size: async (remotePath: string) => (legacy.size ? await legacy.size(remotePath) : 1), + elapsedUptimeMs: async () => + legacy.elapsedUptimeMs ? await legacy.elapsedUptimeMs() : undefined, inspect: async (processIdentity: { pid: string }) => legacy.inspect ? await legacy.inspect(processIdentity) diff --git a/packages/platform-android/src/recording/launch.ts b/packages/platform-android/src/recording/launch.ts index bfab3a1a34..0b330ad128 100644 --- a/packages/platform-android/src/recording/launch.ts +++ b/packages/platform-android/src/recording/launch.ts @@ -7,6 +7,7 @@ import { rollbackChunks, startChunkAt, } from './chunks.ts'; +import { readElapsedUptimeMs } from './device-clock.ts'; import { createNativeManifest, type NativeChunk } from './manifest.ts'; import { persistNativeManifest, removeNativeManifest } from './manifest-store.ts'; import { reconcileStartEvidence } from './start-reconciliation.ts'; @@ -20,7 +21,9 @@ export async function startInitialTransaction(params: { startedAt: number; signal: AbortSignal; prepareOutput: () => Promise; -}): Promise> { +}): Promise< + Readonly<{ chunk: NativeChunk; manifestPath: string; startedUptimeMs: number | undefined }> +> { const { transport, device, input, startedAt, signal, prepareOutput } = params; await reconcileStartEvidence(transport, device); await prepareOutput(); @@ -34,7 +37,11 @@ export async function startInitialTransaction(params: { signal, ); let chunk: NativeChunk; + let startedUptimeMs: number | undefined; try { + // Read the recorder clock before launching so the window covers the recording's own startup, + // the way the stop-side read covers the stop signal. + startedUptimeMs = await readElapsedUptimeMs(transport, signal); chunk = await startChunkAt(transport, remotePath, input, signal); } catch (error) { if (signal.aborted) { @@ -56,7 +63,7 @@ export async function startInitialTransaction(params: { await rollbackPublishedChunk(transport, manifestPath, chunk); throw error; } - return { chunk, manifestPath }; + return { chunk, manifestPath, startedUptimeMs }; } throw last ?? new Error('Android screenrecord did not begin producing frames'); } diff --git a/packages/platform-android/src/recording/manifest-validation.test.ts b/packages/platform-android/src/recording/manifest-validation.test.ts index 2513f80e1d..783a3af4fd 100644 --- a/packages/platform-android/src/recording/manifest-validation.test.ts +++ b/packages/platform-android/src/recording/manifest-validation.test.ts @@ -62,3 +62,41 @@ test('rejects terminal evidence whose result coordinates diverge from its manife }), ).toBe(false); }); + +test('accepts a measured clip length in terminal evidence and refuses an uncountable one', () => { + const input = recordingInput(); + const active = createNativeManifest( + androidRecordingDevice, + input, + 1, + [ + { + index: 1, + remotePath: '/sdcard/agent-device-recording-1.mp4', + remotePid: '41', + remoteStartTime: '7', + }, + ], + undefined, + 'local', + ); + const complete = createCompletedNativeManifest(active, { + backend: 'adb screenrecord', + outPath: input.outputPath, + startedAt: 1, + completedAt: 2, + capturedDurationMs: 7_000, + scope: input.scope, + showTouches: input.showTouches, + recordOnlySession: input.recordOnlySession, + }); + expect(isValidNativeManifest(complete)).toBe(true); + for (const capturedDurationMs of ['7000', Number.NaN]) { + expect( + isValidNativeManifest({ + ...complete, + completion: { ...complete.completion!, capturedDurationMs }, + }), + ).toBe(false); + } +}); diff --git a/packages/platform-android/src/recording/manifest-validation.ts b/packages/platform-android/src/recording/manifest-validation.ts index 9942f1d776..df33ce44b8 100644 --- a/packages/platform-android/src/recording/manifest-validation.ts +++ b/packages/platform-android/src/recording/manifest-validation.ts @@ -134,7 +134,8 @@ function completionIdentityIsValid(candidate: Partial typeof candidate.outPath === 'string' && (candidate.clientOutPath === undefined || typeof candidate.clientOutPath === 'string') && Number.isFinite(candidate.startedAt) && - Number.isFinite(candidate.completedAt) + Number.isFinite(candidate.completedAt) && + (candidate.capturedDurationMs === undefined || Number.isFinite(candidate.capturedDurationMs)) ); } diff --git a/packages/platform-android/src/recording/runtime.ts b/packages/platform-android/src/recording/runtime.ts index 25dc8e21af..6fee324924 100644 --- a/packages/platform-android/src/recording/runtime.ts +++ b/packages/platform-android/src/recording/runtime.ts @@ -160,6 +160,7 @@ async function startAndroidRecording(params: { evidence: createNativeManifest(device, input, startedAt, chunks, undefined, transport.mode), manifestPath, recording: current, + startedUptimeMs: initial.startedUptimeMs, }); nativeCleanupConfirmed = true; return outcome; diff --git a/src/commands/recording/index.ts b/src/commands/recording/index.ts index 40b1ecea3c..4d092ded73 100644 --- a/src/commands/recording/index.ts +++ b/src/commands/recording/index.ts @@ -104,7 +104,7 @@ export const recordCommandFacet = defineCommandFacet({ text: { summary: 'Start or stop screen recording', cliDetail: - 'The default --scope app requires an active app session from open ; use --scope device/system to explicitly request whole-screen recording where the selected backend supports it. Android record start publishes a durable device manifest, recordings longer than the 180s adb screenrecord limit are returned as multiple MP4 chunks while the daemon stays alive, and daemon-restart recovery uses only manifest-owned chunks. An Android manifest left by an unreachable recording is retired on the next start once its recorders are proven gone; one still owned refuses with non-retriable DEVICE_IN_USE and reason native_recovery_evidence_open naming the session to run record stop for, and a recorder that is still writing an artifact refuses with DEVICE_IN_USE and reason native_recording_artifact_claimed, which clears itself once that recorder ends at the 180s limit. HarmonyOS supports whole-screen recording on physical devices only: use --scope device/system; --fps, --quality, and --hide-touches are unsupported. Use --quality to choose medium or high export quality on supported backends. An iOS simulator host recording lock returns non-retriable DEVICE_IN_USE with reason apple_simulator_recording_busy. Stop the recording in its owning session; if a dead recorder left the host locked, ask the host operator to restart the CoreSimulator stream service.', + 'The default --scope app requires an active app session from open ; use --scope device/system to explicitly request whole-screen recording where the selected backend supports it. Android record start publishes a durable device manifest, recordings longer than the 180s adb screenrecord limit are returned as multiple MP4 chunks while the daemon stays alive, and daemon-restart recovery uses only manifest-owned chunks. Android screenrecord encodes a frame only when the screen changes, so a clip can end at the last frame the recorder encoded instead of at record stop; durationMs is host wall clock from record start until the export finished, and capturedDurationMs reports the video timeline when it can be measured, with a warning naming how much of the window that video covers. An Android manifest left by an unreachable recording is retired on the next start once its recorders are proven gone; one still owned refuses with non-retriable DEVICE_IN_USE and reason native_recovery_evidence_open naming the session to run record stop for, and a recorder that is still writing an artifact refuses with DEVICE_IN_USE and reason native_recording_artifact_claimed, which clears itself once that recorder ends at the 180s limit. HarmonyOS supports whole-screen recording on physical devices only: use --scope device/system; --fps, --quality, and --hide-touches are unsupported. Use --quality to choose medium or high export quality on supported backends. An iOS simulator host recording lock returns non-retriable DEVICE_IN_USE with reason apple_simulator_recording_busy. Stop the recording in its owning session; if a dead recorder left the host locked, ask the host operator to restart the CoreSimulator stream service.', }, metadata: recordCommandMetadata, run: (client, input) => client.recording.record(input as RecordOptions), diff --git a/src/daemon/__tests__/screen-recording-stop-recovery.test.ts b/src/daemon/__tests__/screen-recording-stop-recovery.test.ts index 191f20d05a..2bced757c5 100644 --- a/src/daemon/__tests__/screen-recording-stop-recovery.test.ts +++ b/src/daemon/__tests__/screen-recording-stop-recovery.test.ts @@ -56,6 +56,7 @@ test('a completed manifest whose stored response is damaged serves nothing', asy { outPath: 0 }, { clientOutPath: '' }, { completedAt: 'later' }, + { capturedDurationMs: 'soon' }, { chunks: [{ index: 'first', path: '/daemon/capture-0.mp4' }] }, { activeSessionApp: { bundleId: '' } }, ]; @@ -125,6 +126,7 @@ function fullCompletion(outPath: string): ScreenRecordingCompletion { outPath, startedAt: 1, completedAt: 4, + capturedDurationMs: 3, scope: 'app', showTouches: true, recordOnlySession: false, diff --git a/src/daemon/handlers/__tests__/record-runtime-response.test.ts b/src/daemon/handlers/__tests__/record-runtime-response.test.ts index 07b2c4f630..829f573f14 100644 --- a/src/daemon/handlers/__tests__/record-runtime-response.test.ts +++ b/src/daemon/handlers/__tests__/record-runtime-response.test.ts @@ -65,3 +65,21 @@ test('stop response derives client telemetry and chunk artifact paths', () => { }), ); }); + +test('stop response separates the recording window from the clip that was captured', () => { + const completion = { + backend: 'adb screenrecord', + outPath: '/daemon/capture.mp4', + startedAt: 1_000, + completedAt: 17_000, + scope: 'device', + showTouches: false, + recordOnlySession: false, + } as const; + const measured = buildRecordingStopResponse({ ...completion, capturedDurationMs: 7_000 }); + const unmeasured = buildRecordingStopResponse(completion); + if (!measured.ok || !unmeasured.ok) throw new Error('expected a successful recording stop'); + + expect(measured.data).toMatchObject({ durationMs: 16_000, capturedDurationMs: 7_000 }); + expect(unmeasured.data).not.toHaveProperty('capturedDurationMs'); +}); diff --git a/src/daemon/handlers/record-runtime-response.ts b/src/daemon/handlers/record-runtime-response.ts index cfb52a580e..74d59e8458 100644 --- a/src/daemon/handlers/record-runtime-response.ts +++ b/src/daemon/handlers/record-runtime-response.ts @@ -89,6 +89,9 @@ export function buildRecordingStopResponse(completion: ScreenRecordingCompletion recordOnlySession: completion.recordOnlySession, activeSessionApp: completion.activeSessionApp, durationMs: Math.max(0, completion.completedAt - completion.startedAt), + ...(completion.capturedDurationMs === undefined + ? {} + : { capturedDurationMs: completion.capturedDurationMs }), showTouches: completion.showTouches, warning: completion.warning, overlayWarning: completion.overlayWarning, diff --git a/src/daemon/screen-recording-session-resource.ts b/src/daemon/screen-recording-session-resource.ts index b57a3ae2f9..f3f6f14319 100644 --- a/src/daemon/screen-recording-session-resource.ts +++ b/src/daemon/screen-recording-session-resource.ts @@ -79,6 +79,9 @@ export function encodeScreenRecordingCompletionMetadata( outPath: completion.outPath, startedAt: completion.startedAt, completedAt: completion.completedAt, + ...(completion.capturedDurationMs === undefined + ? {} + : { capturedDurationMs: completion.capturedDurationMs }), scope: completion.scope, showTouches: completion.showTouches, recordOnlySession: completion.recordOnlySession, diff --git a/src/daemon/screen-recording-stop-recovery.ts b/src/daemon/screen-recording-stop-recovery.ts index 5457522ee5..3ac3220cfe 100644 --- a/src/daemon/screen-recording-stop-recovery.ts +++ b/src/daemon/screen-recording-stop-recovery.ts @@ -123,6 +123,7 @@ function isServedCompletion(stored: Record): boolean { isNonEmptyString(stored.backend) && isFiniteNumber(stored.startedAt) && isFiniteNumber(stored.completedAt) && + (stored.capturedDurationMs === undefined || isFiniteNumber(stored.capturedDurationMs)) && isRecordingScope(stored.scope) && typeof stored.showTouches === 'boolean' && typeof stored.recordOnlySession === 'boolean' diff --git a/src/mcp/command-output-schemas.ts b/src/mcp/command-output-schemas.ts index 20e83f6b49..a5db763308 100644 --- a/src/mcp/command-output-schemas.ts +++ b/src/mcp/command-output-schemas.ts @@ -843,6 +843,7 @@ const BASE_COMMAND_OUTPUT_SCHEMAS = { recordOnlySession: booleanSchema(), activeSessionApp: looseObjectSchema(), durationMs: numberSchema(), + capturedDurationMs: numberSchema(), showTouches: booleanSchema(), warning: stringSchema(), overlayWarning: stringSchema(), diff --git a/src/platform-runtime-screen-recording-android-host.test.ts b/src/platform-runtime-screen-recording-android-host.test.ts index 403c79abbd..f1b28b5816 100644 --- a/src/platform-runtime-screen-recording-android-host.test.ts +++ b/src/platform-runtime-screen-recording-android-host.test.ts @@ -383,3 +383,49 @@ test('retains unavailable manifest reads and confirms manifest deletion', async ); expect(commands).toHaveLength(2); }); + +test('reads elapsed device uptime as the clock that timestamps recorded frames', async () => { + const budgets: Record = {}; + const signal = new AbortController().signal; + await withAndroidAdbProvider( + { + exec: async (args: string[], options?: { timeoutMs?: number; signal?: AbortSignal }) => { + const command = args[1] ?? ''; + budgets[command] = options?.timeoutMs; + if (command === 'cat /proc/uptime') { + expect(options?.signal).toBe(signal); + return result('1234.56 3456.78\n'); + } + if (command.startsWith('test -e')) return result(); + return command.startsWith('stat -c') ? result('42\n') : result('', 'gone', 1); + }, + }, + { serial: android.id }, + async () => { + const transport = await createAndroidScreenRecordingTransport(android); + await expect(transport.elapsedUptimeMs(signal)).resolves.toBe(1_234_560); + await expect(transport.size('/sdcard/capture.mp4')).resolves.toBe(42); + }, + ); + // The optional clock read precedes recording work, so it may not spend the window that work needs. + expect(budgets['cat /proc/uptime']).toBeLessThan( + budgets["stat -c %s '/sdcard/capture.mp4'"] as number, + ); +}); + +test('leaves the recorder clock unanswered when uptime is unreadable or unparseable', async () => { + for (const reading of [ + result('', 'No such file or directory', 1), + result('not-a-number 0.00\n'), + result('\n'), + ]) { + await withAndroidAdbProvider( + { exec: async () => reading }, + { serial: android.id }, + async () => { + const transport = await createAndroidScreenRecordingTransport(android); + await expect(transport.elapsedUptimeMs()).resolves.toBeUndefined(); + }, + ); + } +}); diff --git a/src/platform-runtime-screen-recording-android-host.ts b/src/platform-runtime-screen-recording-android-host.ts index 6ae7eb061f..7b84b8b83b 100644 --- a/src/platform-runtime-screen-recording-android-host.ts +++ b/src/platform-runtime-screen-recording-android-host.ts @@ -11,6 +11,9 @@ import { loadAndroidMechanics } from './platform-runtime-android-mechanics.ts'; const ANDROID_MANIFEST_NAME = 'agent-device-recording-active.json'; const ADB_TIMEOUT_MS = 5_000; +// Budget for the optional elapsed-uptime read that precedes starting and stopping a recording: +// short enough that it cannot eat the request window of the recording it describes. +const UPTIME_PROBE_TIMEOUT_MS = 1_500; const BIT_RATE = { medium: 8_000_000, high: 20_000_000 } as const; export async function createAndroidScreenRecordingTransport( @@ -20,10 +23,10 @@ export async function createAndroidScreenRecordingTransport( await loadAndroidMechanics(); const adb = resolveAndroidAdbExecutor(device); const scoped = resolveScopedAndroidAdbBackgroundTransport(device); - const shell = async (command: string, signal?: AbortSignal) => + const shell = async (command: string, signal?: AbortSignal, timeoutMs = ADB_TIMEOUT_MS) => await adb(['shell', command], { allowFailure: true, - timeoutMs: ADB_TIMEOUT_MS, + timeoutMs, signal, }); return Object.freeze({ @@ -67,6 +70,13 @@ export async function createAndroidScreenRecordingTransport( const size = Number(result.stdout.trim()); return Number.isSafeInteger(size) && size >= 0 ? size : 'uncertain'; }, + elapsedUptimeMs: async (signal) => { + const result = await shell('cat /proc/uptime', signal, UPTIME_PROBE_TIMEOUT_MS); + if (result.exitCode !== 0) return undefined; + const [seconds] = result.stdout.trim().split(/\s+/); + if (!seconds || !/^\d+(?:\.\d+)?$/.test(seconds)) return undefined; + return Math.round(Number(seconds) * 1000); + }, probeRunningWriters: async (remotePath, signal) => { const result = await shell('ps -A -o pid=', signal); if (result.exitCode !== 0) return { writers: [], conclusive: false }; diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index fc89e79e54..ccbc56aeda 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -965,6 +965,7 @@ agent-device record stop # Stop active recording - On Linux or other non-macOS hosts, `record stop` still succeeds and returns the raw video plus telemetry sidecar, and includes `overlayWarning` when burn-in overlays were skipped. - On iOS simulators, a busy CoreSimulator host recording slot makes `record start` return non-retriable `DEVICE_IN_USE` with `details.reason: apple_simulator_recording_busy`. Use `record stop` in the session that owns the active recording. If a previous recorder died and no recording is active, ask the host operator to restart the CoreSimulator stream service before retrying. - Android uses `adb shell screenrecord`, which has a 180s platform limit. `record start` publishes a durable device manifest. Longer recordings are split into MP4 chunks while the daemon stays alive; after daemon restart, `record stop` recovers only manifest-owned chunks and warns when gesture overlay telemetry was lost. +- Android `screenrecord` encodes a frame only when the screen changes, so a clip ends at the last frame the recorder encoded instead of at `record stop`: a window that ends on an unchanged screen yields a shorter video, while every on-screen change inside the window stays at its real offset in it. `record stop` reports `durationMs` as host wall clock from `record start` until the export finished, and when the video can be measured it also reports `capturedDurationMs` and warns with how much of the window that video covers. - `record stop` is safe to repeat. When its request window ends while the daemon is still exporting — typical for a long touch-overlay burn-in on a remote daemon — the export keeps running there, and a second `record stop` in the same session returns that completed recording, including the caller-side output path, without starting another recording. A finished recording whose video file is already gone reports `no active recording`. **Session app logs (token-efficient debugging):** Logging is off by default in normal flows. Enable it on demand for debugging. Logs are written to a file so agents can grep instead of loading full output into context. From 343bbe7459afae656488c1f2c95d7e9dfea0e957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 13 Sep 2026 19:58:19 +0200 Subject: [PATCH 2/4] chore(gates): enumerate the capture-kit MP4 subpaths the layering scan holds `@agent-device/capture-kit/recording-mp4-duration` and its fixture sibling are new declared package subpaths, so the boundary enumeration that holds every exported workspace subpath has to name them for the layering scan to accept the Android recorder's read of a pulled clip's timeline. --- scripts/layering/package-boundaries.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index e048965066..9e2955290e 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -409,6 +409,8 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/capture-kit/post-gesture-stability', '@agent-device/capture-kit/quality-warnings', '@agent-device/capture-kit/react-native-overlay', + '@agent-device/capture-kit/recording-mp4-duration', + '@agent-device/capture-kit/recording-mp4-fixtures', '@agent-device/capture-kit/recording-output-path', '@agent-device/capture-kit/recording-overlay', '@agent-device/capture-kit/recording-telemetry', From 625dc00bab63cddade61403241e3977fdf86f313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 13 Sep 2026 20:44:57 +0200 Subject: [PATCH 3/4] fix(capture-kit): evaluate the MP4 box scan only when a file is validated The Coverage job's ADR-0019 eager-closure probe failed: `recording/video.ts` evaluated 25 modules on import where the merge-base evaluated 24, because the MP4 container gate statically imported the box walk it now shares with the clip-duration read, and `recording/overlay.ts` grew by the same module. An entry the merge-base already carries gets no growth budget, so the edge moves behind a function-scoped `await import`: the scan is something recording completion asks for, and importing this module for `waitForStableFile` or WebM detection should not evaluate a box walker. The alternative the probe offered -- hosting the walker in a module both growing entries already evaluate -- would have put an ISO-BMFF walk in `swift-cache.ts` or `video-webm.ts`, or made the duration read import the Swift validator machinery that sits behind `video.ts`. --- packages/capture-kit/src/recording/video.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/capture-kit/src/recording/video.ts b/packages/capture-kit/src/recording/video.ts index ba75579e9a..81ff9297c0 100644 --- a/packages/capture-kit/src/recording/video.ts +++ b/packages/capture-kit/src/recording/video.ts @@ -4,7 +4,6 @@ import { runCmd } from '@agent-device/host-kit/command'; import { sleep } from '@agent-device/host-kit/retry'; import { buildSwiftToolEnv, compileSwiftSourceText } from './swift-cache.ts'; -import { findMp4Atom } from './mp4-atoms.ts'; import { hasPlayableWebmStructure } from './video-webm.ts'; // Duration zero must pass: a recording of a fully static screen legitimately contains a single @@ -69,7 +68,7 @@ export async function waitForStableFile( } export async function isPlayableVideo(filePath: string): Promise { - const container = likelyPlayableVideoContainer(filePath); + const container = await likelyPlayableVideoContainer(filePath); if (!container) return false; // AVFoundation is the MP4 semantic validator. It does not reliably load WebM on supported // macOS hosts, so WebM completion is established by its EBML document type + Segment marker. @@ -142,7 +141,7 @@ function isSwiftVideoValidatorUnavailable(stderr: string, stdout: string): boole ); } -function likelyPlayableVideoContainer(filePath: string): 'mp4' | 'webm' | undefined { +async function likelyPlayableVideoContainer(filePath: string): Promise<'mp4' | 'webm' | undefined> { try { const stats = fs.statSync(filePath); if (!stats.isFile() || stats.size <= 0) { @@ -155,10 +154,13 @@ function likelyPlayableVideoContainer(filePath: string): 'mp4' | 'webm' | undefi if (filePath.toLowerCase().endsWith('.webm')) { return hasPlayableWebmStructure(filePath) ? 'webm' : undefined; } - return isMp4Container(filePath) ? 'mp4' : undefined; + return (await isMp4Container(filePath)) ? 'mp4' : undefined; } -function isMp4Container(filePath: string): boolean { +// Loaded on the first validated file rather than on import: this scan is what a recording +// completion asks for, and nothing that merely imports this module should evaluate it. +async function isMp4Container(filePath: string): Promise { + const { findMp4Atom } = await import('./mp4-atoms.ts'); return ( findMp4Atom(filePath, ['ftyp']) !== undefined && findMp4Atom(filePath, ['moov']) !== undefined ); From d4c0ad2adb0328997fd4a62708560d959f161c02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 14 Sep 2026 12:25:15 +0200 Subject: [PATCH 4/4] refactor(android): bracket the recording window with the host clock Human review of #2566: the device-clock read defended against host-vs-encoder drift that does not matter at this threshold. Quartz drifts by tens of ppm, so a 30-minute chunked recording moves the window under 100 ms against a 2s warning threshold, while the read cost a transport operation, its own probe budget, and two adb round trips per recording. The window is now the host elapsed time between `Date.now()` immediately before the recorder launches and `Date.now()` immediately before the stop signal, so the contract change and the extra device I/O are gone, and the one case where the clocks genuinely diverge -- a host that sleeps mid-recording -- reports a shorter window and misses the warning rather than inventing one. The surviving clock arithmetic is one subtraction, so it lives in the window module that already owns that concern instead of a module of its own. A stop recovered through daemon recovery now passes the manifest's own start instant, which is the first host timestamp the recording ever had, so a recovered stop gets the same comparison a live stop gets. --- .../src/screen-recording-runtime-host.ts | 7 --- .../src/recording/captured-window.test.ts | 14 +++--- .../src/recording/captured-window.ts | 20 ++++---- .../src/recording/completion.test.ts | 7 ++- .../src/recording/completion.ts | 8 ++-- .../src/recording/device-clock.test.ts | 40 ---------------- .../src/recording/device-clock.ts | 28 ----------- .../src/recording/finalize.test.ts | 13 ++---- .../src/recording/finalize.ts | 17 +++---- .../src/recording/fixtures.ts | 2 - .../platform-android/src/recording/launch.ts | 15 +++--- .../src/recording/recovery.ts | 1 + .../platform-android/src/recording/runtime.ts | 2 +- ...time-screen-recording-android-host.test.ts | 46 ------------------- ...m-runtime-screen-recording-android-host.ts | 14 +----- 15 files changed, 51 insertions(+), 183 deletions(-) delete mode 100644 packages/platform-android/src/recording/device-clock.test.ts delete mode 100644 packages/platform-android/src/recording/device-clock.ts diff --git a/packages/contracts/src/screen-recording-runtime-host.ts b/packages/contracts/src/screen-recording-runtime-host.ts index 0d8d6a22ca..5d89d2b3c3 100644 --- a/packages/contracts/src/screen-recording-runtime-host.ts +++ b/packages/contracts/src/screen-recording-runtime-host.ts @@ -178,13 +178,6 @@ export type AndroidScreenRecordingTransport = Readonly<{ ): Promise; exists(remotePath: string, signal?: AbortSignal): Promise; size(remotePath: string, signal?: AbortSignal): Promise; - /** - * Milliseconds the device has been running, counting any time it spent suspended. A clip's media - * timeline is measured from the moment its recorder started, so a recording window has to be - * measured against the same elapsed-device-time clock rather than a host wall clock, which drifts - * against it. Reads may fail; `undefined` costs the caller its duration claim, not the recording. - */ - elapsedUptimeMs(signal?: AbortSignal): Promise; probeRunningWriters( remotePath: string, signal?: AbortSignal, diff --git a/packages/platform-android/src/recording/captured-window.test.ts b/packages/platform-android/src/recording/captured-window.test.ts index c15e60c2b1..f73d0c81c1 100644 --- a/packages/platform-android/src/recording/captured-window.test.ts +++ b/packages/platform-android/src/recording/captured-window.test.ts @@ -5,15 +5,17 @@ import { measureCapturedWindow } from './captured-window.ts'; vi.mock('@agent-device/capture-kit/recording-mp4-duration', () => ({ readMp4DurationMs: vi.fn() })); const measured = vi.mocked(readMp4DurationMs); +const STARTED_AT_MS = 1_789_000_000_000; -function capture(durations: readonly (number | undefined)[], windowMs: number | undefined) { +function capture(durations: readonly (number | undefined)[], windowMs: number) { let index = 0; measured.mockImplementation(() => durations[index++]); return measureCapturedWindow({ chunkPaths: durations.map((_, offset) => offset === 0 ? '/tmp/capture.mp4' : `/tmp/capture.part-${offset + 1}.mp4`, ), - windowMs, + startedAtMs: STARTED_AT_MS, + stoppedAtMs: STARTED_AT_MS + windowMs, }); } @@ -40,10 +42,6 @@ describe('measureCapturedWindow', () => { expect(capture([6_998], 7_000)).toEqual({ capturedDurationMs: 6_998 }); }); - test('reports the clip length without a window measured on the recorder clock', () => { - expect(capture([7_000], undefined)).toEqual({ capturedDurationMs: 7_000 }); - }); - test('sums the video of a chunked capture', () => { expect(capture([170_000, 9_000], 180_500)).toEqual({ capturedDurationMs: 179_000 }); }); @@ -52,6 +50,10 @@ describe('measureCapturedWindow', () => { expect(capture([180_000], 60_000)).toEqual({ capturedDurationMs: 180_000 }); }); + test('reports the clip length when the host clock moved the window backwards', () => { + expect(capture([7_000], -1_000)).toEqual({ capturedDurationMs: 7_000 }); + }); + test('stays silent when a chunk cannot answer with a duration', () => { expect(capture([7_000, undefined], 16_000)).toEqual({}); }); diff --git a/packages/platform-android/src/recording/captured-window.ts b/packages/platform-android/src/recording/captured-window.ts index 25aed5d8f6..651ebd6b71 100644 --- a/packages/platform-android/src/recording/captured-window.ts +++ b/packages/platform-android/src/recording/captured-window.ts @@ -4,22 +4,24 @@ import { readMp4DurationMs } from '@agent-device/capture-kit/recording-mp4-durat const IDLE_TAIL_WARNING_MS = 2_000; /** - * Measures how much video really reached the pulled chunks. Android's `screenrecord` encodes a - * frame only when the screen changes, so a clip ends at the frame it encoded last rather than at - * `record stop`, and the caller needs to know which of the two lengths they are holding. + * Measures how much video really reached the pulled chunks and how much of the recording window it + * covers. Android's `screenrecord` encodes a frame only when the screen changes, so a clip ends at + * the frame it encoded last rather than at `record stop`, and the caller needs to know which of the + * two lengths they are holding. * - * `windowMs` has to be elapsed device time, the span a clip's media timeline is measured from; a - * host wall-clock window drifts against it and invents a tail that never happened. An unreadable - * chunk or window costs the caller the measurement, never the recording. + * The window is host elapsed time between launching the recorder and sending the stop signal, the + * span this tool itself bracketed; its own export latency belongs to neither. A host that slept + * mid-recording reports a window shorter than the device saw, which costs the warning rather than + * inventing one. An unreadable chunk costs the measurement, never the recording. */ export function measureCapturedWindow(params: { chunkPaths: readonly string[]; - windowMs: number | undefined; + startedAtMs: number; + stoppedAtMs: number; }): Readonly<{ capturedDurationMs?: number; idleTailWarning?: string }> { const capturedDurationMs = sumCapturedDurationMs(params.chunkPaths); if (capturedDurationMs === undefined) return {}; - const windowMs = params.windowMs; - if (windowMs === undefined) return { capturedDurationMs }; + const windowMs = params.stoppedAtMs - params.startedAtMs; const idleTailMs = windowMs - capturedDurationMs; if (idleTailMs < IDLE_TAIL_WARNING_MS) return { capturedDurationMs }; return { diff --git a/packages/platform-android/src/recording/completion.test.ts b/packages/platform-android/src/recording/completion.test.ts index 3569610d9e..5f7b89cef4 100644 --- a/packages/platform-android/src/recording/completion.test.ts +++ b/packages/platform-android/src/recording/completion.test.ts @@ -5,9 +5,11 @@ import { recordingHost, recordingInput } from './fixtures.ts'; vi.mock('@agent-device/capture-kit/recording-mp4-duration', () => ({ readMp4DurationMs: vi.fn() })); +const STARTED_AT_MS = 1_789_000_000_000; + async function capture(params: { clipMs: number; - windowMs: number | undefined; + windowMs: number; chunks?: number; reachedLimit?: boolean; finalization?: Record; @@ -22,7 +24,8 @@ async function capture(params: { })), targetLabel: 'Android recording', reachedLimit: params.reachedLimit ?? false, - windowMs: params.windowMs, + startedAtMs: STARTED_AT_MS, + stoppedAtMs: STARTED_AT_MS + params.windowMs, }); } diff --git a/packages/platform-android/src/recording/completion.ts b/packages/platform-android/src/recording/completion.ts index a18b6b3c3c..6cad4d07f9 100644 --- a/packages/platform-android/src/recording/completion.ts +++ b/packages/platform-android/src/recording/completion.ts @@ -41,9 +41,10 @@ export async function completed(params: { chunks: readonly ScreenRecordingChunk[]; targetLabel: string; reachedLimit: boolean; - windowMs: number | undefined; + startedAtMs: number; + stoppedAtMs: number; }): Promise> { - const { host, recording, chunks, targetLabel, reachedLimit, windowMs } = params; + const { host, recording, chunks, targetLabel, reachedLimit, startedAtMs, stoppedAtMs } = params; const chunked = chunks.length > 1; const finalization = await host.screenRecording.finalize.complete({ outputPath: recording.outPath, @@ -54,7 +55,8 @@ export async function completed(params: { }); const captured = measureCapturedWindow({ chunkPaths: chunks.map((chunk) => chunk.path), - windowMs, + startedAtMs, + stoppedAtMs, }); const warnings = [ ...(finalization.warning ? [finalization.warning] : []), diff --git a/packages/platform-android/src/recording/device-clock.test.ts b/packages/platform-android/src/recording/device-clock.test.ts deleted file mode 100644 index 0a24cc6893..0000000000 --- a/packages/platform-android/src/recording/device-clock.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { describe, expect, test } from 'vitest'; -import { recordingHost, androidRecordingDevice } from './fixtures.ts'; -import { readElapsedUptimeMs, recordingWindowMs } from './device-clock.ts'; - -describe('readElapsedUptimeMs', () => { - test('reads the recorder clock from the device', async () => { - const host = recordingHost({ elapsedUptimeMs: async () => 12_345 }); - const transport = await host.screenRecording.android.resolve(androidRecordingDevice); - await expect(readElapsedUptimeMs(transport)).resolves.toBe(12_345); - }); - - test('gives up on a device that cannot answer', async () => { - const host = recordingHost({ - elapsedUptimeMs: async () => { - throw new Error('adb gone'); - }, - }); - const transport = await host.screenRecording.android.resolve(androidRecordingDevice); - await expect(readElapsedUptimeMs(transport)).resolves.toBeUndefined(); - }); -}); - -describe('recordingWindowMs', () => { - test('measures recorder time between two reads of the device clock', () => { - expect(recordingWindowMs({ stoppedUptimeMs: 20_000, startedUptimeMs: 12_345 })).toBe(7_655); - }); - - test('never reads a negative window out of a clock that went backwards', () => { - expect(recordingWindowMs({ stoppedUptimeMs: 10_000, startedUptimeMs: 12_345 })).toBe(0); - }); - - test('stays unanswered when either read failed', () => { - expect( - recordingWindowMs({ stoppedUptimeMs: undefined, startedUptimeMs: 12_345 }), - ).toBeUndefined(); - expect( - recordingWindowMs({ stoppedUptimeMs: 20_000, startedUptimeMs: undefined }), - ).toBeUndefined(); - }); -}); diff --git a/packages/platform-android/src/recording/device-clock.ts b/packages/platform-android/src/recording/device-clock.ts deleted file mode 100644 index b1ff55623a..0000000000 --- a/packages/platform-android/src/recording/device-clock.ts +++ /dev/null @@ -1,28 +0,0 @@ -import type { AndroidScreenRecordingTransport } from '@agent-device/contracts/screen-recording-runtime-host'; - -type RecorderClock = Pick; - -/** - * Reads how long the device has been running: the elapsed recorder time a clip's media timeline is - * measured from. A failed read costs the caller its duration claim, never the recording. - */ -export async function readElapsedUptimeMs( - recorderClock: RecorderClock, - signal?: AbortSignal, -): Promise { - try { - return await recorderClock.elapsedUptimeMs(signal); - } catch { - return undefined; - } -} - -/** Recorder time between two uptime reads, or `undefined` when either read failed. */ -export function recordingWindowMs(params: { - stoppedUptimeMs: number | undefined; - startedUptimeMs: number | undefined; -}): number | undefined { - const { stoppedUptimeMs, startedUptimeMs } = params; - if (stoppedUptimeMs === undefined || startedUptimeMs === undefined) return undefined; - return Math.max(0, stoppedUptimeMs - startedUptimeMs); -} diff --git a/packages/platform-android/src/recording/finalize.test.ts b/packages/platform-android/src/recording/finalize.test.ts index dd9064d7ef..1c6646f693 100644 --- a/packages/platform-android/src/recording/finalize.test.ts +++ b/packages/platform-android/src/recording/finalize.test.ts @@ -43,19 +43,16 @@ test('writes terminal coordinates before removing a fenced Android artifact', as evidence, manifestPath: '/sdcard/agent-device-recording-active.json', recording: snapshot(input, 1), + startedAtMs: 1, }), ).resolves.toMatchObject({ status: 'completed' }); expect(calls).toEqual(['completed', 'remove:/sdcard/agent-device-recording-1.mp4']); }); -test('measures the window on the recorder clock it reads before stopping the recorder', async () => { +test('measures a pulled MP4 against the window the host bracketed around the recorder', async () => { const directory = mkdtempForTestSync('agent-device-android-finalize-'); const calls: string[] = []; const host = recordingHost({ - elapsedUptimeMs: async () => { - calls.push('uptime'); - return 26_000; - }, stop: async () => { calls.push('stop'); return 'stopped' as const; @@ -98,10 +95,10 @@ test('measures the window on the recorder clock it reads before stopping the rec ), manifestPath: '/sdcard/agent-device-recording-active.json', recording: snapshot(input, 1), - startedUptimeMs: 10_000, + startedAtMs: Date.now() - 16_000, }); - expect(calls.slice(0, 2)).toEqual(['uptime', 'stop']); + expect(calls).toEqual(['stop', 'pull']); expect(outcome.result.capturedDurationMs).toBe(7_000); - expect(outcome.result.warning).toContain('it covers 7.0s of the 16.0s recording window.'); + expect(outcome.result.warning).toMatch(/it covers 7\.0s of the 16\.\ds recording window\./); }); diff --git a/packages/platform-android/src/recording/finalize.ts b/packages/platform-android/src/recording/finalize.ts index bb43f4e2fa..b38b563203 100644 --- a/packages/platform-android/src/recording/finalize.ts +++ b/packages/platform-android/src/recording/finalize.ts @@ -1,7 +1,6 @@ import type { PlatformRuntimeHost } from '@agent-device/contracts/platform-runtime-operations'; import type { ScreenRecordingLiveSnapshot } from '@agent-device/contracts/screen-recording-runtime'; import { cleanupChunks, pullChunks, stopOwnedChunks, waitForStableArtifacts } from './chunks.ts'; -import { readElapsedUptimeMs, recordingWindowMs } from './device-clock.ts'; import { completed } from './completion.ts'; import { createCompletedNativeManifest, type NativeManifest } from './manifest.ts'; import { persistNativeManifest } from './manifest-store.ts'; @@ -15,8 +14,8 @@ export async function finalizeAndroidRecording(params: { evidence: NativeManifest; manifestPath: string; recording: ScreenRecordingLiveSnapshot; - /** Recorder clock at launch. Without it the clip length is still measured, just never compared. */ - startedUptimeMs?: number; + /** Host instant the recorder was launched, which is where a clip's timeline begins. */ + startedAtMs: number; reachedLimit?: boolean; }): Promise< Readonly<{ @@ -24,9 +23,9 @@ export async function finalizeAndroidRecording(params: { result: import('@agent-device/contracts/screen-recording-runtime').ScreenRecordingCompletion; }> > { - // Read the recorder's clock before the signal below: everything after that signal is this - // tool's own export latency rather than time the screen sat unchanged. - const stoppedUptimeMs = await readElapsedUptimeMs(params.transport); + // Read the clock before the signal below: everything after that signal is this tool's own export + // latency rather than time the screen sat unchanged. + const stoppedAtMs = Date.now(); const reachedLimit = (await stopOwnedChunks(params.transport, params.evidence.chunks)) || params.reachedLimit === true; @@ -43,10 +42,8 @@ export async function finalizeAndroidRecording(params: { chunks: outputChunks, targetLabel: 'Android recording', reachedLimit, - windowMs: recordingWindowMs({ - stoppedUptimeMs, - startedUptimeMs: params.startedUptimeMs, - }), + startedAtMs: params.startedAtMs, + stoppedAtMs, }); await persistNativeManifest( params.transport, diff --git a/packages/platform-android/src/recording/fixtures.ts b/packages/platform-android/src/recording/fixtures.ts index ebcfb2e84b..6ea8de1834 100644 --- a/packages/platform-android/src/recording/fixtures.ts +++ b/packages/platform-android/src/recording/fixtures.ts @@ -36,8 +36,6 @@ export function recordingHost(overrides: Record): PlatformRunti }, exists: async (remotePath: string) => (legacy.exists ? await legacy.exists(remotePath) : true), size: async (remotePath: string) => (legacy.size ? await legacy.size(remotePath) : 1), - elapsedUptimeMs: async () => - legacy.elapsedUptimeMs ? await legacy.elapsedUptimeMs() : undefined, inspect: async (processIdentity: { pid: string }) => legacy.inspect ? await legacy.inspect(processIdentity) diff --git a/packages/platform-android/src/recording/launch.ts b/packages/platform-android/src/recording/launch.ts index 0b330ad128..d0e6001f75 100644 --- a/packages/platform-android/src/recording/launch.ts +++ b/packages/platform-android/src/recording/launch.ts @@ -7,7 +7,6 @@ import { rollbackChunks, startChunkAt, } from './chunks.ts'; -import { readElapsedUptimeMs } from './device-clock.ts'; import { createNativeManifest, type NativeChunk } from './manifest.ts'; import { persistNativeManifest, removeNativeManifest } from './manifest-store.ts'; import { reconcileStartEvidence } from './start-reconciliation.ts'; @@ -21,9 +20,7 @@ export async function startInitialTransaction(params: { startedAt: number; signal: AbortSignal; prepareOutput: () => Promise; -}): Promise< - Readonly<{ chunk: NativeChunk; manifestPath: string; startedUptimeMs: number | undefined }> -> { +}): Promise> { const { transport, device, input, startedAt, signal, prepareOutput } = params; await reconcileStartEvidence(transport, device); await prepareOutput(); @@ -37,11 +34,11 @@ export async function startInitialTransaction(params: { signal, ); let chunk: NativeChunk; - let startedUptimeMs: number | undefined; + let startedAtMs: number; try { - // Read the recorder clock before launching so the window covers the recording's own startup, - // the way the stop-side read covers the stop signal. - startedUptimeMs = await readElapsedUptimeMs(transport, signal); + // Timestamp immediately before launching, the way the stop signal is timestamped, so the + // window a short clip is measured against holds only the recording. + startedAtMs = Date.now(); chunk = await startChunkAt(transport, remotePath, input, signal); } catch (error) { if (signal.aborted) { @@ -63,7 +60,7 @@ export async function startInitialTransaction(params: { await rollbackPublishedChunk(transport, manifestPath, chunk); throw error; } - return { chunk, manifestPath, startedUptimeMs }; + return { chunk, manifestPath, startedAtMs }; } throw last ?? new Error('Android screenrecord did not begin producing frames'); } diff --git a/packages/platform-android/src/recording/recovery.ts b/packages/platform-android/src/recording/recovery.ts index 063a15b4c9..3b517fedf4 100644 --- a/packages/platform-android/src/recording/recovery.ts +++ b/packages/platform-android/src/recording/recovery.ts @@ -176,6 +176,7 @@ async function reattachEvidence(params: { evidence, manifestPath: descriptor.manifestPath, recording: current, + startedAtMs: evidence.startedAt, reachedLimit: provesAndroidScreenRecordTermination(running), }); nativeCleanupConfirmed = true; diff --git a/packages/platform-android/src/recording/runtime.ts b/packages/platform-android/src/recording/runtime.ts index 6fee324924..f6099ddcb8 100644 --- a/packages/platform-android/src/recording/runtime.ts +++ b/packages/platform-android/src/recording/runtime.ts @@ -160,7 +160,7 @@ async function startAndroidRecording(params: { evidence: createNativeManifest(device, input, startedAt, chunks, undefined, transport.mode), manifestPath, recording: current, - startedUptimeMs: initial.startedUptimeMs, + startedAtMs: initial.startedAtMs, }); nativeCleanupConfirmed = true; return outcome; diff --git a/src/platform-runtime-screen-recording-android-host.test.ts b/src/platform-runtime-screen-recording-android-host.test.ts index f1b28b5816..403c79abbd 100644 --- a/src/platform-runtime-screen-recording-android-host.test.ts +++ b/src/platform-runtime-screen-recording-android-host.test.ts @@ -383,49 +383,3 @@ test('retains unavailable manifest reads and confirms manifest deletion', async ); expect(commands).toHaveLength(2); }); - -test('reads elapsed device uptime as the clock that timestamps recorded frames', async () => { - const budgets: Record = {}; - const signal = new AbortController().signal; - await withAndroidAdbProvider( - { - exec: async (args: string[], options?: { timeoutMs?: number; signal?: AbortSignal }) => { - const command = args[1] ?? ''; - budgets[command] = options?.timeoutMs; - if (command === 'cat /proc/uptime') { - expect(options?.signal).toBe(signal); - return result('1234.56 3456.78\n'); - } - if (command.startsWith('test -e')) return result(); - return command.startsWith('stat -c') ? result('42\n') : result('', 'gone', 1); - }, - }, - { serial: android.id }, - async () => { - const transport = await createAndroidScreenRecordingTransport(android); - await expect(transport.elapsedUptimeMs(signal)).resolves.toBe(1_234_560); - await expect(transport.size('/sdcard/capture.mp4')).resolves.toBe(42); - }, - ); - // The optional clock read precedes recording work, so it may not spend the window that work needs. - expect(budgets['cat /proc/uptime']).toBeLessThan( - budgets["stat -c %s '/sdcard/capture.mp4'"] as number, - ); -}); - -test('leaves the recorder clock unanswered when uptime is unreadable or unparseable', async () => { - for (const reading of [ - result('', 'No such file or directory', 1), - result('not-a-number 0.00\n'), - result('\n'), - ]) { - await withAndroidAdbProvider( - { exec: async () => reading }, - { serial: android.id }, - async () => { - const transport = await createAndroidScreenRecordingTransport(android); - await expect(transport.elapsedUptimeMs()).resolves.toBeUndefined(); - }, - ); - } -}); diff --git a/src/platform-runtime-screen-recording-android-host.ts b/src/platform-runtime-screen-recording-android-host.ts index 7b84b8b83b..6ae7eb061f 100644 --- a/src/platform-runtime-screen-recording-android-host.ts +++ b/src/platform-runtime-screen-recording-android-host.ts @@ -11,9 +11,6 @@ import { loadAndroidMechanics } from './platform-runtime-android-mechanics.ts'; const ANDROID_MANIFEST_NAME = 'agent-device-recording-active.json'; const ADB_TIMEOUT_MS = 5_000; -// Budget for the optional elapsed-uptime read that precedes starting and stopping a recording: -// short enough that it cannot eat the request window of the recording it describes. -const UPTIME_PROBE_TIMEOUT_MS = 1_500; const BIT_RATE = { medium: 8_000_000, high: 20_000_000 } as const; export async function createAndroidScreenRecordingTransport( @@ -23,10 +20,10 @@ export async function createAndroidScreenRecordingTransport( await loadAndroidMechanics(); const adb = resolveAndroidAdbExecutor(device); const scoped = resolveScopedAndroidAdbBackgroundTransport(device); - const shell = async (command: string, signal?: AbortSignal, timeoutMs = ADB_TIMEOUT_MS) => + const shell = async (command: string, signal?: AbortSignal) => await adb(['shell', command], { allowFailure: true, - timeoutMs, + timeoutMs: ADB_TIMEOUT_MS, signal, }); return Object.freeze({ @@ -70,13 +67,6 @@ export async function createAndroidScreenRecordingTransport( const size = Number(result.stdout.trim()); return Number.isSafeInteger(size) && size >= 0 ? size : 'uncertain'; }, - elapsedUptimeMs: async (signal) => { - const result = await shell('cat /proc/uptime', signal, UPTIME_PROBE_TIMEOUT_MS); - if (result.exitCode !== 0) return undefined; - const [seconds] = result.stdout.trim().split(/\s+/); - if (!seconds || !/^\d+(?:\.\d+)?$/.test(seconds)) return undefined; - return Math.round(Number(seconds) * 1000); - }, probeRunningWriters: async (remotePath, signal) => { const result = await shell('ps -A -o pid=', signal); if (result.exitCode !== 0) return { writers: [], conclusive: false };