diff --git a/packages/capture-kit/package.json b/packages/capture-kit/package.json index 4a997111cf..76acc5357d 100644 --- a/packages/capture-kit/package.json +++ b/packages/capture-kit/package.json @@ -82,6 +82,14 @@ "types": "./src/react-native-overlay.ts", "default": "./src/react-native-overlay.ts" }, + "./recording-mp4-fixtures": { + "types": "./src/recording/mp4.fixtures.ts", + "default": "./src/recording/mp4.fixtures.ts" + }, + "./recording-mp4-duration": { + "types": "./src/recording/mp4-duration.ts", + "default": "./src/recording/mp4-duration.ts" + }, "./recording-output-path": { "types": "./src/recording/output-path.ts", "default": "./src/recording/output-path.ts" diff --git a/packages/capture-kit/src/recording/mp4-atoms.test.ts b/packages/capture-kit/src/recording/mp4-atoms.test.ts new file mode 100644 index 0000000000..580722ef48 --- /dev/null +++ b/packages/capture-kit/src/recording/mp4-atoms.test.ts @@ -0,0 +1,79 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { describe, expect, test } from 'vitest'; +import { mkdtempForTestSync } from '../tmp-dir.fixtures.ts'; +import { findMp4Atom } from './mp4-atoms.ts'; +import { mp4Atom, mp4AtomWithExtendedSize } from './mp4.fixtures.ts'; + +const directory = mkdtempForTestSync('agent-device-mp4-atoms-'); + +function write(name: string, contents: Buffer): string { + const filePath = path.join(directory, name); + fs.writeFileSync(filePath, contents); + return filePath; +} + +describe('findMp4Atom', () => { + test('locates a nested box and reports where its payload starts', () => { + const movieHeader = mp4Atom('mvhd', Buffer.alloc(100)); + const file = write( + 'nested.mp4', + Buffer.concat([mp4Atom('ftyp', Buffer.alloc(8)), mp4Atom('moov', movieHeader)]), + ); + const atom = findMp4Atom(file, ['moov', 'mvhd']); + expect(atom?.type).toBe('mvhd'); + expect(atom?.offset).toBe(24); + expect(atom?.size).toBe(108); + expect(findMp4Atom(file, ['moov', 'udta'])).toBeUndefined(); + }); + + test('walks past a 64-bit sized box', () => { + const file = write( + 'extended.mp4', + Buffer.concat([ + mp4AtomWithExtendedSize('mdat', Buffer.alloc(64)), + mp4Atom('moov', Buffer.alloc(16)), + ]), + ); + expect(findMp4Atom(file, ['moov'])?.offset).toBe(16 + 64); + }); + + test('accepts a box whose declared size runs to end of file', () => { + const streamingMdat = Buffer.concat([ + Buffer.from([0, 0, 0, 0]), + Buffer.from('mdat', 'latin1'), + Buffer.alloc(32), + ]); + const file = write( + 'streaming.mp4', + Buffer.concat([mp4Atom('ftyp', Buffer.alloc(8)), streamingMdat]), + ); + const atom = findMp4Atom(file, ['mdat']); + expect(atom?.offset).toBe(16); + expect(atom?.size).toBe(streamingMdat.length); + }); + + test('reports a box that overruns its container and stops looking past it', () => { + const overrunning = Buffer.concat([ + Buffer.from([0, 0, 0, 200]), + Buffer.from('wide', 'latin1'), + Buffer.alloc(40), + ]); + const file = write( + 'overrun.mp4', + Buffer.concat([ + mp4Atom('ftyp', Buffer.alloc(8)), + overrunning, + mp4Atom('moov', Buffer.alloc(8)), + ]), + ); + expect(findMp4Atom(file, ['wide'])?.size).toBe(200); + expect(findMp4Atom(file, ['moov'])).toBeUndefined(); + }); + + test('answers undefined for an absent file, an unreadable path, and a short box', () => { + expect(findMp4Atom(path.join(directory, 'absent.mp4'), ['moov'])).toBeUndefined(); + expect(findMp4Atom(directory, ['moov'])).toBeUndefined(); + expect(findMp4Atom(write('short.mp4', Buffer.from([0, 0])), ['moov'])).toBeUndefined(); + }); +}); diff --git a/packages/capture-kit/src/recording/mp4-atoms.ts b/packages/capture-kit/src/recording/mp4-atoms.ts new file mode 100644 index 0000000000..09b89623fc --- /dev/null +++ b/packages/capture-kit/src/recording/mp4-atoms.ts @@ -0,0 +1,99 @@ +import fs from 'node:fs'; + +// Bounded so a corrupt container cannot widen the walk; a movie header past the budget simply +// reads as absent. +const MAX_SIBLINGS_PER_LEVEL = 32; +const ATOM_HEADER_BYTES = 8; +const EXTENDED_SIZE_BYTES = 8; + +export type Mp4Atom = Readonly<{ + /** Four-character box type, for example `moov`. */ + type: string; + /** Byte offset where the box header starts. */ + offset: number; + headerSize: number; + /** Total box size in bytes, including the header. */ + size: number; +}>; + +/** Locates a box by its ancestor-to-leaf type path, for example `['moov', 'mvhd']`. */ +export function findMp4Atom(filePath: string, names: readonly string[]): Mp4Atom | undefined { + if (names.length === 0) return undefined; + let fd: number | undefined; + try { + fd = fs.openSync(filePath, 'r'); + const root: Mp4Atom = { type: '', offset: 0, headerSize: 0, size: fs.fstatSync(fd).size }; + return findDescendant(fd, root, names, 0); + } catch { + return undefined; + } finally { + if (fd !== undefined) fs.closeSync(fd); + } +} + +function findDescendant( + fd: number, + container: Mp4Atom, + names: readonly string[], + depth: number, +): Mp4Atom | undefined { + for (const atom of childAtoms(fd, container)) { + if (atom.type !== names[depth]) continue; + if (depth + 1 === names.length) return atom; + const match = findDescendant(fd, atom, names, depth + 1); + if (match) return match; + } + return undefined; +} + +function* childAtoms(fd: number, container: Mp4Atom): Generator { + const end = container.offset + container.size; + let offset = container.offset + container.headerSize; + for ( + let seen = 0; + offset + ATOM_HEADER_BYTES <= end && seen < MAX_SIBLINGS_PER_LEVEL; + seen += 1 + ) { + const atom = readAtomHeader(fd, offset, end); + if (!atom) return; + yield atom; + // A box that claims more room than its container holds, or an unrepresentable 64-bit size, + // ends this level: nothing past it can be located. + if ( + !Number.isSafeInteger(atom.size) || + atom.size < atom.headerSize || + atom.offset + atom.size > end + ) + return; + offset += atom.size; + } +} + +function readAtomHeader(fd: number, offset: number, end: number): Mp4Atom | undefined { + const header = readBytes(fd, offset, ATOM_HEADER_BYTES); + if (!header) return undefined; + let size = header.readUInt32BE(0); + let headerSize = ATOM_HEADER_BYTES; + if (size === 1) { + const extended = readBytes(fd, offset + ATOM_HEADER_BYTES, EXTENDED_SIZE_BYTES); + if (!extended) return undefined; + size = Number(extended.readBigUInt64BE(0)); + headerSize += EXTENDED_SIZE_BYTES; + } + // A declared size of 0 means the box runs to the end of its container. + return { + type: header.toString('latin1', 4, 8), + offset, + headerSize, + size: size === 0 ? end - offset : size, + }; +} + +function readBytes(fd: number, offset: number, length: number): Buffer | undefined { + const buffer = Buffer.alloc(length); + try { + return fs.readSync(fd, buffer, 0, length, offset) === length ? buffer : undefined; + } catch { + return undefined; + } +} diff --git a/packages/capture-kit/src/recording/mp4-duration.test.ts b/packages/capture-kit/src/recording/mp4-duration.test.ts new file mode 100644 index 0000000000..00b60e67d4 --- /dev/null +++ b/packages/capture-kit/src/recording/mp4-duration.test.ts @@ -0,0 +1,72 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { describe, expect, test } from 'vitest'; +import { mkdtempForTestSync } from '../tmp-dir.fixtures.ts'; +import { readMp4DurationMs } from './mp4-duration.ts'; +import { mp4Atom, mp4MovieHeader } from './mp4.fixtures.ts'; + +const directory = mkdtempForTestSync('agent-device-mp4-duration-'); + +function recording(name: string, movieHeaderPayload: Buffer): string { + const filePath = path.join(directory, `${name}.mp4`); + fs.writeFileSync( + filePath, + Buffer.concat([ + mp4Atom('ftyp', Buffer.alloc(24)), + mp4Atom('mdat', Buffer.alloc(16)), + mp4Atom('moov', mp4Atom('mvhd', movieHeaderPayload)), + ]), + ); + return filePath; +} + +describe('readMp4DurationMs', () => { + test('reads a version 0 movie header in its own timescale', () => { + const file = recording( + 'v0', + mp4MovieHeader({ version: 0, timescale: 90_000, duration: 630_000 }), + ); + expect(readMp4DurationMs(file)).toBe(7_000); + }); + + test('reports a timeline of zero for a clip of a screen that never changed', () => { + const file = recording('static', mp4MovieHeader({ version: 0, timescale: 1_000, duration: 0 })); + expect(readMp4DurationMs(file)).toBe(0); + }); + + test('reads the 64-bit duration of a version 1 movie header', () => { + const file = recording('v1', mp4MovieHeader({ version: 1, timescale: 1_000, duration: 9_500 })); + expect(readMp4DurationMs(file)).toBe(9_500); + }); + + test('rounds a duration that does not divide evenly into milliseconds', () => { + const file = recording('odd', mp4MovieHeader({ version: 0, timescale: 3, duration: 2 })); + expect(readMp4DurationMs(file)).toBe(667); + }); + + test('answers undefined when the timeline cannot be trusted', () => { + expect( + readMp4DurationMs( + recording('idle', mp4MovieHeader({ version: 0, timescale: 1_000, duration: 0xffff_ffff })), + ), + ).toBeUndefined(); + expect( + readMp4DurationMs( + recording('no-timescale', mp4MovieHeader({ version: 0, timescale: 0, duration: 5_000 })), + ), + ).toBeUndefined(); + const unsupportedVersion = mp4MovieHeader({ version: 0, timescale: 1_000, duration: 5_000 }); + unsupportedVersion.writeUInt8(2, 0); + expect(readMp4DurationMs(recording('version-2', unsupportedVersion))).toBeUndefined(); + }); + + test('answers undefined for a file with no movie header or a truncated one', () => { + const withoutMovieHeader = path.join(directory, 'no-moov.mp4'); + fs.writeFileSync(withoutMovieHeader, mp4Atom('mdat', Buffer.alloc(16))); + expect(readMp4DurationMs(withoutMovieHeader)).toBeUndefined(); + + const truncated = path.join(directory, 'truncated.mp4'); + fs.writeFileSync(truncated, mp4Atom('moov', mp4Atom('mvhd', Buffer.alloc(24)))); + expect(readMp4DurationMs(truncated)).toBeUndefined(); + }); +}); diff --git a/packages/capture-kit/src/recording/mp4-duration.ts b/packages/capture-kit/src/recording/mp4-duration.ts new file mode 100644 index 0000000000..2ecd91c1f2 --- /dev/null +++ b/packages/capture-kit/src/recording/mp4-duration.ts @@ -0,0 +1,56 @@ +import fs from 'node:fs'; +import { findMp4Atom } from './mp4-atoms.ts'; + +const MVHD = ['moov', 'mvhd'] as const; +const MVHD_TIMESCALE_VERSION_0 = 12; +const MVHD_DURATION_VERSION_0 = 16; +const MVHD_TIMESCALE_VERSION_1 = 20; +const MVHD_DURATION_VERSION_1 = 24; +const MVHD_FIXED_LAYOUT_BYTES = 32; +const UNKNOWABLE_32_BIT_DURATION = 0xffffffff; + +/** + * Duration the MP4 timeline actually covers, in milliseconds, or `undefined` when the container + * cannot answer. A screen-recording timeline can be shorter than the window it was captured in: + * `screenrecord` encodes a frame only when the screen changes. + */ +export function readMp4DurationMs(filePath: string): number | undefined { + const header = readMovieHeader(filePath); + if (!header) return undefined; + const version = header[0]; + if (version === 1) { + return movieDurationMs( + header.readUInt32BE(MVHD_TIMESCALE_VERSION_1), + Number(header.readBigUInt64BE(MVHD_DURATION_VERSION_1)), + ); + } + if (version !== 0) return undefined; + const duration = header.readUInt32BE(MVHD_DURATION_VERSION_0); + if (duration === UNKNOWABLE_32_BIT_DURATION) return undefined; + return movieDurationMs(header.readUInt32BE(MVHD_TIMESCALE_VERSION_0), duration); +} + +function movieDurationMs(timescale: number, duration: number): number | undefined { + // Zero is a real timeline: a clip of a screen that never changed holds one frame. + if (timescale <= 0 || duration < 0 || !Number.isSafeInteger(duration)) return undefined; + return Math.round((duration * 1000) / timescale); +} + +function readMovieHeader(filePath: string): Buffer | undefined { + const movieHeader = findMp4Atom(filePath, MVHD); + if (!movieHeader || !Number.isSafeInteger(movieHeader.size)) return undefined; + const payloadOffset = movieHeader.offset + movieHeader.headerSize; + if (movieHeader.size - movieHeader.headerSize < MVHD_FIXED_LAYOUT_BYTES) return undefined; + const buffer = Buffer.alloc(MVHD_FIXED_LAYOUT_BYTES); + let fd: number | undefined; + try { + fd = fs.openSync(filePath, 'r'); + return fs.readSync(fd, buffer, 0, buffer.length, payloadOffset) === buffer.length + ? buffer + : undefined; + } catch { + return undefined; + } finally { + if (fd !== undefined) fs.closeSync(fd); + } +} diff --git a/packages/capture-kit/src/recording/mp4.fixtures.ts b/packages/capture-kit/src/recording/mp4.fixtures.ts new file mode 100644 index 0000000000..b7fc625311 --- /dev/null +++ b/packages/capture-kit/src/recording/mp4.fixtures.ts @@ -0,0 +1,39 @@ +const ATOM_HEADER_BYTES = 8; +const EXTENDED_SIZE_BYTES = 8; +const MOVIE_HEADER_V0_PAYLOAD_BYTES = 100; +const MOVIE_HEADER_V1_PAYLOAD_BYTES = 112; + +export function mp4Atom(type: string, payload: Buffer): Buffer { + const header = Buffer.alloc(ATOM_HEADER_BYTES); + header.writeUInt32BE(ATOM_HEADER_BYTES + payload.length, 0); + header.write(type, 4, 'latin1'); + return Buffer.concat([header, payload]); +} + +export function mp4AtomWithExtendedSize(type: string, payload: Buffer): Buffer { + const header = Buffer.alloc(ATOM_HEADER_BYTES + EXTENDED_SIZE_BYTES); + header.writeUInt32BE(1, 0); + header.write(type, 4, 'latin1'); + header.writeBigUInt64BE(BigInt(header.length + payload.length), ATOM_HEADER_BYTES); + return Buffer.concat([header, payload]); +} + +export function mp4MovieHeader(params: { + version: 0 | 1; + timescale: number; + duration: number; +}): Buffer { + const versioned = params.version === 1; + const payload = Buffer.alloc( + versioned ? MOVIE_HEADER_V1_PAYLOAD_BYTES : MOVIE_HEADER_V0_PAYLOAD_BYTES, + ); + payload.writeUInt8(params.version, 0); + if (versioned) { + payload.writeUInt32BE(params.timescale, 20); + payload.writeBigUInt64BE(BigInt(params.duration), 24); + } else { + payload.writeUInt32BE(params.timescale, 12); + payload.writeUInt32BE(params.duration, 16); + } + return payload; +} diff --git a/packages/capture-kit/src/recording/video.test.ts b/packages/capture-kit/src/recording/video.test.ts index d075ec0d7a..f334608259 100644 --- a/packages/capture-kit/src/recording/video.test.ts +++ b/packages/capture-kit/src/recording/video.test.ts @@ -1,10 +1,24 @@ import fs from 'node:fs'; import path from 'node:path'; -import { expect, test } from 'vitest'; +import { beforeEach, expect, test, vi } from 'vitest'; +import { runCmd } from '@agent-device/host-kit/command'; import { likelyPlayableWebmContainer } from '../__tests__/test-utils/video-fixtures.ts'; import { mkdtempForTestSync } from '../tmp-dir.fixtures.ts'; +import { mp4Atom, mp4MovieHeader } from './mp4.fixtures.ts'; import { isPlayableVideo } from './video.ts'; +vi.mock('@agent-device/host-kit/command', () => ({ + runCmd: vi.fn(async () => ({ exitCode: 0, stdout: '', stderr: '' })), +})); +vi.mock('./swift-cache.ts', () => ({ + buildSwiftToolEnv: () => ({}), + compileSwiftSourceText: async () => '/bin/true', +})); + +beforeEach(() => { + vi.mocked(runCmd).mockClear(); +}); + const directory = mkdtempForTestSync('agent-device-video-webm-'); const playableWebm = likelyPlayableWebmContainer(); @@ -21,3 +35,21 @@ function writeFixture(name: string, bytes: Buffer): string { fs.writeFileSync(filePath, bytes); return filePath; } + +const ftyp = mp4Atom('ftyp', Buffer.from('isom', 'latin1')); +const moov = mp4Atom( + 'moov', + mp4Atom('mvhd', mp4MovieHeader({ version: 0, timescale: 1_000, duration: 2_000 })), +); + +test('reaches the semantic validator with an MP4 container whose last box runs past the file', async () => { + const truncated = Buffer.concat([ftyp, moov]).subarray(0, ftyp.length + moov.length - 4); + await expect(isPlayableVideo(writeFixture('truncated.mp4', truncated))).resolves.toBe(true); + expect(runCmd).toHaveBeenCalledTimes(1); +}); + +test('refuses an MP4 container that never declares a movie header', async () => { + const container = Buffer.concat([ftyp, mp4Atom('mdat', Buffer.alloc(8))]); + await expect(isPlayableVideo(writeFixture('no-moov.mp4', container))).resolves.toBe(false); + expect(runCmd).not.toHaveBeenCalled(); +}); diff --git a/packages/capture-kit/src/recording/video.ts b/packages/capture-kit/src/recording/video.ts index c3634b2bcd..81ff9297c0 100644 --- a/packages/capture-kit/src/recording/video.ts +++ b/packages/capture-kit/src/recording/video.ts @@ -68,7 +68,7 @@ export async function waitForStableFile( } export async function isPlayableVideo(filePath: string): Promise { - const container = likelyPlayableVideoContainer(filePath); + const container = await likelyPlayableVideoContainer(filePath); if (!container) return false; // AVFoundation is the MP4 semantic validator. It does not reliably load WebM on supported // macOS hosts, so WebM completion is established by its EBML document type + Segment marker. @@ -141,7 +141,7 @@ function isSwiftVideoValidatorUnavailable(stderr: string, stdout: string): boole ); } -function likelyPlayableVideoContainer(filePath: string): 'mp4' | 'webm' | undefined { +async function likelyPlayableVideoContainer(filePath: string): Promise<'mp4' | 'webm' | undefined> { try { const stats = fs.statSync(filePath); if (!stats.isFile() || stats.size <= 0) { @@ -154,49 +154,14 @@ function likelyPlayableVideoContainer(filePath: string): 'mp4' | 'webm' | undefi if (filePath.toLowerCase().endsWith('.webm')) { return hasPlayableWebmStructure(filePath) ? 'webm' : undefined; } - const atoms = inspectTopLevelAtoms(filePath); - return atoms.includes('ftyp') && atoms.includes('moov') ? 'mp4' : undefined; + return (await isMp4Container(filePath)) ? 'mp4' : undefined; } -function inspectTopLevelAtoms(filePath: string): string[] { - try { - const fd = fs.openSync(filePath, 'r'); - try { - const size = fs.fstatSync(fd).size; - let offset = 0; - const atoms: string[] = []; - while (offset + 8 <= size && atoms.length < 16) { - const header = Buffer.alloc(8); - const bytesRead = fs.readSync(fd, header, 0, 8, offset); - if (bytesRead < 8) { - break; - } - - let atomSize = header.readUInt32BE(0); - const atomType = header.toString('latin1', 4, 8); - atoms.push(atomType); - - if (atomSize === 1) { - const extended = Buffer.alloc(8); - const extendedRead = fs.readSync(fd, extended, 0, 8, offset + 8); - if (extendedRead < 8) { - break; - } - atomSize = Number(extended.readBigUInt64BE(0)); - } - - // A top-level MP4 atom size of 0 extends to EOF. We stop here because there is no - // next sibling atom to inspect, and advancing by 0 would loop forever. - if (!Number.isFinite(atomSize) || atomSize <= 0) { - break; - } - offset += atomSize; - } - return atoms; - } finally { - fs.closeSync(fd); - } - } catch { - return []; - } +// Loaded on the first validated file rather than on import: this scan is what a recording +// completion asks for, and nothing that merely imports this module should evaluate it. +async function isMp4Container(filePath: string): Promise { + const { findMp4Atom } = await import('./mp4-atoms.ts'); + return ( + findMp4Atom(filePath, ['ftyp']) !== undefined && findMp4Atom(filePath, ['moov']) !== undefined + ); } diff --git a/packages/contracts/src/recording.ts b/packages/contracts/src/recording.ts index 401a2e3701..92104d976f 100644 --- a/packages/contracts/src/recording.ts +++ b/packages/contracts/src/recording.ts @@ -27,6 +27,7 @@ export type RecordingStopCommandResult = { recordOnlySession?: boolean; activeSessionApp?: RecordingAppIdentity; durationMs: number; + capturedDurationMs?: number; showTouches: boolean; warning?: string; overlayWarning?: string; diff --git a/packages/contracts/src/screen-recording-runtime.ts b/packages/contracts/src/screen-recording-runtime.ts index 3b151c0245..f692d867b3 100644 --- a/packages/contracts/src/screen-recording-runtime.ts +++ b/packages/contracts/src/screen-recording-runtime.ts @@ -84,6 +84,13 @@ export type ScreenRecordingCompletion = Readonly<{ clientOutPath?: string; startedAt: number; completedAt: number; + /** + * Duration the finished video timelines actually cover, when the backend can measure them. A + * backend that encodes only on screen changes can capture less video than the `startedAt` to + * `completedAt` window reports as the recording duration; a chunked recording sums its chunk + * timelines, which excludes whatever a chunk handover cost. + */ + capturedDurationMs?: number; scope: RecordingScope; showTouches: boolean; recordOnlySession: boolean; diff --git a/packages/platform-android/src/recording/captured-window.test.ts b/packages/platform-android/src/recording/captured-window.test.ts new file mode 100644 index 0000000000..f73d0c81c1 --- /dev/null +++ b/packages/platform-android/src/recording/captured-window.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, test, vi } from 'vitest'; +import { readMp4DurationMs } from '@agent-device/capture-kit/recording-mp4-duration'; +import { measureCapturedWindow } from './captured-window.ts'; + +vi.mock('@agent-device/capture-kit/recording-mp4-duration', () => ({ readMp4DurationMs: vi.fn() })); + +const measured = vi.mocked(readMp4DurationMs); +const STARTED_AT_MS = 1_789_000_000_000; + +function capture(durations: readonly (number | undefined)[], windowMs: number) { + let index = 0; + measured.mockImplementation(() => durations[index++]); + return measureCapturedWindow({ + chunkPaths: durations.map((_, offset) => + offset === 0 ? '/tmp/capture.mp4' : `/tmp/capture.part-${offset + 1}.mp4`, + ), + startedAtMs: STARTED_AT_MS, + stoppedAtMs: STARTED_AT_MS + windowMs, + }); +} + +describe('measureCapturedWindow', () => { + test('names the clip length and the window length it fell short of', () => { + expect(capture([7_000], 16_000)).toEqual({ + capturedDurationMs: 7_000, + idleTailWarning: + 'Android screenrecord encodes a frame only when the screen changes, so this video ends at ' + + 'the last frame it encoded: it covers 7.0s of the 16.0s recording window.', + }); + }); + + test('warns for a window the recorder never drew on', () => { + expect(capture([0], 13_000)).toEqual({ + capturedDurationMs: 0, + idleTailWarning: + 'Android screenrecord encodes a frame only when the screen changes, so this video ends at ' + + 'the last frame it encoded: it covers 0.0s of the 13.0s recording window.', + }); + }); + + test('reports the clip length without a warning when the tail is export latency', () => { + expect(capture([6_998], 7_000)).toEqual({ capturedDurationMs: 6_998 }); + }); + + test('sums the video of a chunked capture', () => { + expect(capture([170_000, 9_000], 180_500)).toEqual({ capturedDurationMs: 179_000 }); + }); + + test('leaves a capture longer than its window unexplained rather than inventing a tail', () => { + expect(capture([180_000], 60_000)).toEqual({ capturedDurationMs: 180_000 }); + }); + + test('reports the clip length when the host clock moved the window backwards', () => { + expect(capture([7_000], -1_000)).toEqual({ capturedDurationMs: 7_000 }); + }); + + test('stays silent when a chunk cannot answer with a duration', () => { + expect(capture([7_000, undefined], 16_000)).toEqual({}); + }); +}); diff --git a/packages/platform-android/src/recording/captured-window.ts b/packages/platform-android/src/recording/captured-window.ts new file mode 100644 index 0000000000..651ebd6b71 --- /dev/null +++ b/packages/platform-android/src/recording/captured-window.ts @@ -0,0 +1,48 @@ +import { readMp4DurationMs } from '@agent-device/capture-kit/recording-mp4-duration'; + +/** Below this, a clip that ends before `record stop` is normal request-and-export latency. */ +const IDLE_TAIL_WARNING_MS = 2_000; + +/** + * Measures how much video really reached the pulled chunks and how much of the recording window it + * covers. Android's `screenrecord` encodes a frame only when the screen changes, so a clip ends at + * the frame it encoded last rather than at `record stop`, and the caller needs to know which of the + * two lengths they are holding. + * + * The window is host elapsed time between launching the recorder and sending the stop signal, the + * span this tool itself bracketed; its own export latency belongs to neither. A host that slept + * mid-recording reports a window shorter than the device saw, which costs the warning rather than + * inventing one. An unreadable chunk costs the measurement, never the recording. + */ +export function measureCapturedWindow(params: { + chunkPaths: readonly string[]; + startedAtMs: number; + stoppedAtMs: number; +}): Readonly<{ capturedDurationMs?: number; idleTailWarning?: string }> { + const capturedDurationMs = sumCapturedDurationMs(params.chunkPaths); + if (capturedDurationMs === undefined) return {}; + const windowMs = params.stoppedAtMs - params.startedAtMs; + const idleTailMs = windowMs - capturedDurationMs; + if (idleTailMs < IDLE_TAIL_WARNING_MS) return { capturedDurationMs }; + return { + capturedDurationMs, + idleTailWarning: + 'Android screenrecord encodes a frame only when the screen changes, so this video ends at ' + + 'the last frame it encoded: it covers ' + + `${formatSeconds(capturedDurationMs)}s of the ${formatSeconds(windowMs)}s recording window.`, + }; +} + +function sumCapturedDurationMs(chunkPaths: readonly string[]): number | undefined { + let total = 0; + for (const chunkPath of chunkPaths) { + const durationMs = readMp4DurationMs(chunkPath); + if (durationMs === undefined) return undefined; + total += durationMs; + } + return total; +} + +function formatSeconds(ms: number): string { + return (ms / 1000).toFixed(1); +} diff --git a/packages/platform-android/src/recording/completion.test.ts b/packages/platform-android/src/recording/completion.test.ts new file mode 100644 index 0000000000..5f7b89cef4 --- /dev/null +++ b/packages/platform-android/src/recording/completion.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, test, vi } from 'vitest'; +import { readMp4DurationMs } from '@agent-device/capture-kit/recording-mp4-duration'; +import { completed, snapshot } from './completion.ts'; +import { recordingHost, recordingInput } from './fixtures.ts'; + +vi.mock('@agent-device/capture-kit/recording-mp4-duration', () => ({ readMp4DurationMs: vi.fn() })); + +const STARTED_AT_MS = 1_789_000_000_000; + +async function capture(params: { + clipMs: number; + windowMs: number; + chunks?: number; + reachedLimit?: boolean; + finalization?: Record; +}) { + vi.mocked(readMp4DurationMs).mockReturnValue(params.clipMs); + return await completed({ + host: recordingHost({ finalize: { complete: async () => params.finalization ?? {} } }), + recording: snapshot(recordingInput(), 1), + chunks: Array.from({ length: params.chunks ?? 1 }, (_, offset) => ({ + index: offset + 1, + path: offset === 0 ? '/tmp/capture.mp4' : `/tmp/capture.part-${offset + 1}.mp4`, + })), + targetLabel: 'Android recording', + reachedLimit: params.reachedLimit ?? false, + startedAtMs: STARTED_AT_MS, + stoppedAtMs: STARTED_AT_MS + params.windowMs, + }); +} + +describe('completed', () => { + test('reports the clip the recorder really captured beside the requested window', async () => { + const outcome = await capture({ + clipMs: 7_000, + windowMs: 16_000, + finalization: { telemetryPath: '/tmp/capture.gesture-telemetry.json' }, + }); + expect(outcome.result).toMatchObject({ + telemetryPath: '/tmp/capture.gesture-telemetry.json', + capturedDurationMs: 7_000, + warning: + 'Android screenrecord encodes a frame only when the screen changes, so this video ends at ' + + 'the last frame it encoded: it covers 7.0s of the 16.0s recording window.', + }); + }); + + test('appends every recording warning behind the finalizer warning', async () => { + const outcome = await capture({ + clipMs: 180_000, + windowMs: 400_000, + chunks: 2, + reachedLimit: true, + finalization: { warning: 'recording was exported without touch overlays' }, + }); + const warning = outcome.result.warning ?? ''; + expect(warning).toMatch( + /^recording was exported without touch overlays Android adb screenrecord stopped before record stop/, + ); + expect(warning.indexOf('is capped at 180s')).toBeLessThan( + warning.indexOf('encodes a frame only'), + ); + expect(outcome.result.capturedDurationMs).toBe(360_000); + }); +}); diff --git a/packages/platform-android/src/recording/completion.ts b/packages/platform-android/src/recording/completion.ts index 83952dd8be..6cad4d07f9 100644 --- a/packages/platform-android/src/recording/completion.ts +++ b/packages/platform-android/src/recording/completion.ts @@ -6,6 +6,16 @@ import type { ScreenRecordingLiveSnapshot, ScreenRecordingStartInput, } from '@agent-device/contracts/screen-recording-runtime'; +import { measureCapturedWindow } from './captured-window.ts'; + +const PLATFORM_LIMIT_WARNING = + 'Android adb screenrecord stopped before record stop, likely after reaching the 180s platform ' + + 'limit. The MP4 may be truncated; final interactions after the limit are not in the video.'; +const CHUNKED_WARNING = + 'Android adb screenrecord is capped at 180s, so this recording was split into multiple MP4 chunks.'; +const CHUNKED_OVERLAY_WARNING = + 'touch overlay burn-in is skipped for chunked Android recordings; returning raw chunks plus ' + + 'gesture telemetry'; export function snapshot( input: ScreenRecordingStartInput, @@ -25,13 +35,16 @@ export function snapshot( }); } -export async function completed( - host: PlatformRuntimeHost, - recording: ScreenRecordingLiveSnapshot, - chunks: readonly ScreenRecordingChunk[], - targetLabel: string, - reachedLimit = false, -): Promise> { +export async function completed(params: { + host: PlatformRuntimeHost; + recording: ScreenRecordingLiveSnapshot; + chunks: readonly ScreenRecordingChunk[]; + targetLabel: string; + reachedLimit: boolean; + startedAtMs: number; + stoppedAtMs: number; +}): Promise> { + const { host, recording, chunks, targetLabel, reachedLimit, startedAtMs, stoppedAtMs } = params; const chunked = chunks.length > 1; const finalization = await host.screenRecording.finalize.complete({ outputPath: recording.outPath, @@ -40,18 +53,18 @@ export async function completed( exportQuality: recording.exportQuality ?? 'medium', targetLabel, }); + const captured = measureCapturedWindow({ + chunkPaths: chunks.map((chunk) => chunk.path), + startedAtMs, + stoppedAtMs, + }); const warnings = [ - ...(reachedLimit - ? [ - 'Android adb screenrecord stopped before record stop, likely after reaching the 180s platform limit. The MP4 may be truncated; final interactions after the limit are not in the video.', - ] - : []), - ...(chunked - ? [ - 'Android adb screenrecord is capped at 180s, so this recording was split into multiple MP4 chunks.', - ] - : []), + ...(finalization.warning ? [finalization.warning] : []), + ...(reachedLimit ? [PLATFORM_LIMIT_WARNING] : []), + ...(chunked ? [CHUNKED_WARNING] : []), + ...(captured.idleTailWarning ? [captured.idleTailWarning] : []), ]; + const completedAt = Date.now(); return { status: 'completed', result: { @@ -59,19 +72,19 @@ export async function completed( outPath: recording.outPath, ...(recording.clientOutPath ? { clientOutPath: recording.clientOutPath } : {}), startedAt: recording.startedAt, - completedAt: Date.now(), + completedAt, scope: recording.scope, showTouches: recording.showTouches, recordOnlySession: recording.recordOnlySession, ...(recording.activeSessionApp ? { activeSessionApp: recording.activeSessionApp } : {}), ...(chunked ? { chunks } : {}), - ...(warnings.length ? { warning: warnings.join(' ') } : {}), ...finalization, + ...(captured.capturedDurationMs === undefined + ? {} + : { capturedDurationMs: captured.capturedDurationMs }), + ...(warnings.length ? { warning: warnings.join(' ') } : {}), ...(chunked && recording.showTouches && recording.gestureEvents.length > 0 - ? { - overlayWarning: - 'touch overlay burn-in is skipped for chunked Android recordings; returning raw chunks plus gesture telemetry', - } + ? { overlayWarning: CHUNKED_OVERLAY_WARNING } : {}), }, }; diff --git a/packages/platform-android/src/recording/finalize.test.ts b/packages/platform-android/src/recording/finalize.test.ts index eb58845441..1c6646f693 100644 --- a/packages/platform-android/src/recording/finalize.test.ts +++ b/packages/platform-android/src/recording/finalize.test.ts @@ -1,4 +1,8 @@ +import fs from 'node:fs'; +import path from 'node:path'; import { expect, test } from 'vitest'; +import { mp4Atom, mp4MovieHeader } from '@agent-device/capture-kit/recording-mp4-fixtures'; +import { mkdtempForTestSync } from '../__tests__/test-utils/tmp-dir.ts'; import { finalizeAndroidRecording } from './finalize.ts'; import { androidRecordingDevice, recordingHost, recordingInput } from './fixtures.ts'; import { createNativeManifest } from './manifest.ts'; @@ -39,7 +43,62 @@ test('writes terminal coordinates before removing a fenced Android artifact', as evidence, manifestPath: '/sdcard/agent-device-recording-active.json', recording: snapshot(input, 1), + startedAtMs: 1, }), ).resolves.toMatchObject({ status: 'completed' }); expect(calls).toEqual(['completed', 'remove:/sdcard/agent-device-recording-1.mp4']); }); + +test('measures a pulled MP4 against the window the host bracketed around the recorder', async () => { + const directory = mkdtempForTestSync('agent-device-android-finalize-'); + const calls: string[] = []; + const host = recordingHost({ + stop: async () => { + calls.push('stop'); + return 'stopped' as const; + }, + isRunning: async () => false, + pullPlayable: async ({ outputPath }: { outputPath: string }) => { + calls.push('pull'); + fs.writeFileSync( + outputPath, + Buffer.concat([ + mp4Atom('mdat', Buffer.alloc(8)), + mp4Atom( + 'moov', + mp4Atom('mvhd', mp4MovieHeader({ version: 0, timescale: 1_000, duration: 7_000 })), + ), + ]), + ); + return { stdout: '', stderr: '', exitCode: 0, playable: true }; + }, + }); + const input = { ...recordingInput(), outputPath: path.join(directory, 'capture.mp4') }; + const transport = await host.screenRecording.android.resolve(androidRecordingDevice); + const outcome = await finalizeAndroidRecording({ + host, + transport, + evidence: createNativeManifest( + androidRecordingDevice, + input, + 1, + [ + { + index: 1, + remotePath: '/sdcard/agent-device-recording-1.mp4', + remotePid: '41', + remoteStartTime: '7', + }, + ], + undefined, + 'local', + ), + manifestPath: '/sdcard/agent-device-recording-active.json', + recording: snapshot(input, 1), + startedAtMs: Date.now() - 16_000, + }); + + expect(calls).toEqual(['stop', 'pull']); + expect(outcome.result.capturedDurationMs).toBe(7_000); + expect(outcome.result.warning).toMatch(/it covers 7\.0s of the 16\.\ds recording window\./); +}); diff --git a/packages/platform-android/src/recording/finalize.ts b/packages/platform-android/src/recording/finalize.ts index bdc51af5ca..b38b563203 100644 --- a/packages/platform-android/src/recording/finalize.ts +++ b/packages/platform-android/src/recording/finalize.ts @@ -14,6 +14,8 @@ export async function finalizeAndroidRecording(params: { evidence: NativeManifest; manifestPath: string; recording: ScreenRecordingLiveSnapshot; + /** Host instant the recorder was launched, which is where a clip's timeline begins. */ + startedAtMs: number; reachedLimit?: boolean; }): Promise< Readonly<{ @@ -21,6 +23,9 @@ export async function finalizeAndroidRecording(params: { result: import('@agent-device/contracts/screen-recording-runtime').ScreenRecordingCompletion; }> > { + // Read the clock before the signal below: everything after that signal is this tool's own export + // latency rather than time the screen sat unchanged. + const stoppedAtMs = Date.now(); const reachedLimit = (await stopOwnedChunks(params.transport, params.evidence.chunks)) || params.reachedLimit === true; @@ -31,13 +36,15 @@ export async function finalizeAndroidRecording(params: { params.recording.outPath, params.recording.clientOutPath, ); - const outcome = await completed( - params.host, - params.recording, - outputChunks, - 'Android recording', + const outcome = await completed({ + host: params.host, + recording: params.recording, + chunks: outputChunks, + targetLabel: 'Android recording', reachedLimit, - ); + startedAtMs: params.startedAtMs, + stoppedAtMs, + }); await persistNativeManifest( params.transport, params.manifestPath, diff --git a/packages/platform-android/src/recording/launch.ts b/packages/platform-android/src/recording/launch.ts index bfab3a1a34..d0e6001f75 100644 --- a/packages/platform-android/src/recording/launch.ts +++ b/packages/platform-android/src/recording/launch.ts @@ -20,7 +20,7 @@ export async function startInitialTransaction(params: { startedAt: number; signal: AbortSignal; prepareOutput: () => Promise; -}): Promise> { +}): Promise> { const { transport, device, input, startedAt, signal, prepareOutput } = params; await reconcileStartEvidence(transport, device); await prepareOutput(); @@ -34,7 +34,11 @@ export async function startInitialTransaction(params: { signal, ); let chunk: NativeChunk; + let startedAtMs: number; try { + // Timestamp immediately before launching, the way the stop signal is timestamped, so the + // window a short clip is measured against holds only the recording. + startedAtMs = Date.now(); chunk = await startChunkAt(transport, remotePath, input, signal); } catch (error) { if (signal.aborted) { @@ -56,7 +60,7 @@ export async function startInitialTransaction(params: { await rollbackPublishedChunk(transport, manifestPath, chunk); throw error; } - return { chunk, manifestPath }; + return { chunk, manifestPath, startedAtMs }; } throw last ?? new Error('Android screenrecord did not begin producing frames'); } diff --git a/packages/platform-android/src/recording/manifest-validation.test.ts b/packages/platform-android/src/recording/manifest-validation.test.ts index 2513f80e1d..783a3af4fd 100644 --- a/packages/platform-android/src/recording/manifest-validation.test.ts +++ b/packages/platform-android/src/recording/manifest-validation.test.ts @@ -62,3 +62,41 @@ test('rejects terminal evidence whose result coordinates diverge from its manife }), ).toBe(false); }); + +test('accepts a measured clip length in terminal evidence and refuses an uncountable one', () => { + const input = recordingInput(); + const active = createNativeManifest( + androidRecordingDevice, + input, + 1, + [ + { + index: 1, + remotePath: '/sdcard/agent-device-recording-1.mp4', + remotePid: '41', + remoteStartTime: '7', + }, + ], + undefined, + 'local', + ); + const complete = createCompletedNativeManifest(active, { + backend: 'adb screenrecord', + outPath: input.outputPath, + startedAt: 1, + completedAt: 2, + capturedDurationMs: 7_000, + scope: input.scope, + showTouches: input.showTouches, + recordOnlySession: input.recordOnlySession, + }); + expect(isValidNativeManifest(complete)).toBe(true); + for (const capturedDurationMs of ['7000', Number.NaN]) { + expect( + isValidNativeManifest({ + ...complete, + completion: { ...complete.completion!, capturedDurationMs }, + }), + ).toBe(false); + } +}); diff --git a/packages/platform-android/src/recording/manifest-validation.ts b/packages/platform-android/src/recording/manifest-validation.ts index 9942f1d776..df33ce44b8 100644 --- a/packages/platform-android/src/recording/manifest-validation.ts +++ b/packages/platform-android/src/recording/manifest-validation.ts @@ -134,7 +134,8 @@ function completionIdentityIsValid(candidate: Partial typeof candidate.outPath === 'string' && (candidate.clientOutPath === undefined || typeof candidate.clientOutPath === 'string') && Number.isFinite(candidate.startedAt) && - Number.isFinite(candidate.completedAt) + Number.isFinite(candidate.completedAt) && + (candidate.capturedDurationMs === undefined || Number.isFinite(candidate.capturedDurationMs)) ); } diff --git a/packages/platform-android/src/recording/recovery.ts b/packages/platform-android/src/recording/recovery.ts index 063a15b4c9..3b517fedf4 100644 --- a/packages/platform-android/src/recording/recovery.ts +++ b/packages/platform-android/src/recording/recovery.ts @@ -176,6 +176,7 @@ async function reattachEvidence(params: { evidence, manifestPath: descriptor.manifestPath, recording: current, + startedAtMs: evidence.startedAt, reachedLimit: provesAndroidScreenRecordTermination(running), }); nativeCleanupConfirmed = true; diff --git a/packages/platform-android/src/recording/runtime.ts b/packages/platform-android/src/recording/runtime.ts index 25dc8e21af..f6099ddcb8 100644 --- a/packages/platform-android/src/recording/runtime.ts +++ b/packages/platform-android/src/recording/runtime.ts @@ -160,6 +160,7 @@ async function startAndroidRecording(params: { evidence: createNativeManifest(device, input, startedAt, chunks, undefined, transport.mode), manifestPath, recording: current, + startedAtMs: initial.startedAtMs, }); nativeCleanupConfirmed = true; return outcome; diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index e048965066..9e2955290e 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -409,6 +409,8 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/capture-kit/post-gesture-stability', '@agent-device/capture-kit/quality-warnings', '@agent-device/capture-kit/react-native-overlay', + '@agent-device/capture-kit/recording-mp4-duration', + '@agent-device/capture-kit/recording-mp4-fixtures', '@agent-device/capture-kit/recording-output-path', '@agent-device/capture-kit/recording-overlay', '@agent-device/capture-kit/recording-telemetry', diff --git a/src/commands/recording/index.ts b/src/commands/recording/index.ts index 40b1ecea3c..4d092ded73 100644 --- a/src/commands/recording/index.ts +++ b/src/commands/recording/index.ts @@ -104,7 +104,7 @@ export const recordCommandFacet = defineCommandFacet({ text: { summary: 'Start or stop screen recording', cliDetail: - 'The default --scope app requires an active app session from open ; use --scope device/system to explicitly request whole-screen recording where the selected backend supports it. Android record start publishes a durable device manifest, recordings longer than the 180s adb screenrecord limit are returned as multiple MP4 chunks while the daemon stays alive, and daemon-restart recovery uses only manifest-owned chunks. An Android manifest left by an unreachable recording is retired on the next start once its recorders are proven gone; one still owned refuses with non-retriable DEVICE_IN_USE and reason native_recovery_evidence_open naming the session to run record stop for, and a recorder that is still writing an artifact refuses with DEVICE_IN_USE and reason native_recording_artifact_claimed, which clears itself once that recorder ends at the 180s limit. HarmonyOS supports whole-screen recording on physical devices only: use --scope device/system; --fps, --quality, and --hide-touches are unsupported. Use --quality to choose medium or high export quality on supported backends. An iOS simulator host recording lock returns non-retriable DEVICE_IN_USE with reason apple_simulator_recording_busy. Stop the recording in its owning session; if a dead recorder left the host locked, ask the host operator to restart the CoreSimulator stream service.', + 'The default --scope app requires an active app session from open ; use --scope device/system to explicitly request whole-screen recording where the selected backend supports it. Android record start publishes a durable device manifest, recordings longer than the 180s adb screenrecord limit are returned as multiple MP4 chunks while the daemon stays alive, and daemon-restart recovery uses only manifest-owned chunks. Android screenrecord encodes a frame only when the screen changes, so a clip can end at the last frame the recorder encoded instead of at record stop; durationMs is host wall clock from record start until the export finished, and capturedDurationMs reports the video timeline when it can be measured, with a warning naming how much of the window that video covers. An Android manifest left by an unreachable recording is retired on the next start once its recorders are proven gone; one still owned refuses with non-retriable DEVICE_IN_USE and reason native_recovery_evidence_open naming the session to run record stop for, and a recorder that is still writing an artifact refuses with DEVICE_IN_USE and reason native_recording_artifact_claimed, which clears itself once that recorder ends at the 180s limit. HarmonyOS supports whole-screen recording on physical devices only: use --scope device/system; --fps, --quality, and --hide-touches are unsupported. Use --quality to choose medium or high export quality on supported backends. An iOS simulator host recording lock returns non-retriable DEVICE_IN_USE with reason apple_simulator_recording_busy. Stop the recording in its owning session; if a dead recorder left the host locked, ask the host operator to restart the CoreSimulator stream service.', }, metadata: recordCommandMetadata, run: (client, input) => client.recording.record(input as RecordOptions), diff --git a/src/daemon/__tests__/screen-recording-stop-recovery.test.ts b/src/daemon/__tests__/screen-recording-stop-recovery.test.ts index 191f20d05a..2bced757c5 100644 --- a/src/daemon/__tests__/screen-recording-stop-recovery.test.ts +++ b/src/daemon/__tests__/screen-recording-stop-recovery.test.ts @@ -56,6 +56,7 @@ test('a completed manifest whose stored response is damaged serves nothing', asy { outPath: 0 }, { clientOutPath: '' }, { completedAt: 'later' }, + { capturedDurationMs: 'soon' }, { chunks: [{ index: 'first', path: '/daemon/capture-0.mp4' }] }, { activeSessionApp: { bundleId: '' } }, ]; @@ -125,6 +126,7 @@ function fullCompletion(outPath: string): ScreenRecordingCompletion { outPath, startedAt: 1, completedAt: 4, + capturedDurationMs: 3, scope: 'app', showTouches: true, recordOnlySession: false, diff --git a/src/daemon/handlers/__tests__/record-runtime-response.test.ts b/src/daemon/handlers/__tests__/record-runtime-response.test.ts index 07b2c4f630..829f573f14 100644 --- a/src/daemon/handlers/__tests__/record-runtime-response.test.ts +++ b/src/daemon/handlers/__tests__/record-runtime-response.test.ts @@ -65,3 +65,21 @@ test('stop response derives client telemetry and chunk artifact paths', () => { }), ); }); + +test('stop response separates the recording window from the clip that was captured', () => { + const completion = { + backend: 'adb screenrecord', + outPath: '/daemon/capture.mp4', + startedAt: 1_000, + completedAt: 17_000, + scope: 'device', + showTouches: false, + recordOnlySession: false, + } as const; + const measured = buildRecordingStopResponse({ ...completion, capturedDurationMs: 7_000 }); + const unmeasured = buildRecordingStopResponse(completion); + if (!measured.ok || !unmeasured.ok) throw new Error('expected a successful recording stop'); + + expect(measured.data).toMatchObject({ durationMs: 16_000, capturedDurationMs: 7_000 }); + expect(unmeasured.data).not.toHaveProperty('capturedDurationMs'); +}); diff --git a/src/daemon/handlers/record-runtime-response.ts b/src/daemon/handlers/record-runtime-response.ts index cfb52a580e..74d59e8458 100644 --- a/src/daemon/handlers/record-runtime-response.ts +++ b/src/daemon/handlers/record-runtime-response.ts @@ -89,6 +89,9 @@ export function buildRecordingStopResponse(completion: ScreenRecordingCompletion recordOnlySession: completion.recordOnlySession, activeSessionApp: completion.activeSessionApp, durationMs: Math.max(0, completion.completedAt - completion.startedAt), + ...(completion.capturedDurationMs === undefined + ? {} + : { capturedDurationMs: completion.capturedDurationMs }), showTouches: completion.showTouches, warning: completion.warning, overlayWarning: completion.overlayWarning, diff --git a/src/daemon/screen-recording-session-resource.ts b/src/daemon/screen-recording-session-resource.ts index b57a3ae2f9..f3f6f14319 100644 --- a/src/daemon/screen-recording-session-resource.ts +++ b/src/daemon/screen-recording-session-resource.ts @@ -79,6 +79,9 @@ export function encodeScreenRecordingCompletionMetadata( outPath: completion.outPath, startedAt: completion.startedAt, completedAt: completion.completedAt, + ...(completion.capturedDurationMs === undefined + ? {} + : { capturedDurationMs: completion.capturedDurationMs }), scope: completion.scope, showTouches: completion.showTouches, recordOnlySession: completion.recordOnlySession, diff --git a/src/daemon/screen-recording-stop-recovery.ts b/src/daemon/screen-recording-stop-recovery.ts index 5457522ee5..3ac3220cfe 100644 --- a/src/daemon/screen-recording-stop-recovery.ts +++ b/src/daemon/screen-recording-stop-recovery.ts @@ -123,6 +123,7 @@ function isServedCompletion(stored: Record): boolean { isNonEmptyString(stored.backend) && isFiniteNumber(stored.startedAt) && isFiniteNumber(stored.completedAt) && + (stored.capturedDurationMs === undefined || isFiniteNumber(stored.capturedDurationMs)) && isRecordingScope(stored.scope) && typeof stored.showTouches === 'boolean' && typeof stored.recordOnlySession === 'boolean' diff --git a/src/mcp/command-output-schemas.ts b/src/mcp/command-output-schemas.ts index 20e83f6b49..a5db763308 100644 --- a/src/mcp/command-output-schemas.ts +++ b/src/mcp/command-output-schemas.ts @@ -843,6 +843,7 @@ const BASE_COMMAND_OUTPUT_SCHEMAS = { recordOnlySession: booleanSchema(), activeSessionApp: looseObjectSchema(), durationMs: numberSchema(), + capturedDurationMs: numberSchema(), showTouches: booleanSchema(), warning: stringSchema(), overlayWarning: stringSchema(), diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index fc89e79e54..ccbc56aeda 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -965,6 +965,7 @@ agent-device record stop # Stop active recording - On Linux or other non-macOS hosts, `record stop` still succeeds and returns the raw video plus telemetry sidecar, and includes `overlayWarning` when burn-in overlays were skipped. - On iOS simulators, a busy CoreSimulator host recording slot makes `record start` return non-retriable `DEVICE_IN_USE` with `details.reason: apple_simulator_recording_busy`. Use `record stop` in the session that owns the active recording. If a previous recorder died and no recording is active, ask the host operator to restart the CoreSimulator stream service before retrying. - Android uses `adb shell screenrecord`, which has a 180s platform limit. `record start` publishes a durable device manifest. Longer recordings are split into MP4 chunks while the daemon stays alive; after daemon restart, `record stop` recovers only manifest-owned chunks and warns when gesture overlay telemetry was lost. +- Android `screenrecord` encodes a frame only when the screen changes, so a clip ends at the last frame the recorder encoded instead of at `record stop`: a window that ends on an unchanged screen yields a shorter video, while every on-screen change inside the window stays at its real offset in it. `record stop` reports `durationMs` as host wall clock from `record start` until the export finished, and when the video can be measured it also reports `capturedDurationMs` and warns with how much of the window that video covers. - `record stop` is safe to repeat. When its request window ends while the daemon is still exporting — typical for a long touch-overlay burn-in on a remote daemon — the export keeps running there, and a second `record stop` in the same session returns that completed recording, including the caller-side output path, without starting another recording. A finished recording whose video file is already gone reports `no active recording`. **Session app logs (token-efficient debugging):** Logging is off by default in normal flows. Enable it on demand for debugging. Logs are written to a file so agents can grep instead of loading full output into context.