diff --git a/apps/desktop/e2e/browser-tools.spec.ts b/apps/desktop/e2e/browser-tools.spec.ts index fa12465e190..2f4eaf9b148 100644 --- a/apps/desktop/e2e/browser-tools.spec.ts +++ b/apps/desktop/e2e/browser-tools.spec.ts @@ -96,7 +96,7 @@ test.describe('browser tools', () => { test.beforeEach(async () => { app = await electron.launch({ - args: ['.'], + args: [process.env.SIM_DESKTOP_E2E_MAIN ?? '.'], cwd: DESKTOP_DIR, env: { ...process.env, @@ -105,9 +105,15 @@ test.describe('browser tools', () => { }, }) window = await app.firstWindow() - await app.evaluate(({ BrowserWindow }) => - BrowserWindow.getAllWindows()[0].webContents.setBackgroundThrottling(false) - ) + await app.evaluate(({ app, BrowserWindow }) => { + const host = BrowserWindow.getAllWindows()[0] + host.webContents.setBackgroundThrottling(false) + app.focus({ steal: true }) + host.focus() + }) + await expect + .poll(() => app.evaluate(({ BrowserWindow }) => BrowserWindow.getAllWindows()[0].isFocused())) + .toBe(true) await expect(window.getByRole('heading')).toHaveText('Browser tools fixture') await window.evaluate(async (scope) => { const api = (globalThis as typeof globalThis & { simDesktop: SimDesktopApi }).simDesktop @@ -303,6 +309,180 @@ test.describe('browser tools', () => { }) } + test('recovers a permanently pending native capture without losing the page', async () => { + await openForm() + const before = await app.evaluate(async ({ webContents }, origin) => { + const contents = webContents + .getAllWebContents() + .find((wc) => wc.getURL() === `${origin}/form`) + if (!contents) throw new Error('Missing capture fixture') + await contents.executeJavaScript(` + document.getElementById('name').value = 'Unsaved work'; + document.getElementById('name').focus(); + `) + contents.capturePage = () => new Promise(() => {}) + return { id: contents.id, url: contents.getURL() } + }, origin) + for (const [color, dominantChannel] of [ + ['rgb(240, 20, 30)', 0], + ['rgb(30, 40, 230)', 2], + ['rgb(20, 220, 50)', 1], + ] as const) { + await app.evaluate( + async ({ webContents }, { id, color }) => { + const contents = webContents.fromId(id) + if (!contents) throw new Error('Capture fixture was replaced') + await contents.executeJavaScript(` + document.body.style.background = ${JSON.stringify(color)}; + new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))) + `) + }, + { id: before.id, color } + ) + const response = await execute('browser_screenshot', {}) + expect(response.ok, response.error).toBe(true) + const shot = response.result as { dataUrl: string } + const pixel = await app.evaluate(({ nativeImage }, dataUrl) => { + const bitmap = nativeImage.createFromDataURL(dataUrl).toBitmap() + return [bitmap[2], bitmap[1], bitmap[0]] + }, shot.dataUrl) + expect(pixel[dominantChannel]).toBeGreaterThan(180) + for (let channel = 0; channel < 3; channel++) { + if (channel !== dominantChannel) + expect(pixel[dominantChannel] - pixel[channel]).toBeGreaterThan(80) + } + } + const after = await app.evaluate(async ({ webContents }, id) => { + const contents = webContents.fromId(id) + if (!contents) throw new Error('Capture fixture was replaced') + return { + id: contents.id, + url: contents.getURL(), + page: await contents.executeJavaScript( + `({value:document.getElementById('name').value,focus:document.activeElement.id})` + ), + } + }, before.id) + expect(after).toEqual({ ...before, page: { value: 'Unsaved work', focus: 'name' } }) + }) + + test('maps a fractional narrow crop back to its actual viewport position', async () => { + await openForm() + const target = await app.evaluate(async ({ webContents }, origin) => { + const contents = webContents + .getAllWebContents() + .find((wc) => wc.getURL() === `${origin}/form`) + if (!contents) throw new Error('Missing crop fixture') + return contents.executeJavaScript(` + const button = document.createElement('button'); + button.textContent = 'Narrow target'; + button.style.cssText = 'position:absolute;left:20.1px;top:60.1px;width:1.1px;height:100px;padding:0;border:0;overflow:hidden'; + button.onclick = () => { document.body.dataset.cropClicks = Number(document.body.dataset.cropClicks || 0) + 1 }; + document.body.append(button); + const rect = button.getBoundingClientRect(); + ({x:rect.x,y:rect.y,width:rect.width,height:rect.height,devicePixelRatio}); + `) as Promise<{ + x: number + y: number + width: number + height: number + devicePixelRatio: number + }> + }, origin) + const snapshot = await execute('browser_snapshot', {}) + expect(snapshot.ok, snapshot.error).toBe(true) + const line = (snapshot.result as { outline: string }).outline + .split('\n') + .find((line) => line.includes('"Narrow target"')) + const match = line?.match(/\[ref=(\d+)\]/) + if (!match) throw new Error('Missing narrow target reference') + const response = await execute('browser_screenshot', { elementId: Number(match[1]) }) + expect(response.ok, response.error).toBe(true) + const shot = response.result as { + imageSize: { width: number; height: number } + clip: { x: number; y: number; width: number; height: number } + scale: number + } + expect(shot.clip.x).toBeLessThanOrEqual(target.x) + expect(shot.clip.y).toBeLessThanOrEqual(target.y) + expect(shot.clip.x + shot.clip.width).toBeGreaterThanOrEqual(target.x + target.width) + expect(shot.clip.y + shot.clip.height).toBeGreaterThanOrEqual(target.y + target.height) + expect(target.x - shot.clip.x).toBeLessThan(1 / target.devicePixelRatio) + expect(target.y - shot.clip.y).toBeLessThan(1 / target.devicePixelRatio) + expect(shot.scale).toBeCloseTo(shot.imageSize.width / shot.clip.width) + const clicked = await execute('browser_click_at', { + x: shot.clip.x + shot.clip.width / 2, + y: shot.clip.y + shot.clip.height / 2, + }) + expect(clicked.ok, clicked.error).toBe(true) + const count = await app.evaluate(async ({ webContents }, origin) => { + const contents = webContents + .getAllWebContents() + .find((wc) => wc.getURL() === `${origin}/form`) + return contents?.executeJavaScript('document.body.dataset.cropClicks') + }, origin) + expect(count).toBe('1') + }) + + for (const mode of ['hidden', 'minimized']) { + test(`recovers a stalled capture after restoring a ${mode} window`, async () => { + test.skip(mode === 'minimized' && process.platform !== 'darwin', 'Requires minimize events') + await openForm() + await app.evaluate( + async ({ BrowserWindow, webContents }, { origin, mode }) => { + const contents = webContents + .getAllWebContents() + .find((wc) => wc.getURL() === `${origin}/form`) + if (!contents) throw new Error('Missing capture fixture') + contents.capturePage = () => new Promise(() => {}) + await contents.executeJavaScript("document.getElementById('name').value = 'Unsaved work'") + const win = BrowserWindow.getAllWindows()[0] + win.blur() + if (mode === 'hidden') win.hide() + else { + const minimized = new Promise((resolve) => win.once('minimize', resolve)) + win.minimize() + await minimized + } + }, + { origin, mode } + ) + const state = () => + app.evaluate(async ({ BrowserWindow, webContents }, origin) => { + const win = BrowserWindow.getAllWindows()[0] + const contents = webContents + .getAllWebContents() + .find((wc) => wc.getURL() === `${origin}/form`) + if (!contents) throw new Error('Missing capture fixture') + return { + id: contents.id, + visible: win.isVisible(), + minimized: win.isMinimized(), + focused: BrowserWindow.getFocusedWindow()?.id ?? null, + bounds: win.getBounds(), + value: await contents.executeJavaScript("document.getElementById('name').value"), + } + }, origin) + const before = await state() + const start = Date.now() + const hiddenCapture = await execute('browser_screenshot', {}) + expect(Date.now() - start).toBeLessThan(12_000) + if (!hiddenCapture.ok) + expect(hiddenCapture.error).toContain('Screenshot frame capture timed out') + expect(await state()).toEqual(before) + await app.evaluate(({ BrowserWindow }, mode) => { + const win = BrowserWindow.getAllWindows()[0] + if (mode === 'minimized') win.restore() + else win.showInactive() + }, mode) + for (let attempt = 0; attempt < 2; attempt++) { + const response = await execute('browser_screenshot', {}) + expect(response.ok, response.error).toBe(true) + } + expect(await state()).toMatchObject({ id: before.id, value: 'Unsaved work' }) + }) + } + for (const mode of ['visible', 'hidden', 'minimized']) { test(`captures a ${mode} window without changing its state`, async () => { test.skip( @@ -383,7 +563,8 @@ test.describe('browser tools', () => { .find((wc) => wc.getURL() === `${origin}/form`) if (!contents) throw new Error('Missing screenshot fixture') await contents.executeJavaScript( - `document.body.style.background = ${JSON.stringify(color)}; void 0` + `document.body.style.background = ${JSON.stringify(color)}; + new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))` ) }, { origin, color } diff --git a/apps/desktop/src/main/browser-agent/cdp.test.ts b/apps/desktop/src/main/browser-agent/cdp.test.ts index 9f7b32b6071..fdc0884e3f7 100644 --- a/apps/desktop/src/main/browser-agent/cdp.test.ts +++ b/apps/desktop/src/main/browser-agent/cdp.test.ts @@ -1,8 +1,14 @@ -import { describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('electron', () => import('@/test/electron-mock')) -import { type nativeImage, WebContentsView, type WebFrameMain } from 'electron' +import { + type NativeImage, + type nativeImage, + type WebContents, + WebContentsView, + type WebFrameMain, +} from 'electron' import { captureScreenshot, clickAt, @@ -491,7 +497,10 @@ describe('browser-agent CDP theme', () => { * snapping back. Resolution is bounded on the returned image instead. */ describe('browser-agent screenshot capture', () => { - function captureFixture(imageSize: { width: number; height: number } | null) { + function captureFixture( + imageSize: { width: number; height: number } | null, + imageContent = 'sim' + ) { const contents = new WebContentsView().webContents vi.mocked(contents.debugger.sendCommand).mockImplementation((method: string) => { if (method === 'Page.getLayoutMetrics') { @@ -513,7 +522,7 @@ describe('browser-agent screenshot capture', () => { getSize: vi.fn(() => imageSize ?? { width: 0, height: 0 }), crop: vi.fn(() => cropped), resize: vi.fn(() => resized), - toJPEG: vi.fn(() => Buffer.from('sim')), + toJPEG: vi.fn(() => Buffer.from(imageContent)), } as unknown as ReturnType vi.mocked(contents.capturePage).mockResolvedValue(image) return { contents, resized, cropped, image } @@ -548,9 +557,54 @@ describe('browser-agent screenshot capture', () => { scale: 2, viewport: { width: 2048, height: 1024 }, imageSize: { width: 400, height: 200 }, + clip: { x: 100, y: 50, width: 200, height: 100 }, }) }) + it('reports the actual CSS crop after rounding a narrow fractional element to pixels', async () => { + const { contents, cropped, image } = captureFixture({ width: 4096, height: 2048 }) + cropped.getSize.mockReturnValue({ width: 3, height: 201 }) + + const shot = await captureScreenshot(contents, { x: 0.1, y: 0.2, width: 1.1, height: 100 }) + + expect(image.crop).toHaveBeenCalledWith({ x: 0, y: 0, width: 3, height: 201 }) + expect(shot).toMatchObject({ + clip: { x: 0, y: 0, width: 1.5, height: 100.5 }, + imageSize: { width: 3, height: 201 }, + scale: 2, + }) + expect(100 / shot.scale).toBe(50) + expect(cropped.resize).not.toHaveBeenCalled() + }) + + it.each([ + { + requested: { x: -10, y: -20, width: 30, height: 40 }, + crop: { x: 0, y: 0, width: 40, height: 40 }, + captured: { x: 0, y: 0, width: 20, height: 20 }, + }, + { + requested: { x: 2040, y: 1020, width: 30, height: 40 }, + crop: { x: 4080, y: 2040, width: 16, height: 8 }, + captured: { x: 2040, y: 1020, width: 8, height: 4 }, + }, + ])( + 'reports only the encoded portion of a crop clamped to the viewport: $requested', + async ({ requested, crop, captured }) => { + const { contents, cropped, image } = captureFixture({ width: 4096, height: 2048 }) + cropped.getSize.mockReturnValue({ width: crop.width, height: crop.height }) + + const shot = await captureScreenshot(contents, requested) + + expect(image.crop).toHaveBeenCalledWith(crop) + expect(shot).toMatchObject({ + clip: captured, + imageSize: { width: crop.width, height: crop.height }, + scale: 2, + }) + } + ) + /** * A 2048px CSS viewport bounded to 1024px is scale 0.5, and the capture * arrives at device resolution (4096px on a 2x display). The resize is what @@ -591,34 +645,217 @@ describe('browser-agent screenshot capture', () => { await expect(captureScreenshot(contents)).rejects.toThrow('empty image') }) - it('bounds a stalled capture and prevents overlapping native surface copies', async () => { - vi.useFakeTimers() - try { + describe('stalled native capture recovery', () => { + beforeEach(() => vi.useFakeTimers()) + afterEach(() => vi.useRealTimers()) + + function observeFrames(contents: WebContents) { + const frames: Array<(image: NativeImage) => void> = [] + vi.mocked(contents.beginFrameSubscription).mockImplementation((...args: unknown[]) => { + const callback = args.at(-1) as (image: NativeImage) => void + frames.push((image) => callback(image)) + }) + return frames + } + + it('recovers repeatedly with fresh frames without overlapping native surface copies', async () => { + const { contents } = captureFixture({ width: 1024, height: 512 }) + vi.mocked(contents.capturePage).mockReturnValue(new Promise(() => {})) + const frames = observeFrames(contents) + + for (let index = 0; index < 5; index++) { + const { image } = captureFixture({ width: 1024, height: 512 }, `frame-${index}`) + const capture = captureScreenshot(contents) + await vi.advanceTimersByTimeAsync(index === 0 ? 5_000 : 0) + expect(contents.capturePage).toHaveBeenCalledOnce() + expect(contents.beginFrameSubscription).toHaveBeenLastCalledWith( + false, + expect.any(Function) + ) + expect(frames).toHaveLength(index + 1) + frames[index](image) + await expect(capture).resolves.toMatchObject({ + dataUrl: `data:image/jpeg;base64,${Buffer.from(`frame-${index}`).toString('base64')}`, + imageSize: { width: 1024, height: 512 }, + }) + expect(contents.endFrameSubscription).toHaveBeenCalledTimes(index + 1) + expect(vi.getTimerCount()).toBe(0) + } + const registered = vi + .mocked(contents.once) + .mock.calls.filter(([event]) => String(event) === 'destroyed') + for (const [, listener] of registered) { + expect(contents.removeListener).toHaveBeenCalledWith('destroyed', listener) + } + expect(contents.reload).not.toHaveBeenCalled() + expect(contents.loadURL).not.toHaveBeenCalled() + }) + + it('bounds both waits and allows another frame attempt after a timeout', async () => { const { contents, image } = captureFixture({ width: 1024, height: 512 }) - let release: (captured: typeof image) => void = () => {} - vi.mocked(contents.capturePage).mockImplementationOnce( - () => - new Promise((resolve) => { - release = resolve - }) - ) - const failed = expect(captureScreenshot(contents)).rejects.toThrow('pixel capture timed out') - await vi.advanceTimersByTimeAsync(5_000) + vi.mocked(contents.capturePage).mockReturnValue(new Promise(() => {})) + const frames = observeFrames(contents) + const failed = expect(captureScreenshot(contents)).rejects.toThrow('frame capture timed out') + await vi.advanceTimersByTimeAsync(9_999) + expect(contents.endFrameSubscription).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) await failed + expect(contents.endFrameSubscription).toHaveBeenCalledOnce() expect(vi.getTimerCount()).toBe(0) - await expect(captureScreenshot(contents)).rejects.toThrow( - 'previous screenshot capture is still pending' - ) + + const recovered = captureScreenshot(contents) + await vi.advanceTimersByTimeAsync(0) expect(contents.capturePage).toHaveBeenCalledOnce() - release(image) - await Promise.resolve() - await expect(captureScreenshot(contents)).resolves.toMatchObject({ + frames[1](image) + await expect(recovered).resolves.toMatchObject({ imageSize: { width: 1024, height: 512 } }) + expect(contents.endFrameSubscription).toHaveBeenCalledTimes(2) + expect(vi.getTimerCount()).toBe(0) + }) + + it('ignores a timed-out frame callback while a later subscription is active', async () => { + const { contents, image } = captureFixture({ width: 1024, height: 512 }, 'fresh') + const stale = captureFixture({ width: 1024, height: 512 }, 'stale').image + vi.mocked(contents.capturePage).mockReturnValue(new Promise(() => {})) + const frames = observeFrames(contents) + const failed = expect(captureScreenshot(contents)).rejects.toThrow('frame capture timed out') + await vi.advanceTimersByTimeAsync(10_000) + await failed + + const recovered = captureScreenshot(contents) + const settled = vi.fn() + void recovered.then(settled) + await vi.advanceTimersByTimeAsync(0) + frames[0](stale) + await vi.advanceTimersByTimeAsync(0) + expect(settled).not.toHaveBeenCalled() + expect(contents.endFrameSubscription).toHaveBeenCalledOnce() + frames[1](image) + await expect(recovered).resolves.toMatchObject({ + dataUrl: `data:image/jpeg;base64,${Buffer.from('fresh').toString('base64')}`, + }) + expect(contents.endFrameSubscription).toHaveBeenCalledTimes(2) + }) + + it.each(['resolve', 'reject'] as const)( + 'ignores a late native %s and resumes native captures afterward', + async (outcome) => { + const { contents, image } = captureFixture({ width: 1024, height: 512 }, 'current') + const stale = captureFixture({ width: 1024, height: 512 }, 'stale').image + let settleNative: () => void = () => {} + vi.mocked(contents.capturePage).mockImplementationOnce( + () => + new Promise((resolve, reject) => { + settleNative = () => + outcome === 'resolve' ? resolve(stale) : reject(new Error('late failure')) + }) + ) + const frames = observeFrames(contents) + const capture = captureScreenshot(contents) + const settled = vi.fn() + void capture.then(settled) + await vi.advanceTimersByTimeAsync(5_000) + settleNative() + await vi.advanceTimersByTimeAsync(0) + expect(settled).not.toHaveBeenCalled() + expect(contents.endFrameSubscription).not.toHaveBeenCalled() + frames[0](image) + await expect(capture).resolves.toMatchObject({ + dataUrl: `data:image/jpeg;base64,${Buffer.from('current').toString('base64')}`, + }) + await expect(captureScreenshot(contents)).resolves.toMatchObject({ + dataUrl: `data:image/jpeg;base64,${Buffer.from('current').toString('base64')}`, + }) + expect(contents.capturePage).toHaveBeenCalledTimes(2) + expect(contents.beginFrameSubscription).toHaveBeenCalledOnce() + expect(vi.getTimerCount()).toBe(0) + } + ) + + it('rejects concurrent captures without replacing the active subscription or blocking another tab', async () => { + const { contents, image } = captureFixture({ width: 1024, height: 512 }) + const other = captureFixture({ width: 1024, height: 512 }) + vi.mocked(contents.capturePage).mockReturnValue(new Promise(() => {})) + const frames = observeFrames(contents) + const capture = captureScreenshot(contents) + await vi.advanceTimersByTimeAsync(0) + await expect(captureScreenshot(contents)).rejects.toThrow('already in progress') + expect(contents.capturePage).toHaveBeenCalledOnce() + await vi.advanceTimersByTimeAsync(5_000) + await expect(captureScreenshot(contents)).rejects.toThrow('already in progress') + expect(contents.beginFrameSubscription).toHaveBeenCalledOnce() + expect(contents.endFrameSubscription).not.toHaveBeenCalled() + await expect(captureScreenshot(other.contents)).resolves.toMatchObject({ imageSize: { width: 1024, height: 512 }, }) - expect(contents.capturePage).toHaveBeenCalledTimes(2) - } finally { - vi.useRealTimers() - } + frames[0](image) + await capture + expect(contents.endFrameSubscription).toHaveBeenCalledOnce() + }) + + it.each(['cancel', 'destroy'] as const)( + 'releases frame resources on %s and ignores a subsequent frame', + async (reason) => { + const { contents, image } = captureFixture({ width: 1024, height: 512 }) + vi.mocked(contents.capturePage).mockReturnValue(new Promise(() => {})) + const frames = observeFrames(contents) + const controller = new AbortController() + const removeAbort = vi.spyOn(controller.signal, 'removeEventListener') + const failed = expect( + captureScreenshot(contents, undefined, controller.signal) + ).rejects.toThrow(reason === 'cancel' ? 'cancelled' : 'tab was closed') + await vi.advanceTimersByTimeAsync(5_000) + const destroyed = vi + .mocked(contents.once) + .mock.calls.filter(([event]) => String(event) === 'destroyed') + .at(-1)?.[1] as unknown as (() => void) | undefined + expect(destroyed).toBeDefined() + if (reason === 'cancel') controller.abort() + else { + vi.mocked(contents.isDestroyed).mockReturnValue(true) + destroyed?.() + } + await failed + expect(contents.removeListener).toHaveBeenCalledWith('destroyed', destroyed) + expect(removeAbort).toHaveBeenCalledTimes(2) + expect(contents.endFrameSubscription).toHaveBeenCalledTimes(reason === 'cancel' ? 1 : 0) + frames[0](image) + await vi.advanceTimersByTimeAsync(0) + expect(contents.endFrameSubscription).toHaveBeenCalledTimes(reason === 'cancel' ? 1 : 0) + expect(vi.getTimerCount()).toBe(0) + if (reason === 'cancel') { + const recovered = captureScreenshot(contents) + await vi.advanceTimersByTimeAsync(0) + frames[1](image) + await recovered + expect(contents.capturePage).toHaveBeenCalledOnce() + expect(contents.endFrameSubscription).toHaveBeenCalledTimes(2) + } + } + ) + + it.each(['beginFrameSubscription', 'invalidate'] as const)( + 'cleans up a synchronous %s failure and permits another frame attempt', + async (method) => { + const { contents, image } = captureFixture({ width: 1024, height: 512 }) + vi.mocked(contents.capturePage).mockReturnValue(new Promise(() => {})) + const frames = observeFrames(contents) + vi.mocked(contents[method]).mockImplementationOnce(() => { + throw new Error('frame setup failed') + }) + const failed = expect(captureScreenshot(contents)).rejects.toThrow('frame setup failed') + await vi.advanceTimersByTimeAsync(5_000) + await failed + expect(contents.endFrameSubscription).toHaveBeenCalledOnce() + expect(vi.getTimerCount()).toBe(0) + + const recovered = captureScreenshot(contents) + await vi.advanceTimersByTimeAsync(0) + frames.at(-1)?.(image) + await expect(recovered).resolves.toMatchObject({ imageSize: { width: 1024, height: 512 } }) + expect(contents.capturePage).toHaveBeenCalledOnce() + expect(contents.endFrameSubscription).toHaveBeenCalledTimes(2) + } + ) }) it.each(['cancel', 'destroy'] as const)( diff --git a/apps/desktop/src/main/browser-agent/cdp.ts b/apps/desktop/src/main/browser-agent/cdp.ts index 7c30dc6bd07..cef27dbd389 100644 --- a/apps/desktop/src/main/browser-agent/cdp.ts +++ b/apps/desktop/src/main/browser-agent/cdp.ts @@ -374,6 +374,9 @@ const UNSCALED_SCREENSHOT_QUALITY = 90 const SCREENSHOT_CAPTURE_TIMEOUT_MS = 5_000 /** Native surface copies cannot be cancelled; never accumulate them on a stalled tab. */ const pendingScreenshotCaptures = new WeakSet() +const activeScreenshotCaptures = new WeakSet() + +class ScreenshotCaptureTimeoutError extends Error {} interface CdpViewport { clientWidth: number @@ -398,6 +401,7 @@ export interface ScreenshotCapture { scale: number viewport: ScreenshotSize | null imageSize: ScreenshotSize + clip?: ScreenshotClip } export interface ScreenshotClip { @@ -452,16 +456,12 @@ function sameScreenshotViewport( ) } -/** Captures pixels without changing viewport geometry or exposing a hidden window. */ -async function captureViewportImage( +async function captureNativeViewportImage( contents: WebContents, signal?: AbortSignal ): Promise { signal?.throwIfAborted() if (contents.isDestroyed()) throw new Error('The screenshot tab was closed') - if (pendingScreenshotCaptures.has(contents)) { - throw new Error('A previous screenshot capture is still pending on this tab') - } pendingScreenshotCaptures.add(contents) let timer: ReturnType | undefined let onAbort = () => {} @@ -473,7 +473,10 @@ async function captureViewportImage( signal?.addEventListener('abort', onAbort, { once: true }) contents.once('destroyed', onDestroyed) timer = setTimeout( - () => reject(new Error('Screenshot pixel capture timed out after 5 seconds')), + () => + reject( + new ScreenshotCaptureTimeoutError('Screenshot pixel capture timed out after 5 seconds') + ), SCREENSHOT_CAPTURE_TIMEOUT_MS ) }) @@ -492,6 +495,64 @@ async function captureViewportImage( } } +/** Observes one complete frame; unlike a native surface copy, this wait can be cancelled. */ +async function captureViewportFrame( + contents: WebContents, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + if (contents.isDestroyed()) throw new Error('The screenshot tab was closed') + let timer: ReturnType | undefined + let onAbort = () => {} + let onDestroyed = () => {} + let subscribed = false + try { + return await new Promise((resolve, reject) => { + onAbort = () => reject(new Error('Screenshot capture was cancelled')) + onDestroyed = () => reject(new Error('The screenshot tab was closed')) + signal?.addEventListener('abort', onAbort, { once: true }) + contents.once('destroyed', onDestroyed) + timer = setTimeout( + () => reject(new Error('Screenshot frame capture timed out after 5 seconds')), + SCREENSHOT_CAPTURE_TIMEOUT_MS + ) + subscribed = true + contents.beginFrameSubscription(false, (image) => resolve(image)) + contents.invalidate() + }) + } finally { + clearTimeout(timer) + signal?.removeEventListener('abort', onAbort) + contents.removeListener('destroyed', onDestroyed) + if (subscribed && !contents.isDestroyed()) contents.endFrameSubscription() + } +} + +/** Captures pixels without reloading the page, changing geometry, or exposing a hidden window. */ +async function captureViewportImage( + contents: WebContents, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + if (contents.isDestroyed()) throw new Error('The screenshot tab was closed') + if (activeScreenshotCaptures.has(contents)) { + throw new Error('A screenshot capture is already in progress on this tab') + } + activeScreenshotCaptures.add(contents) + try { + if (!pendingScreenshotCaptures.has(contents)) { + try { + return await captureNativeViewportImage(contents, signal) + } catch (error) { + if (!(error instanceof ScreenshotCaptureTimeoutError)) throw error + } + } + return await captureViewportFrame(contents, signal) + } finally { + activeScreenshotCaptures.delete(contents) + } +} + /** * Native viewport capture, bounded in time and resolution. * @@ -504,8 +565,9 @@ async function captureViewportImage( * snapshot capture refuses to scale a visible surface for the same reason. * * Bounding resolution therefore happens here instead, on the returned image. - * Optional element crops also happen in memory. Convert output coordinates - * with cssX = (clip?.x ?? 0) + imageX / scale, and the equivalent Y formula. + * Optional element crops also happen in memory. The returned clip records the + * rounded/clamped CSS bounds. Map each image axis using those bounds and the + * returned imageSize, since resizing can round the two dimensions differently. */ export async function captureScreenshot( contents: WebContents, @@ -564,6 +626,12 @@ export async function captureScreenshot( if (croppedSize.width === 0 || croppedSize.height === 0) { throw new Error('The requested screenshot element produced an empty crop') } + const capturedClip = { + x: cropX / xScale, + y: cropY / yScale, + width: croppedSize.width / xScale, + height: croppedSize.height / yScale, + } const cropScale = Math.min( 1, MAX_SCREENSHOT_EDGE / Math.max(croppedSize.width, croppedSize.height) @@ -579,9 +647,10 @@ export async function captureScreenshot( const outputSize = output.getSize() return { dataUrl: `data:image/jpeg;base64,${output.toJPEG(SCREENSHOT_QUALITY).toString('base64')}`, - scale: outputSize.width / clip.width, + scale: outputSize.width / capturedClip.width, viewport: cssViewport, imageSize: outputSize, + clip: capturedClip, } } if (size.width === targetWidth && size.height === targetHeight) { diff --git a/apps/desktop/src/main/browser-agent/driver.test.ts b/apps/desktop/src/main/browser-agent/driver.test.ts index 285112fb61c..94ef24f1bba 100644 --- a/apps/desktop/src/main/browser-agent/driver.test.ts +++ b/apps/desktop/src/main/browser-agent/driver.test.ts @@ -3507,7 +3507,7 @@ describe('credential protection', () => { const result = await driver.executeTool('chat-test', 'browser_click_at', { x: 9999, y: 5 }) expect(result.ok).toBe(false) - expect(result.error).toMatch(/divide image pixels by its scale/) + expect(result.error).toMatch(/X\/Y coordinate mapping and crop origin/) }) it('inserts text into the focused editable at the caret', async () => { @@ -4121,6 +4121,44 @@ describe('credential protection', () => { } }) + it('returns the encoded crop geometry while checking the original element bounds for movement', async () => { + const contents = await openPage() + const measuredClip = { x: 0.1, y: 0.2, width: 1.1, height: 100 } + const capturedClip = { x: 0, y: 0, width: 1.5, height: 100.5 } + respondWith(contents, { + getElementScreenshotRect: { ...measuredClip, element: 'div', refRecovered: false }, + }) + const capture = vi.spyOn(cdp, 'captureScreenshot').mockResolvedValue({ + dataUrl: 'data:image/jpeg;base64,c2lt', + scale: 2, + viewport: { width: 800, height: 600 }, + imageSize: { width: 3, height: 201 }, + clip: capturedClip, + }) + + try { + const result = await driver.executeTool('chat-test', 'browser_screenshot', { elementId: 0 }) + + expect(capture).toHaveBeenCalledWith(contents, measuredClip, expect.any(AbortSignal)) + expect(result).toMatchObject({ + ok: true, + result: { + element: 'div', + clip: capturedClip, + scale: 2, + imageSize: { width: 3, height: 201 }, + }, + }) + expect( + vi + .mocked(contents.executeJavaScript) + .mock.calls.filter(([expression]) => isPageCall(expression, 'getElementScreenshotRect')) + ).toHaveLength(2) + } finally { + capture.mockRestore() + } + }) + it('rejects navigation during an element screenshot measurement', async () => { const contents = await openPage() vi.mocked(contents.executeJavaScript).mockImplementation(async (expression: string) => { @@ -4171,6 +4209,7 @@ describe('credential protection', () => { ok: true, result: { scale: 0.5, + imageSize: { width: 1024, height: 512 }, viewport: { url: 'https://example.com/login', title: 'Example', diff --git a/apps/desktop/src/main/browser-agent/driver.ts b/apps/desktop/src/main/browser-agent/driver.ts index 94b53b9db52..f593a1884ed 100644 --- a/apps/desktop/src/main/browser-agent/driver.ts +++ b/apps/desktop/src/main/browser-agent/driver.ts @@ -1237,7 +1237,7 @@ function unwrapPageResult(result: unknown): unknown { } if (code === 'outside-viewport') { throw new ToolError( - 'That point is outside the visible viewport. Coordinates are CSS pixels within the current viewport — when reading them off a browser_screenshot, divide image pixels by its scale, and scroll the target into view first.' + "That point is outside the visible viewport. Coordinates are CSS pixels within the current viewport — when reading them off a browser_screenshot, follow its caption's X/Y coordinate mapping and crop origin, and scroll the target into view first." ) } if (code === 'ambiguous-editable') { @@ -2714,13 +2714,14 @@ async function executeToolInner( } return { dataUrl: shot.dataUrl, + imageSize: shot.imageSize, viewport, scale, ...(clip ? { element: elementClip?.element, refRecovered: elementClip?.refRecovered === true, - clip, + clip: shot.clip ?? clip, } : {}), } @@ -4218,7 +4219,7 @@ async function executeToolInner( ) if (!isRecordLike(pointTarget) || pointTarget.found !== true) { throw new ToolError( - 'Nothing is rendered at that point. Coordinates are CSS pixels in the current viewport — when reading them off a browser_screenshot, divide image pixels by its scale.' + "Nothing is rendered at that point. Coordinates are CSS pixels in the current viewport — when reading them off a browser_screenshot, follow its caption's X/Y coordinate mapping and crop origin." ) } if (pointTarget.fileInput === true) { @@ -4462,7 +4463,7 @@ async function executeToolInner( ) if (!isRecordLike(probe) || probe.found !== true) { throw new ToolError( - `Nothing is rendered at the ${which} point. Coordinates are CSS pixels in the current viewport — when reading them off a browser_screenshot, divide image pixels by its scale.` + `Nothing is rendered at the ${which} point. Coordinates are CSS pixels in the current viewport — when reading them off a browser_screenshot, follow its caption's X/Y coordinate mapping and crop origin.` ) } return { diff --git a/apps/desktop/src/test/electron-mock.ts b/apps/desktop/src/test/electron-mock.ts index fea9849d085..bd5dbc9918f 100644 --- a/apps/desktop/src/test/electron-mock.ts +++ b/apps/desktop/src/test/electron-mock.ts @@ -173,6 +173,8 @@ function createWebContentsMock() { print: vi.fn(), focus: vi.fn(), invalidate: vi.fn(), + beginFrameSubscription: vi.fn(), + endFrameSubscription: vi.fn(), isFocused: vi.fn(() => false), close: vi.fn(), isDestroyed: vi.fn(() => false), diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index d82e0066cb7..360c3710983 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -500,12 +500,12 @@ export const BrowserClickAt: ToolCatalogEntry = { x: { type: 'number', description: - 'X in CSS pixels within the current viewport. When read off a browser_screenshot, divide the image pixel value by scale and add clip.x when present.', + "X in CSS pixels within the current viewport. When read off a browser_screenshot, follow its caption's X mapping and crop origin.", }, y: { type: 'number', description: - 'Y in CSS pixels within the current viewport, converted from screenshot pixels the same way as x.', + "Y in CSS pixels within the current viewport. When read off a browser_screenshot, follow its caption's Y mapping and crop origin.", }, }, required: ['x', 'y'], @@ -1519,7 +1519,7 @@ export const BrowserScreenshot: ToolCatalogEntry = { elementId: { type: 'number', description: - "Optional element id from the current tab's latest browser_snapshot. When present, capture only the visible portion of that top-page element without scrolling or changing layout. Scroll explicitly first if needed. Framed elements are rejected; use a viewport screenshot for them. Use the returned clip offset when converting image coordinates.", + "Optional element id from the current tab's latest browser_snapshot. When present, capture only the visible portion of that top-page element without scrolling or changing layout. Scroll explicitly first if needed. Framed elements are rejected; use a viewport screenshot for them. Follow the image caption's coordinate mapping, including its crop origin.", }, }, }, diff --git a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts index a1b5c10ecb9..31b27d61213 100644 --- a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts @@ -225,12 +225,12 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { x: { type: 'number', description: - 'X in CSS pixels within the current viewport. When read off a browser_screenshot, divide the image pixel value by scale and add clip.x when present.', + "X in CSS pixels within the current viewport. When read off a browser_screenshot, follow its caption's X mapping and crop origin.", }, y: { type: 'number', description: - 'Y in CSS pixels within the current viewport, converted from screenshot pixels the same way as x.', + "Y in CSS pixels within the current viewport. When read off a browser_screenshot, follow its caption's Y mapping and crop origin.", }, }, required: ['x', 'y'], @@ -1430,7 +1430,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { elementId: { type: 'number', description: - "Optional element id from the current tab's latest browser_snapshot. When present, capture only the visible portion of that top-page element without scrolling or changing layout. Scroll explicitly first if needed. Framed elements are rejected; use a viewport screenshot for them. Use the returned clip offset when converting image coordinates.", + "Optional element id from the current tab's latest browser_snapshot. When present, capture only the visible portion of that top-page element without scrolling or changing layout. Scroll explicitly first if needed. Framed elements are rejected; use a viewport screenshot for them. Follow the image caption's coordinate mapping, including its crop origin.", }, }, }, diff --git a/apps/sim/lib/copilot/tools/client/browser-tool-execution.test.ts b/apps/sim/lib/copilot/tools/client/browser-tool-execution.test.ts index 1fdb9165092..a4ee6e8a4f3 100644 --- a/apps/sim/lib/copilot/tools/client/browser-tool-execution.test.ts +++ b/apps/sim/lib/copilot/tools/client/browser-tool-execution.test.ts @@ -1215,6 +1215,8 @@ describe('executeBrowserToolOnClient', () => { it('reshapes a screenshot into an image attachment the model can see', async () => { mockExecuteBrowserTool.mockResolvedValue({ dataUrl: 'data:image/jpeg;base64,/9j/4AAQ', + imageSize: { width: 512, height: 320 }, + scale: 0.5, viewport: { url: 'https://example.com/pricing', title: 'Pricing', @@ -1233,6 +1235,9 @@ describe('executeBrowserToolOnClient', () => { source: { type: 'base64', media_type: 'image/jpeg', data: '/9j/4AAQ' }, }) expect(reported.content).toContain('https://example.com/pricing') + expect(reported.content).toContain('Viewport: 1024 × 640 CSS pixels') + expect(reported.content).toContain('Encoded image: 512 × 320 pixels') + expect(reported.content).toContain('cssX = 0 + imageX / 0.5; cssY = 0 + imageY / 0.5') expect(reported.dataUrl).toBeUndefined() expect(reported.viewport).toMatchObject({ width: 1024, height: 640 }) }) @@ -1253,6 +1258,7 @@ describe('executeBrowserToolOnClient', () => { mockExecuteBrowserTool.mockResolvedValue({ dataUrl: 'data:image/jpeg;base64,/9j/4AAQ', clip: { x: 20, y: 30, width: 200, height: 100 }, + imageSize: { width: 400, height: 200 }, scale: 2, }) executeBrowserToolOnClient(nextToolCallId(), 'browser_screenshot', { elementId: 0 }) @@ -1261,8 +1267,8 @@ describe('executeBrowserToolOnClient', () => { const reported = mockReportCompletion.mock.calls[0][3] expect(reported.clip).toEqual({ x: 20, y: 30, width: 200, height: 100 }) expect(reported.scale).toBe(2) - expect(reported.content).toContain('cssX = clip.x + imageX / scale') - expect(reported.content).toContain('cssY = clip.y + imageY / scale') + expect(reported.content).toContain('cssX = 20 + imageX / 2') + expect(reported.content).toContain('cssY = 30 + imageY / 2') }) it('gives restored-tab switching the renderer navigation budget', async () => { diff --git a/apps/sim/lib/copilot/tools/client/browser-tool-execution.ts b/apps/sim/lib/copilot/tools/client/browser-tool-execution.ts index ced31164d4f..d09705eec7d 100644 --- a/apps/sim/lib/copilot/tools/client/browser-tool-execution.ts +++ b/apps/sim/lib/copilot/tools/client/browser-tool-execution.ts @@ -24,6 +24,7 @@ import { } from '@/lib/copilot/async-runs/lifecycle' import { COPILOT_CONFIRM_API_PATH } from '@/lib/copilot/constants' import { BrowserToolReplayLedger } from '@/lib/copilot/tools/client/browser-tool-replay-ledger' +import { sanitizeBrowserToolResultForModel } from '@/lib/copilot/tools/client/browser-tool-result' import { reportClientToolCompletion, reportClientToolCompletionOnPageExit, @@ -545,59 +546,6 @@ function timeoutForTool(toolName: BrowserToolName, params: Record;base64,` URL into its parts. */ -function parseBase64DataUrl(dataUrl: string): { mediaType: string; data: string } | null { - const match = /^data:([^;,]+);base64,(.+)$/s.exec(dataUrl) - if (!match) return null - return { mediaType: match[1], data: match[2] } -} - -/** - * Reshapes a screenshot into the `attachment` contract the copilot serializes - * into a real image content block, so the model sees the page rather than a - * note about it. The data URL itself never goes inline: `content` is the text - * the model reads beside the image, and the bytes travel under `attachment`. - * - * A malformed data URL degrades to the text note rather than shipping an - * attachment the provider would reject. - */ -function sanitizeResultForModel( - toolName: BrowserToolName, - result: unknown -): Record | undefined { - if (!isRecordLike(result)) { - return result === undefined ? undefined : { value: result } - } - if (toolName === 'browser_screenshot' && typeof result.dataUrl === 'string') { - const { dataUrl, ...rest } = result - const image = parseBase64DataUrl(dataUrl) - if (!image) { - return { - ...rest, - note: 'The screenshot could not be encoded. Use browser_snapshot or browser_read_text instead.', - } - } - const viewport = isRecordLike(rest.viewport) ? rest.viewport : null - const screenshotUrl = - typeof rest.url === 'string' && rest.url - ? rest.url - : viewport && typeof viewport.url === 'string' - ? viewport.url - : '' - const location = screenshotUrl ? ` of ${screenshotUrl}` : '' - const isElementCapture = isRecordLike(rest.clip) - return { - ...rest, - content: `Screenshot${location}. This is the rendered ${isElementCapture ? 'element' : 'viewport'} only — it carries no element ids, so use browser_snapshot before interacting.${isElementCapture ? ' For coordinate actions: cssX = clip.x + imageX / scale; cssY = clip.y + imageY / scale.' : ''}`, - attachment: { - type: 'image', - source: { type: 'base64', media_type: image.mediaType, data: image.data }, - }, - } - } - return result -} - /** * Fire-and-forget entry point invoked by the stream tool-event handler when a * `browser_*` client tool call arrives. @@ -997,7 +945,7 @@ async function doExecuteBrowserTool( : effectUnconfirmed ? 'Browser input completed; its effect is unconfirmed. Inspect the current state before retrying.' : 'Browser action completed', - data: sanitizeResultForModel(toolName, result), + data: sanitizeBrowserToolResultForModel(toolName, result), }, 'Failed to report successful browser tool completion' ) diff --git a/apps/sim/lib/copilot/tools/client/browser-tool-result.test.ts b/apps/sim/lib/copilot/tools/client/browser-tool-result.test.ts new file mode 100644 index 00000000000..fee5723b4e0 --- /dev/null +++ b/apps/sim/lib/copilot/tools/client/browser-tool-result.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from 'vitest' +import { sanitizeBrowserToolResultForModel } from '@/lib/copilot/tools/client/browser-tool-result' + +describe('browser screenshot model projection', () => { + it('keeps an image usable when an older desktop omits coordinate metadata', () => { + const result = sanitizeBrowserToolResultForModel('browser_screenshot', { + dataUrl: 'data:image/jpeg;base64,/9j/4AAQ', + }) + expect(result?.attachment).toBeDefined() + expect(result?.content).toContain('coordinate mapping is unavailable') + expect(result?.content).not.toContain('cssX =') + }) + + it.each([0, -1, Number.NaN, Number.POSITIVE_INFINITY, '0.5'])( + 'does not publish an invalid scale %s', + (scale) => { + const result = sanitizeBrowserToolResultForModel('browser_screenshot', { + dataUrl: 'data:image/jpeg;base64,/9j/4AAQ', + scale, + viewport: { width: 1600, height: 900 }, + }) + expect(result?.content).toContain('Viewport: 1600 × 900 CSS pixels') + expect(result?.content).toContain('coordinate mapping is unavailable') + expect(result?.content).not.toContain('cssX =') + } + ) + + it('does not invent a crop origin or encode malformed dimensions into the caption', () => { + const result = sanitizeBrowserToolResultForModel('browser_screenshot', { + dataUrl: 'data:image/jpeg;base64,/9j/4AAQ', + scale: 2, + clip: { x: '10', y: 20 }, + viewport: { width: 0, height: 900 }, + imageSize: { width: Number.POSITIVE_INFINITY, height: 640 }, + }) + expect(result?.attachment).toBeDefined() + expect(result?.content).toContain('coordinate mapping is unavailable') + expect(result?.content).not.toMatch(/Viewport:|Encoded image:|Crop origin:|cssX =/) + }) + + it('maps each crop axis independently when pixel rounding changes its aspect ratio', () => { + const result = sanitizeBrowserToolResultForModel('browser_screenshot', { + dataUrl: 'data:image/jpeg;base64,/9j/4AAQ', + scale: 2.5, + clip: { x: 0, y: 20, width: 1.5, height: 100 }, + imageSize: { width: 3, height: 200 }, + }) + expect(result?.content).toContain('Crop origin: (0, 20) in viewport CSS pixels') + expect(result?.content).toContain('cssX = 0 + imageX / 2; cssY = 20 + imageY / 2') + }) + + it('keeps a legacy crop image without publishing its unverified scalar mapping', () => { + const result = sanitizeBrowserToolResultForModel('browser_screenshot', { + dataUrl: 'data:image/jpeg;base64,/9j/4AAQ', + scale: 3 / 1.1, + clip: { x: 0.1, y: 20, width: 1.1, height: 100 }, + }) + expect(result?.attachment).toBeDefined() + expect(result?.content).toContain('coordinate mapping is unavailable') + expect(result?.content).not.toContain('cssX =') + }) + + it('uses both encoded dimensions for a resized viewport', () => { + const result = sanitizeBrowserToolResultForModel('browser_screenshot', { + dataUrl: 'data:image/jpeg;base64,/9j/4AAQ', + scale: 0.5, + viewport: { width: 1600, height: 901 }, + imageSize: { width: 800, height: 451 }, + }) + expect(result?.content).toContain(`cssX = 0 + imageX / 0.5; cssY = 0 + imageY / ${451 / 901}`) + }) + + it('withholds a legacy viewport scalar when encoded dimensions are unavailable', () => { + const result = sanitizeBrowserToolResultForModel('browser_screenshot', { + dataUrl: 'data:image/jpeg;base64,/9j/4AAQ', + scale: 0.5, + viewport: { width: 2048, height: 1025 }, + }) + expect(result?.attachment).toBeDefined() + expect(result?.scale).toBe(0.5) + expect(result?.content).toContain('Viewport: 2048 × 1025 CSS pixels') + expect(result?.content).toContain('coordinate mapping is unavailable') + expect(result?.content).not.toContain('cssY =') + }) + + it('leaves non-image tool results unchanged', () => { + const result = { outline: 'button "Continue" [ref=3]' } + expect(sanitizeBrowserToolResultForModel('browser_snapshot', result)).toBe(result) + expect(sanitizeBrowserToolResultForModel('browser_snapshot', undefined)).toBeUndefined() + }) +}) diff --git a/apps/sim/lib/copilot/tools/client/browser-tool-result.ts b/apps/sim/lib/copilot/tools/client/browser-tool-result.ts new file mode 100644 index 00000000000..7ffafbcd616 --- /dev/null +++ b/apps/sim/lib/copilot/tools/client/browser-tool-result.ts @@ -0,0 +1,81 @@ +import type { BrowserToolName } from '@sim/browser-protocol' +import { isRecordLike } from '@sim/utils/object' + +function finiteNumber(value: unknown): value is number { + return typeof value === 'number' && Number.isFinite(value) +} + +function imageDimensions(value: unknown): { width: number; height: number } | null { + if ( + !isRecordLike(value) || + !finiteNumber(value.width) || + !finiteNumber(value.height) || + value.width <= 0 || + value.height <= 0 + ) { + return null + } + return { width: value.width, height: value.height } +} + +/** Projects image bytes and coordinate metadata into the model's image-content contract. */ +export function sanitizeBrowserToolResultForModel( + toolName: BrowserToolName, + result: unknown +): Record | undefined { + if (!isRecordLike(result)) { + return result === undefined ? undefined : { value: result } + } + if (toolName !== 'browser_screenshot' || typeof result.dataUrl !== 'string') return result + + const { dataUrl, ...rest } = result + const image = /^data:([^;,]+);base64,(.+)$/s.exec(dataUrl) + if (!image) { + return { + ...rest, + note: 'The screenshot could not be encoded. Use browser_snapshot or browser_read_text instead.', + } + } + const viewport = isRecordLike(rest.viewport) ? rest.viewport : null + const screenshotUrl = + typeof rest.url === 'string' && rest.url + ? rest.url + : viewport && typeof viewport.url === 'string' + ? viewport.url + : '' + const location = screenshotUrl ? ` of ${screenshotUrl}` : '' + const clip = isRecordLike(rest.clip) ? rest.clip : null + const cropSize = imageDimensions(clip) + const viewportSize = imageDimensions(viewport) + const imageSize = imageDimensions(rest.imageSize) + const capturedSize = clip ? cropSize : viewportSize + const scaleX = imageSize && capturedSize ? imageSize.width / capturedSize.width : null + const scaleY = imageSize && capturedSize ? imageSize.height / capturedSize.height : null + const hasScale = finiteNumber(scaleX) && scaleX > 0 && finiteNumber(scaleY) && scaleY > 0 + const origin = clip + ? finiteNumber(clip.x) && finiteNumber(clip.y) + ? { x: clip.x, y: clip.y } + : null + : { x: 0, y: 0 } + const content = [ + `Screenshot${location}. This is the rendered ${clip ? 'element' : 'viewport'} only — it carries no element ids. Use browser_snapshot for element-ref actions and the mapping below for coordinate actions.`, + viewportSize && `Viewport: ${viewportSize.width} × ${viewportSize.height} CSS pixels.`, + imageSize && `Encoded image: ${imageSize.width} × ${imageSize.height} pixels.`, + hasScale && `Image scale: X=${scaleX}, Y=${scaleY} encoded image pixels per CSS pixel.`, + cropSize && `Crop size: ${cropSize.width} × ${cropSize.height} CSS pixels.`, + clip && origin && `Crop origin: (${origin.x}, ${origin.y}) in viewport CSS pixels.`, + hasScale && origin + ? `Coordinate actions use viewport CSS pixels: cssX = ${origin.x} + imageX / ${scaleX}; cssY = ${origin.y} + imageY / ${scaleY}. imageX/imageY refer to the encoded image before any display resizing.` + : 'Screenshot coordinate mapping is unavailable; use browser_snapshot element references or take a new viewport screenshot before coordinate actions.', + ] + .filter(Boolean) + .join(' ') + return { + ...rest, + content, + attachment: { + type: 'image', + source: { type: 'base64', media_type: image[1], data: image[2] }, + }, + } +} diff --git a/scripts/check-tool-registry-boundary.baseline.json b/scripts/check-tool-registry-boundary.baseline.json index e2ca0a52ad2..9be3e4c4193 100644 --- a/scripts/check-tool-registry-boundary.baseline.json +++ b/scripts/check-tool-registry-boundary.baseline.json @@ -93,9 +93,9 @@ "gateways": {} }, "app/workspace/[workspaceId]/chat/[chatId]/page.tsx": { - "modules": 3125, + "modules": 3126, "gateways": { - "apps/sim/app/workspace/[workspaceId]/home/home.tsx": 1521, + "apps/sim/app/workspace/[workspaceId]/home/home.tsx": 1522, "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx": 961, "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/index.ts": 795, "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/index.ts": 790, @@ -183,9 +183,9 @@ "gateways": {} }, "app/workspace/[workspaceId]/home/page.tsx": { - "modules": 3125, + "modules": 3126, "gateways": { - "apps/sim/app/workspace/[workspaceId]/home/home.tsx": 1521, + "apps/sim/app/workspace/[workspaceId]/home/home.tsx": 1522, "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx": 961, "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/index.ts": 795, "apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/index.ts": 790, @@ -613,23 +613,23 @@ } }, "app/workspace/[workspaceId]/w/[workflowId]/page.tsx": { - "modules": 2168, + "modules": 2169, "gateways": { - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/workflow.tsx": 2167, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/workflow.tsx": 2168, "apps/sim/triggers/registry.ts": 522, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/index.ts": 379, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/index.ts": 380, "apps/sim/blocks/registry.ts": 350, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/index.ts": 342, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/index.ts": 343, "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/index.ts": 275, "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/index.ts": 168, "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block.tsx": 149 } }, "app/workspace/[workspaceId]/w/page.tsx": { - "modules": 2149, + "modules": 2150, "gateways": { - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/index.ts": 963, - "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/index.ts": 633, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/index.ts": 964, + "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/index.ts": 634, "apps/sim/triggers/registry.ts": 522, "apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/index.ts": 355, "apps/sim/blocks/registry.ts": 350,