Let an agent see a frame of the animation it wrote

apps/review-desktop/code-oss/src/vs/platform/native/common/native.ts

@@ -56,6 +56,13 @@ export interface INativeHostOptions {
 	readonly targetWindowId?: number;
 }
 
+export interface IScreenshotOptions {
+	/** JPEG at quality 95 unless 'png'. */
+	readonly format?: 'jpeg' | 'png';
+	/** Scales the capture down so neither edge exceeds this many pixels. */
+	readonly maxEdge?: number;
+}
+
 export interface IUninstallShellCommandOptions {
 	readonly commandName?: string;
 	readonly symlinkOnly?: boolean;
@@ -249,7 +256,7 @@ export interface ICommonNativeHostService {
 	hasWSLFeatureInstalled(): Promise<boolean>;
 
 	// Screenshots
-	getScreenshot(rect?: IRectangle): Promise<VSBuffer | undefined>;
+	getScreenshot(rect?: IRectangle, options?: INativeHostOptions & IScreenshotOptions): Promise<VSBuffer | undefined>;
 
 	// GitHub mobile upload API (runs in main process to avoid CORS)
 	uploadFileViaMobileApi(token: string, repoId: string, fileName: string, fileBytes: VSBuffer, contentType: string): Promise<{ fileName: string; assetUrl: string; contentType: string }>;

apps/review-desktop/code-oss/src/vs/platform/native/electron-main/nativeHostMainService.ts

@@ -27,7 +27,7 @@ import { IEnvironmentMainService } from '../../environment/electron-main/environ
 import { createDecorator, IInstantiationService } from '../../instantiation/common/instantiation.js';
 import { ILifecycleMainService, IRelaunchOptions } from '../../lifecycle/electron-main/lifecycleMainService.js';
 import { ILogService } from '../../log/common/log.js';
-import { FocusMode, ICommonNativeHostService, INativeHostOptions, INativeSystemWideKeybinding, INativeSystemWideKeybindingResult, IOSProperties, IOSStatistics, IStartTracingOptions, IToastOptions, IToastResult, IUninstallShellCommandOptions, PowerSaveBlockerType, SystemIdleState, ThermalState } from '../common/native.js';
+import { FocusMode, ICommonNativeHostService, INativeHostOptions, INativeSystemWideKeybinding, INativeSystemWideKeybindingResult, IOSProperties, IOSStatistics, IStartTracingOptions, IToastOptions, IToastResult, IUninstallShellCommandOptions, PowerSaveBlockerType, SystemIdleState, ThermalState, IScreenshotOptions } from '../common/native.js';
 import { IGlobalKeybindingsMainService } from '../../globalKeybindings/electron-main/globalKeybindingsMainService.js';
 import { IProductService } from '../../product/common/productService.js';
 import { IPartsSplash } from '../../theme/common/themeService.js';
@@ -910,12 +910,21 @@ export class NativeHostMainService extends Disposable implements INativeHostMain
 
 	//#region Screenshots
 
-	async getScreenshot(windowId: number | undefined, rect?: IRectangle, options?: INativeHostOptions): Promise<VSBuffer | undefined> {
+	async getScreenshot(windowId: number | undefined, rect?: IRectangle, options?: INativeHostOptions & IScreenshotOptions): Promise<VSBuffer | undefined> {
 		const window = this.windowById(options?.targetWindowId, windowId);
 		const captured = await window?.win?.webContents.capturePage(rect);
+		if (!captured || captured.isEmpty()) {
+			return undefined;
+		}
+
+		let image = captured;
+		const { width, height } = image.getSize();
+		const maxEdge = options?.maxEdge;
+		if (maxEdge && Math.max(width, height) > maxEdge) {
+			image = image.resize(width >= height ? { width: maxEdge, quality: 'best' } : { height: maxEdge, quality: 'best' });
+		}
 
-		const buf = captured?.toJPEG(95);
-		return buf && VSBuffer.wrap(buf);
+		return VSBuffer.wrap(options?.format === 'png' ? image.toPNG() : image.toJPEG(95));
 	}
 
 	//#endregion

apps/review-desktop/code-oss/src/vs/review/browser/parts/canvas/reviewCanvasPart.ts

@@ -6,7 +6,9 @@
 import { openChildWhiteboard } from "../../reviewChildWhiteboard.js";
 
 import { reviewBottomScrollPadding } from "../../reviewScrollPadding.js";
+import { getZoomFactor } from "../../../../base/browser/browser.js";
 import { $, addDisposableListener, getWindow, type Dimension } from "../../../../base/browser/dom.js";
+import { encodeBase64 } from "../../../../base/common/buffer.js";
 import { mainWindow } from "../../../../base/browser/window.js";
 import type { IHoverOptions, IHoverWidget } from "../../../../base/browser/ui/hover/hover.js";
 import { HoverPosition } from "../../../../base/browser/ui/hover/hoverWidget.js";
@@ -164,6 +166,12 @@ const APPEARANCE_SETTINGS = {
 
 const requestReviewApi: typeof fetch = (url, init) => fetch(url, init);
 
+/** The longest edge of a rendered frame's PNG: a retina capture of a column-wide block, which agents read whole. */
+const RENDER_MAX_EDGE = 1600;
+
+/** How long a render waits for a board that is still opening. */
+const RENDER_READY_WAIT_MS = 10_000;
+
 function embeddedSelectionChangeReason(event: ICursorPositionChangedEvent): EditorPaneSelectionChangeReason {
 	switch (event.source) {
 		case TextEditorSelectionSource.PROGRAMMATIC:
@@ -393,6 +401,9 @@ export class ReviewCanvasEditorPane extends EditorPane {
 				await this.hostService.focus(this.targetDocument?.defaultView ?? window, { mode: FocusMode.Force });
 				return { ok: true };
 			}
+			if (request.name === "renderAnimation") {
+				return this.renderAnimation(request.args);
+			}
 			return this.verbs.dispatch(request);
 		});
 		void this.desktopConnection.initialize().catch((error) => this.renderError(error));
@@ -1513,6 +1524,59 @@ export class ReviewCanvasEditorPane extends EditorPane {
 		};
 	}
 
+	/**
+	 * An agent's look at a frame it wrote: the canvas shows the frame and says
+	 * where its figure sits, and the window captures that rectangle as a PNG.
+	 * Only the board open in this part can be rendered; the canvas measures
+	 * CSS pixels, which the window's zoom scales to the capture's pixels.
+	 */
+	private async renderAnimation(args: { reviewId: string; targetId: string; frame: string }): Promise<ReviewVerbResponse> {
+		if (this.apiContent?.reviewId !== args.reviewId) {
+			return { ok: false, error: "The whiteboard is not open in Desktop. Open it with session_open, then render again." };
+		}
+		if (this.targetDocument !== mainWindow.document) {
+			return { ok: false, error: "The whiteboard is open in a secondary window; only the main window can render it." };
+		}
+		try {
+			const canvas = await this.whenCanvasReady(args.reviewId);
+			const shown = await canvas.renderAnimation(args.targetId, args.frame);
+			if (shown.rect.width < 1 || shown.rect.height < 1) {
+				throw new Error("The animation is outside the window.");
+			}
+			const zoom = getZoomFactor(mainWindow);
+			const png = await this.hostService.getScreenshot(
+				{
+					x: Math.round(shown.rect.x * zoom),
+					y: Math.round(shown.rect.y * zoom),
+					width: Math.round(shown.rect.width * zoom),
+					height: Math.round(shown.rect.height * zoom),
+				},
+				{ format: "png", maxEdge: RENDER_MAX_EDGE },
+			);
+			if (!png) {
+				throw new Error("Desktop could not capture its window; it must be visible on screen.");
+			}
+			return { ok: true, result: { ...shown, png: encodeBase64(png) } };
+		} catch (error) {
+			return { ok: false, error: error instanceof Error ? error.message : String(error) };
+		}
+	}
+
+	/** The mounted canvas once it reports the board drawn, within a bound. */
+	private async whenCanvasReady(reviewId: string): Promise<ReviewCanvasHandle> {
+		const deadline = Date.now() + RENDER_READY_WAIT_MS;
+		for (;;) {
+			const canvas = this.canvas.value;
+			if (canvas && this.readyInput !== undefined && this.readyInput === this.renderedInput && this.apiContent?.reviewId === reviewId) {
+				return canvas;
+			}
+			if (Date.now() > deadline) {
+				throw new Error("The whiteboard has not finished opening in Desktop. Render again in a moment.");
+			}
+			await new Promise((resolve) => setTimeout(resolve, 100));
+		}
+	}
+
 	private canvasSettings(): ReviewCanvasSettings {
 		return {
 			diagramFullscreenOnClick: this.configurationService.getValue<boolean>(REVIEW_DIAGRAM_FULLSCREEN_ON_CLICK_SETTING) === true,

apps/review-desktop/code-oss/src/vs/workbench/services/host/browser/host.ts

@@ -7,7 +7,7 @@ import { VSBuffer } from '../../../../base/common/buffer.js';
 import { CancellationToken } from '../../../../base/common/cancellation.js';
 import { Event } from '../../../../base/common/event.js';
 import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js';
-import { FocusMode } from '../../../../platform/native/common/native.js';
+import { FocusMode, IScreenshotOptions } from '../../../../platform/native/common/native.js';
 import { IWindowOpenable, IOpenWindowOptions, IOpenEmptyWindowOptions, IPoint, IRectangle, IOpenedMainWindow, IOpenedAuxiliaryWindow } from '../../../../platform/window/common/window.js';
 
 export const IHostService = createDecorator<IHostService>('hostService');
@@ -161,7 +161,7 @@ export interface IHostService {
 	/**
 	 * Captures a screenshot.
 	 */
-	getScreenshot(rect?: IRectangle): Promise<VSBuffer | undefined>;
+	getScreenshot(rect?: IRectangle, options?: IScreenshotOptions): Promise<VSBuffer | undefined>;
 
 	//#endregion
 

apps/review-desktop/code-oss/src/vs/workbench/services/host/electron-browser/nativeHostService.ts

@@ -5,7 +5,7 @@
 
 import { Emitter, Event } from '../../../../base/common/event.js';
 import { IHostService, IToastOptions, IToastResult } from '../browser/host.js';
-import { FocusMode, INativeHostService } from '../../../../platform/native/common/native.js';
+import { FocusMode, INativeHostService, IScreenshotOptions } from '../../../../platform/native/common/native.js';
 import { InstantiationType, registerSingleton } from '../../../../platform/instantiation/common/extensions.js';
 import { ILabelService, Verbosity } from '../../../../platform/label/common/label.js';
 import { IWorkbenchEnvironmentService } from '../../environment/common/environmentService.js';
@@ -226,8 +226,8 @@ class WorkbenchHostService extends Disposable implements IHostService {
 
 	//#region Screenshots
 
-	getScreenshot(rect?: IRectangle): Promise<VSBuffer | undefined> {
-		return this.nativeHostService.getScreenshot(rect);
+	getScreenshot(rect?: IRectangle, options?: IScreenshotOptions): Promise<VSBuffer | undefined> {
+		return this.nativeHostService.getScreenshot(rect, options);
 	}
 
 	//#endregion

packages/review-protocol/src/contracts.ts

@@ -1228,9 +1228,31 @@ export interface ReviewCanvasRange {
   worktreeRevision?: string;
 }
 
+/** Where a block's figure sits in the window, in CSS pixels, after the
+ * canvas showed the frame asked for. */
+export interface ReviewRenderedFrame {
+  /** The stop showing: its key, title, and place in the path showing. */
+  frame: { key: string; title: string; position: number; count: number };
+  /** The scenario the stop belongs to, when the block has scenarios. */
+  scenario?: string;
+  /** The part of the figure inside the window, to capture. */
+  rect: { x: number; y: number; width: number; height: number };
+  /** Whether the figure runs past the window, so `rect` cuts it. */
+  clipped: boolean;
+  /** The error the stage reported, if its script threw. */
+  error: string | null;
+}
+
 export interface ReviewCanvasHandle extends ReviewDisposable {
   update(content: ReviewCanvasContent): void;
   focus(): void;
+  /** Shows a frame of an animation (or a clip at a marker) on the board
+   * showing, scrolled into view, and reports where to capture it. Rejects
+   * when the canvas shows another board or the block is not on it. */
+  renderAnimation(
+    targetId: string,
+    frame: string,
+  ): Promise<ReviewRenderedFrame>;
   showFind(seed?: string): boolean;
   /** False when find is closed. */
   navigateFind(delta: 1 | -1): boolean;
@@ -2034,6 +2056,16 @@ export const ReviewVerbRequestSchema = z.discriminatedUnion("name", [
       version: z.number().int().nonnegative().optional(),
     }),
   }),
+  /** Show one frame of an animation (or a clip at one marker) on the open
+   * board and capture it; answers a `ReviewRenderedFrame` with its `png`. */
+  z.strictObject({
+    name: z.literal("renderAnimation"),
+    args: z.strictObject({
+      reviewId: apiReviewIdSchema,
+      targetId: requiredString,
+      frame: requiredString,
+    }),
+  }),
 ]);
 
 export type ReviewVerbRequest = z.infer<typeof ReviewVerbRequestSchema>;