From f61b01ca2d8b75b17e38877e6cbfe42654c7fa92 Mon Sep 17 00:00:00 2001 From: Milan Bhandari Date: Tue, 3 Nov 2026 15:50:45 -0500 Subject: [PATCH 1/2] Let an agent see a frame of the animation it wrote After publishing an animation an agent had no way to look at it short of driving the screen: session_get said only whether the script ran without an error. session_render shows one frame of an animation, or a clip at one marker, on the whiteboard open in Desktop and returns it as a PNG image, with the error the stage reported, if any; the PNG is also written beside the animation drafts and named in the result, for agents that read files rather than images. The animation instructions tell agents to render each frame they changed after publishing, instead of taking screenshots. Desktop captures the frame from its own canvas rather than a headless renderer drawing it: the code peeks and embedded blocks a frame shows are Desktop's own native editors and canvas components laid over the stage, so only Desktop can draw a frame as the reader sees it, and the headless server reaches it through the same relay session_open uses. The canvas asks the block to show the stop at rest (switching scenario when the frame needs one, pausing playback, waiting for the stage to settle, paint in and hold still in view) and reports where its figure is; the canvas part captures that rectangle with Electron's capturePage, now able to answer a PNG scaled to at most 1600 pixels on its longest edge. The board must be open in Desktop and its window on screen; the result says so when it is not, instead of waiting out the relay. Agent-Session: cbacb871-2558-4fe4-9272-77524bb21683 Agent-Session: 62100749-c77c-4d19-ad48-c6d4ba1e867b Agent-Session: b315a4f8-1333-461c-9be6-366ceb231b7f Agent-Session: 7bc797d9-b46c-48dc-b281-4fce85dc15ad Agent-Session: b5f9f841-35c5-4ec1-95a8-6485f64d254b Agent-Session: 32e129b9-586e-4fd2-8961-e7224c5978ad --- .../src/vs/platform/native/common/native.ts | 9 +- .../electron-main/nativeHostMainService.ts | 17 +- .../browser/parts/canvas/reviewCanvasPart.ts | 64 ++++ .../workbench/services/host/browser/host.ts | 4 +- .../electron-browser/nativeHostService.ts | 6 +- packages/review-protocol/src/contracts.ts | 32 ++ .../app/src/animation/animation-block.tsx | 235 ++++++++++++- .../review/app/src/animation/clip-player.tsx | 5 +- .../review/app/src/animation/frame-player.tsx | 14 +- .../review/app/src/animation/stage-player.ts | 5 +- .../animation/stage-renderer.browser.test.tsx | 250 ++++++++++++++ .../app/src/animation/stage-renderer.ts | 46 +++ packages/review/app/src/desktop-entry.tsx | 7 + packages/review/instructions/animation.md | 1 + packages/review/src/review-api/README.md | 1 + packages/review/src/review-api/agent-cli.ts | 5 +- .../src/review-api/animation-render.test.ts | 317 ++++++++++++++++++ .../review/src/review-api/animation-render.ts | 106 ++++++ .../review/src/review-api/authoring-tools.ts | 7 + packages/review/src/review-api/http.ts | 33 ++ packages/review/src/review-api/mcp.ts | 15 +- .../review/src/review-api/public-tools.ts | 69 ++++ .../review/src/server/render-relay.test.ts | 81 +++++ .../review/src/server/review-server-core.ts | 31 +- 24 files changed, 1330 insertions(+), 30 deletions(-) create mode 100644 packages/review/app/src/animation/stage-renderer.browser.test.tsx create mode 100644 packages/review/app/src/animation/stage-renderer.ts create mode 100644 packages/review/src/review-api/animation-render.test.ts create mode 100644 packages/review/src/review-api/animation-render.ts create mode 100644 packages/review/src/server/render-relay.test.ts diff --git a/apps/review-desktop/code-oss/src/vs/platform/native/common/native.ts b/apps/review-desktop/code-oss/src/vs/platform/native/common/native.ts index a42cc8fefd..06d2c78340 100644 --- a/apps/review-desktop/code-oss/src/vs/platform/native/common/native.ts +++ b/apps/review-desktop/code-oss/src/vs/platform/native/common/native.ts @@ -56,6 +56,13 @@ export interface INativeHostOptions { readonly targetWindowId?: number; } +export interface IScreenshotOptions { + /** JPEG at quality 95 unless 'png'. */ + readonly format?: 'jpeg' | 'png'; + /** Scales the capture down so neither edge exceeds this many pixels. */ + readonly maxEdge?: number; +} + export interface IUninstallShellCommandOptions { readonly commandName?: string; readonly symlinkOnly?: boolean; @@ -249,7 +256,7 @@ export interface ICommonNativeHostService { hasWSLFeatureInstalled(): Promise; // Screenshots - getScreenshot(rect?: IRectangle): Promise; + getScreenshot(rect?: IRectangle, options?: INativeHostOptions & IScreenshotOptions): Promise; // GitHub mobile upload API (runs in main process to avoid CORS) uploadFileViaMobileApi(token: string, repoId: string, fileName: string, fileBytes: VSBuffer, contentType: string): Promise<{ fileName: string; assetUrl: string; contentType: string }>; diff --git a/apps/review-desktop/code-oss/src/vs/platform/native/electron-main/nativeHostMainService.ts b/apps/review-desktop/code-oss/src/vs/platform/native/electron-main/nativeHostMainService.ts index 575f647927..ee97877f2d 100644 --- a/apps/review-desktop/code-oss/src/vs/platform/native/electron-main/nativeHostMainService.ts +++ b/apps/review-desktop/code-oss/src/vs/platform/native/electron-main/nativeHostMainService.ts @@ -27,7 +27,7 @@ import { IEnvironmentMainService } from '../../environment/electron-main/environ import { createDecorator, IInstantiationService } from '../../instantiation/common/instantiation.js'; import { ILifecycleMainService, IRelaunchOptions } from '../../lifecycle/electron-main/lifecycleMainService.js'; import { ILogService } from '../../log/common/log.js'; -import { FocusMode, ICommonNativeHostService, INativeHostOptions, INativeSystemWideKeybinding, INativeSystemWideKeybindingResult, IOSProperties, IOSStatistics, IStartTracingOptions, IToastOptions, IToastResult, IUninstallShellCommandOptions, PowerSaveBlockerType, SystemIdleState, ThermalState } from '../common/native.js'; +import { FocusMode, ICommonNativeHostService, INativeHostOptions, INativeSystemWideKeybinding, INativeSystemWideKeybindingResult, IOSProperties, IOSStatistics, IStartTracingOptions, IToastOptions, IToastResult, IUninstallShellCommandOptions, PowerSaveBlockerType, SystemIdleState, ThermalState, IScreenshotOptions } from '../common/native.js'; import { IGlobalKeybindingsMainService } from '../../globalKeybindings/electron-main/globalKeybindingsMainService.js'; import { IProductService } from '../../product/common/productService.js'; import { IPartsSplash } from '../../theme/common/themeService.js'; @@ -910,12 +910,21 @@ export class NativeHostMainService extends Disposable implements INativeHostMain //#region Screenshots - async getScreenshot(windowId: number | undefined, rect?: IRectangle, options?: INativeHostOptions): Promise { + async getScreenshot(windowId: number | undefined, rect?: IRectangle, options?: INativeHostOptions & IScreenshotOptions): Promise { const window = this.windowById(options?.targetWindowId, windowId); const captured = await window?.win?.webContents.capturePage(rect); + if (!captured || captured.isEmpty()) { + return undefined; + } + + let image = captured; + const { width, height } = image.getSize(); + const maxEdge = options?.maxEdge; + if (maxEdge && Math.max(width, height) > maxEdge) { + image = image.resize(width >= height ? { width: maxEdge, quality: 'best' } : { height: maxEdge, quality: 'best' }); + } - const buf = captured?.toJPEG(95); - return buf && VSBuffer.wrap(buf); + return VSBuffer.wrap(options?.format === 'png' ? image.toPNG() : image.toJPEG(95)); } //#endregion diff --git a/apps/review-desktop/code-oss/src/vs/review/browser/parts/canvas/reviewCanvasPart.ts b/apps/review-desktop/code-oss/src/vs/review/browser/parts/canvas/reviewCanvasPart.ts index 3f014a568b..17d10d541a 100644 --- a/apps/review-desktop/code-oss/src/vs/review/browser/parts/canvas/reviewCanvasPart.ts +++ b/apps/review-desktop/code-oss/src/vs/review/browser/parts/canvas/reviewCanvasPart.ts @@ -6,7 +6,9 @@ import { openChildWhiteboard } from "../../reviewChildWhiteboard.js"; import { reviewBottomScrollPadding } from "../../reviewScrollPadding.js"; +import { getZoomFactor } from "../../../../base/browser/browser.js"; import { $, addDisposableListener, getWindow, type Dimension } from "../../../../base/browser/dom.js"; +import { encodeBase64 } from "../../../../base/common/buffer.js"; import { mainWindow } from "../../../../base/browser/window.js"; import type { IHoverOptions, IHoverWidget } from "../../../../base/browser/ui/hover/hover.js"; import { HoverPosition } from "../../../../base/browser/ui/hover/hoverWidget.js"; @@ -164,6 +166,12 @@ const APPEARANCE_SETTINGS = { const requestReviewApi: typeof fetch = (url, init) => fetch(url, init); +/** The longest edge of a rendered frame's PNG: a retina capture of a column-wide block, which agents read whole. */ +const RENDER_MAX_EDGE = 1600; + +/** How long a render waits for a board that is still opening. */ +const RENDER_READY_WAIT_MS = 10_000; + function embeddedSelectionChangeReason(event: ICursorPositionChangedEvent): EditorPaneSelectionChangeReason { switch (event.source) { case TextEditorSelectionSource.PROGRAMMATIC: @@ -387,6 +395,9 @@ export class ReviewCanvasEditorPane extends EditorPane { await this.hostService.focus(this.targetDocument?.defaultView ?? window, { mode: FocusMode.Force }); return { ok: true }; } + if (request.name === "renderAnimation") { + return this.renderAnimation(request.args); + } return this.verbs.dispatch(request); }); void this.desktopConnection.initialize().catch((error) => this.renderError(error)); @@ -1507,6 +1518,59 @@ export class ReviewCanvasEditorPane extends EditorPane { }; } + /** + * An agent's look at a frame it wrote: the canvas shows the frame and says + * where its figure sits, and the window captures that rectangle as a PNG. + * Only the board open in this part can be rendered; the canvas measures + * CSS pixels, which the window's zoom scales to the capture's pixels. + */ + private async renderAnimation(args: { reviewId: string; targetId: string; frame: string }): Promise { + if (this.apiContent?.reviewId !== args.reviewId) { + return { ok: false, error: "The whiteboard is not open in Desktop. Open it with session_open, then render again." }; + } + if (this.targetDocument !== mainWindow.document) { + return { ok: false, error: "The whiteboard is open in a secondary window; only the main window can render it." }; + } + try { + const canvas = await this.whenCanvasReady(args.reviewId); + const shown = await canvas.renderAnimation(args.targetId, args.frame); + if (shown.rect.width < 1 || shown.rect.height < 1) { + throw new Error("The animation is outside the window."); + } + const zoom = getZoomFactor(mainWindow); + const png = await this.hostService.getScreenshot( + { + x: Math.round(shown.rect.x * zoom), + y: Math.round(shown.rect.y * zoom), + width: Math.round(shown.rect.width * zoom), + height: Math.round(shown.rect.height * zoom), + }, + { format: "png", maxEdge: RENDER_MAX_EDGE }, + ); + if (!png) { + throw new Error("Desktop could not capture its window; it must be visible on screen."); + } + return { ok: true, result: { ...shown, png: encodeBase64(png) } }; + } catch (error) { + return { ok: false, error: error instanceof Error ? error.message : String(error) }; + } + } + + /** The mounted canvas once it reports the board drawn, within a bound. */ + private async whenCanvasReady(reviewId: string): Promise { + const deadline = Date.now() + RENDER_READY_WAIT_MS; + for (;;) { + const canvas = this.canvas.value; + if (canvas && this.readyInput !== undefined && this.readyInput === this.renderedInput && this.apiContent?.reviewId === reviewId) { + return canvas; + } + if (Date.now() > deadline) { + throw new Error("The whiteboard has not finished opening in Desktop. Render again in a moment."); + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } + private canvasSettings(): ReviewCanvasSettings { return { diagramFullscreenOnClick: this.configurationService.getValue(REVIEW_DIAGRAM_FULLSCREEN_ON_CLICK_SETTING) === true, diff --git a/apps/review-desktop/code-oss/src/vs/workbench/services/host/browser/host.ts b/apps/review-desktop/code-oss/src/vs/workbench/services/host/browser/host.ts index a43eb767fb..0a4aee601c 100644 --- a/apps/review-desktop/code-oss/src/vs/workbench/services/host/browser/host.ts +++ b/apps/review-desktop/code-oss/src/vs/workbench/services/host/browser/host.ts @@ -7,7 +7,7 @@ import { VSBuffer } from '../../../../base/common/buffer.js'; import { CancellationToken } from '../../../../base/common/cancellation.js'; import { Event } from '../../../../base/common/event.js'; import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js'; -import { FocusMode } from '../../../../platform/native/common/native.js'; +import { FocusMode, IScreenshotOptions } from '../../../../platform/native/common/native.js'; import { IWindowOpenable, IOpenWindowOptions, IOpenEmptyWindowOptions, IPoint, IRectangle, IOpenedMainWindow, IOpenedAuxiliaryWindow } from '../../../../platform/window/common/window.js'; export const IHostService = createDecorator('hostService'); @@ -161,7 +161,7 @@ export interface IHostService { /** * Captures a screenshot. */ - getScreenshot(rect?: IRectangle): Promise; + getScreenshot(rect?: IRectangle, options?: IScreenshotOptions): Promise; //#endregion diff --git a/apps/review-desktop/code-oss/src/vs/workbench/services/host/electron-browser/nativeHostService.ts b/apps/review-desktop/code-oss/src/vs/workbench/services/host/electron-browser/nativeHostService.ts index 86a3d521e1..1eeebd7f56 100644 --- a/apps/review-desktop/code-oss/src/vs/workbench/services/host/electron-browser/nativeHostService.ts +++ b/apps/review-desktop/code-oss/src/vs/workbench/services/host/electron-browser/nativeHostService.ts @@ -5,7 +5,7 @@ import { Emitter, Event } from '../../../../base/common/event.js'; import { IHostService, IToastOptions, IToastResult } from '../browser/host.js'; -import { FocusMode, INativeHostService } from '../../../../platform/native/common/native.js'; +import { FocusMode, INativeHostService, IScreenshotOptions } from '../../../../platform/native/common/native.js'; import { InstantiationType, registerSingleton } from '../../../../platform/instantiation/common/extensions.js'; import { ILabelService, Verbosity } from '../../../../platform/label/common/label.js'; import { IWorkbenchEnvironmentService } from '../../environment/common/environmentService.js'; @@ -226,8 +226,8 @@ class WorkbenchHostService extends Disposable implements IHostService { //#region Screenshots - getScreenshot(rect?: IRectangle): Promise { - return this.nativeHostService.getScreenshot(rect); + getScreenshot(rect?: IRectangle, options?: IScreenshotOptions): Promise { + return this.nativeHostService.getScreenshot(rect, options); } //#endregion diff --git a/packages/review-protocol/src/contracts.ts b/packages/review-protocol/src/contracts.ts index 2c75e63a5b..9b625b2c3c 100644 --- a/packages/review-protocol/src/contracts.ts +++ b/packages/review-protocol/src/contracts.ts @@ -1229,9 +1229,31 @@ export interface ReviewCanvasRange { worktreeRevision?: string; } +/** Where a block's figure sits in the window, in CSS pixels, after the + * canvas showed the frame asked for. */ +export interface ReviewRenderedFrame { + /** The stop showing: its key, title, and place in the path showing. */ + frame: { key: string; title: string; position: number; count: number }; + /** The scenario the stop belongs to, when the block has scenarios. */ + scenario?: string; + /** The part of the figure inside the window, to capture. */ + rect: { x: number; y: number; width: number; height: number }; + /** Whether the figure runs past the window, so `rect` cuts it. */ + clipped: boolean; + /** The error the stage reported, if its script threw. */ + error: string | null; +} + export interface ReviewCanvasHandle extends ReviewDisposable { update(content: ReviewCanvasContent): void; focus(): void; + /** Shows a frame of an animation (or a clip at a marker) on the board + * showing, scrolled into view, and reports where to capture it. Rejects + * when the canvas shows another board or the block is not on it. */ + renderAnimation( + targetId: string, + frame: string, + ): Promise; showFind(seed?: string): boolean; /** False when find is closed. */ navigateFind(delta: 1 | -1): boolean; @@ -2035,6 +2057,16 @@ export const ReviewVerbRequestSchema = z.discriminatedUnion("name", [ version: z.number().int().nonnegative().optional(), }), }), + /** Show one frame of an animation (or a clip at one marker) on the open + * board and capture it; answers a `ReviewRenderedFrame` with its `png`. */ + z.strictObject({ + name: z.literal("renderAnimation"), + args: z.strictObject({ + reviewId: apiReviewIdSchema, + targetId: requiredString, + frame: requiredString, + }), + }), ]); export type ReviewVerbRequest = z.infer; diff --git a/packages/review/app/src/animation/animation-block.tsx b/packages/review/app/src/animation/animation-block.tsx index f38af9a9b3..eb4a95d071 100644 --- a/packages/review/app/src/animation/animation-block.tsx +++ b/packages/review/app/src/animation/animation-block.tsx @@ -32,6 +32,7 @@ import { textStyles } from "@canvas/ui/text"; import { useTooltip } from "@canvas/ui/tooltip"; import { useDiagramSelection } from "@canvas/use-diagram-selection"; import { useWhiteboardSettings } from "@canvas/whiteboard-settings"; +import type { ReviewRenderedFrame } from "@dev.fast/review-protocol"; import { type Block, anchorSelection, @@ -63,7 +64,7 @@ import { useClipPlayer } from "./clip-player"; import { useFramePlayer } from "./frame-player"; import { useReportAnimationRun } from "./report-run"; import { StepStrip } from "./stage-controls"; -import type { StagePlayer } from "./stage-player"; +import type { PlayerStop, StagePlayer } from "./stage-player"; import { type HostMessage, STAGE_BOOT, @@ -73,6 +74,7 @@ import { type StageMessage, stageMessageSchema, } from "./stage-protocol"; +import { registerStageRenderer } from "./stage-renderer"; import { stageRuntime } from "./stage-runtime"; import { SCALE } from "./stage-scale"; @@ -90,6 +92,32 @@ const MEASURE_WAIT = 1000; // at most this much: past 2x a drawing made for a column reads as a poster. const STAGE_MAX_SCALE = 2; +// How long a render waits for the stage to boot, paint, show the stop asked +// for and settle, within Desktop's own wait on the verb. +const RENDER_WAIT = 30_000; + +// A render polls on a timer, not on animation frames: a window in the +// background may paint no frame for a while, and a timer still runs there. +const RENDER_POLL = 50; + +// How many frames a render gives the figure to come to rest in the window +// before it measures where it is: enough for a smooth scroll to finish. +const RENDER_STILL_FRAMES = 60; + +const wait = (ms: number) => + new Promise((resolve) => setTimeout(resolve, ms)); + +// A frame to lay out and paint, or the poll's wait where none comes. +const nextFrame = () => + new Promise((resolve) => { + const timer = setTimeout(resolve, RENDER_POLL * 4); + + requestAnimationFrame(() => { + clearTimeout(timer); + resolve(); + }); + }); + let animeSource: Promise | undefined; // anime.js's source, for the stage: its own chunk, fetched by the first @@ -178,15 +206,43 @@ function embeddableBlocks(document: Block[]): Map { type StageBlockProps = BlockProps<"animation" | "clip">; +/** What the stage was last asked to show and what it reports showing: a + * frame's index once it settled there, or a clip's marker once the playhead + * reached it. A render waits for the two to agree. */ +interface StageShowing { + askedFrame: number | null; + shownFrame: number | null; + askedMarker: string | null; + shownMarker: string | null; +} + +const nothingShowing = (): StageShowing => ({ + askedFrame: null, + shownFrame: null, + askedMarker: null, + shownMarker: null, +}); + /** The channel to the stage showing: set when it boots. */ function useStagePort() { const portRef = useRef(null); + const showingRef = useRef(nothingShowing()); const send = useCallback((message: HostMessage) => { + const showing = showingRef.current; + + if (message.type === "frame") { + showing.askedFrame = message.index; + showing.shownFrame = null; + } else if (message.type === "seek") { + showing.askedMarker = message.marker; + showing.shownMarker = null; + } + portRef.current?.postMessage(message); }, []); - return { portRef, send }; + return { portRef, showingRef, send }; } type StagePort = ReturnType; @@ -233,7 +289,7 @@ function StageBlock({ node, data, children, - port: { portRef, send }, + port: { portRef, showingRef, send }, player, }: StageBlockProps & { port: StagePort; player: StagePlayer }) { const { theme } = useReviewDebugSettings(); @@ -266,11 +322,12 @@ function StageBlock({ const { painted } = useDrawQueueControls(); // The stage showing now: whether it boots unseen, has run its scripts, was - // told to paint, and reported an error. + // told to paint, has painted, and reported an error. const paintRef = useRef({ wanted: false, ready: false, sent: false, + painted: false, failed: false, }); @@ -422,6 +479,8 @@ function StageBlock({ const onMessage = useRef<(message: StageMessage) => void>(() => {}); onMessage.current = (message) => { + const showing = showingRef.current; + switch (message.type) { case "size": setContentHeight(Math.ceil(message.height)); @@ -445,8 +504,21 @@ function StageBlock({ paintStage(); break; case "painted": + paintRef.current.painted = true; endPaint(); break; + case "settled": + showing.shownFrame = message.index; + + if (showing.askedFrame === message.index) showing.askedFrame = null; + player.receive(message); + break; + case "marker": + showing.shownMarker = message.key; + + if (showing.askedMarker === message.key) showing.askedMarker = null; + player.receive(message); + break; default: player.receive(message); } @@ -537,6 +609,7 @@ function StageBlock({ bootedRef.current = frameRef.current; const channel = new MessageChannel(); portRef.current = channel.port1; + showingRef.current = nothingShowing(); channel.port1.onmessage = (event: MessageEvent) => { const message = stageMessageSchema.safeParse(event.data); @@ -557,6 +630,7 @@ function StageBlock({ wanted: paint, ready: false, sent: false, + painted: false, failed: false, }; @@ -574,7 +648,158 @@ function StageBlock({ endPaint(); }, ); - }, [noun, endPaint, portRef, stageInit]); + }, [noun, endPaint, portRef, showingRef, stageInit]); + + // An agent rendering a stop reads the block as it is now, whatever + // render it was registered in. + const playerRef = useRef(player); + playerRef.current = player; + const errorRef = useRef(error); + errorRef.current = error; + const measuringRef = useRef(false); + measuringRef.current = measureRun !== null; + + // Shows a stop for an agent: once the stage is up and painted, in the + // scenario that has the stop, at rest, scrolled into view. + const show = useEffectEvent( + async (key: string): Promise => { + const deadline = Date.now() + RENDER_WAIT; + + // A covered or minimized window paints no frame, and its timers slow to + // a crawl, so nothing here would move: say so instead of waiting. + const hidden = () => { + if (figureRef.current?.ownerDocument.visibilityState === "hidden") + throw new Error( + "Desktop's window is hidden, minimized or covered by other windows, so the stage cannot draw. Bring it onto the screen and render again.", + ); + }; + + // A stage that failed shows what it has: the render reports the error. + const until = async ( + condition: () => boolean, + what: string, + by = deadline, + ) => { + while (!condition() && !paintRef.current.failed) { + hidden(); + + if (Date.now() > by) throw new Error(`The ${noun} did not ${what}.`); + await wait(RENDER_POLL); + } + }; + + hidden(); + + await until(() => paintRef.current.ready, "start in time"); + + const listed = (stops: readonly PlayerStop[]) => + stops.some((stop) => stop.key === key); + + if (!listed(playerRef.current.stops)) { + const scenario = + node.type === "animation" + ? node.scenarios?.find((item) => item.frames.includes(key)) + : undefined; + + if (!scenario) + throw new Error( + `${key} is not a ${clip ? "marker" : "frame"} of the ${noun}.`, + ); + playerRef.current.selectScenario(scenario.key); + await until( + () => listed(playerRef.current.stops), + `switch to scenario ${scenario.key} in time`, + ); + } + + const { stops } = playerRef.current; + const position = stops.findIndex((stop) => stop.key === key); + playerRef.current.pick(position, "render"); + + const showing = showingRef.current; + + if (clip) + await until( + () => showing.shownMarker === key && showing.askedMarker === null, + `reach marker ${key} in time`, + ); + else { + const index = node.frames.findIndex((frame) => frame.key === key); + await until( + () => showing.shownFrame === index && showing.askedFrame === null, + `settle on frame ${key} in time`, + ); + } + + const paint = paintRef.current; + await until(() => !paint.wanted || paint.painted, "paint in time"); + + // Its height follows the tallest frame once measured; a measure that + // takes too long does not hold the render. + await until( + () => !measuringRef.current, + "measure", + Date.now() + MEASURE_WAIT, + ).catch(() => {}); + await nextFrame(); + await nextFrame(); + + const figure = figureRef.current; + const view = figure?.ownerDocument.defaultView; + + if (!figure || !view) throw new Error(`The ${noun} is not on the board.`); + + figure.scrollIntoView({ + block: figure.offsetHeight > view.innerHeight ? "start" : "nearest", + behavior: "instant", + }); + + // Measure once the figure holds still: a scroll may still be moving, and + // a block above may still be taking its height. + let box = figure.getBoundingClientRect(); + + for (let frame = 0; frame < RENDER_STILL_FRAMES; frame++) { + await nextFrame(); + const next = figure.getBoundingClientRect(); + + const still = + Math.abs(next.top - box.top) < 0.5 && + Math.abs(next.left - box.left) < 0.5 && + Math.abs(next.height - box.height) < 0.5 && + Math.abs(next.width - box.width) < 0.5; + + box = next; + + if (still) break; + } + + const x = Math.max(0, box.left); + const y = Math.max(0, box.top); + const right = Math.min(view.innerWidth, box.right); + const bottom = Math.min(view.innerHeight, box.bottom); + const width = Math.max(0, right - x); + const height = Math.max(0, bottom - y); + const stop = stops[position]!; + + return { + frame: { key, title: stop.title, position, count: stops.length }, + ...(playerRef.current.scenario !== undefined && { + scenario: playerRef.current.scenario, + }), + rect: { x, y, width, height }, + clipped: width < box.width - 1 || height < box.height - 1, + error: errorRef.current, + }; + }, + ); + + useEffect(() => { + if (!node.id) return; + + return registerStageRenderer(data.snapshot.reviewId, node.id, { + show: (key) => show(key), + }); + }, [data.snapshot.reviewId, node.id]); // Measure again in a new copy after anything that can move a frame's layout. const remeasure = useCallback(() => { diff --git a/packages/review/app/src/animation/clip-player.tsx b/packages/review/app/src/animation/clip-player.tsx index 0f974add6f..51e68b07fa 100644 --- a/packages/review/app/src/animation/clip-player.tsx +++ b/packages/review/app/src/animation/clip-player.tsx @@ -53,8 +53,9 @@ export function useClipPlayer( if (!stop) return; - // The tour reads a stop at rest; the controls keep playing if it was. - if (by === "tour" && playing) { + // The tour and a render read a stop at rest; the controls keep playing + // if it was. + if (by !== "controls" && playing) { send({ type: "pause" }); setPlaying(false); } diff --git a/packages/review/app/src/animation/frame-player.tsx b/packages/review/app/src/animation/frame-player.tsx index 3d8d0354dd..b44e781b7d 100644 --- a/packages/review/app/src/animation/frame-player.tsx +++ b/packages/review/app/src/animation/frame-player.tsx @@ -23,7 +23,12 @@ import { timerState, timerVars, } from "./stage-controls"; -import type { FrameTimer, StagePlayer, StageSend } from "./stage-player"; +import type { + FrameTimer, + PickedBy, + StagePlayer, + StageSend, +} from "./stage-player"; import type { StageMessage } from "./stage-protocol"; const DEFAULT_DURATION = 3000; @@ -274,8 +279,11 @@ export function useFramePlayer( if (!activeFrames.some((item) => item.key === frame.key)) go(0, true); }, [activeFrames, frame.key, go]); - const pick = (position: number) => + // A render reads the frame at rest; the controls and the tour keep playing. + const pick = (position: number, by: PickedBy) => { + if (by === "render") setPlaying(false); go(position, Math.abs(position - activeIndex) > 1); + }; const steps = activeFrames.every((item) => item.label) ? activeFrames : null; @@ -350,7 +358,7 @@ export function useFramePlayer( title={item.title} current={position === activeIndex} timer={position === activeIndex ? timer : null} - onClick={() => pick(position)} + onClick={() => pick(position, "controls")} /> ))} diff --git a/packages/review/app/src/animation/stage-player.ts b/packages/review/app/src/animation/stage-player.ts index a76cd67233..add3251866 100644 --- a/packages/review/app/src/animation/stage-player.ts +++ b/packages/review/app/src/animation/stage-player.ts @@ -16,8 +16,9 @@ export interface PlayerStop { code?: string[]; } -/** Who picked a stop: the controls under the stage, or the tour. */ -export type PickedBy = "controls" | "tour"; +/** Who picked a stop: the controls under the stage, the tour, or an agent + * rendering the stop, which reads it at rest. */ +export type PickedBy = "controls" | "tour" | "render"; export type StageSend = (message: HostMessage) => void; diff --git a/packages/review/app/src/animation/stage-renderer.browser.test.tsx b/packages/review/app/src/animation/stage-renderer.browser.test.tsx new file mode 100644 index 0000000000..7a6fc7b7b4 --- /dev/null +++ b/packages/review/app/src/animation/stage-renderer.browser.test.tsx @@ -0,0 +1,250 @@ +// An agent's look at a frame it wrote: the block shows the stop asked for, +// at rest and in view, and reports where its figure is for Desktop to +// capture. Here the harness captures that figure instead, so a reference +// animation renders to a real PNG. + +import { + loadFixture, + mountWhiteboard, + settled, +} from "@canvas/whiteboard-fixture"; +import { type JsonObject, isJsonObject, parseJsonText } from "@dev.fast/json"; +import type { ReviewRenderedFrame } from "@dev.fast/review-protocol"; +import { + type Block, + documentSchema, + selectionKey, +} from "@dev.fast/whiteboard-document"; +import type { ReviewProgress } from "@review/review-api/review-progress"; +import type { Snapshot } from "@review/review-api/store"; +import { act } from "react"; +import { afterEach, describe, expect, it } from "vitest"; +import { page } from "vitest/browser"; + +import { renderStage } from "./stage-renderer"; + +import "@canvas/styles.css"; + +const files = import.meta.glob( + "../../../instructions/animations/retry-backoff/*", + { query: "?raw", import: "default", eager: true }, +); + +const ANCHOR = "head/order.ts#L1"; + +const REVIEW_ID = "render-test"; + +/** The retry-backoff reference as a block, its code anchors pointed at this + * board's one range, with the frames it declares. It has two scenarios; + * "a3fail" sits only in the second. */ +function retryBackoff() { + const folder = Object.fromEntries( + Object.entries(files).map(([file, text]) => [file.split("/").at(-1), text]), + ); + + const block = parseJsonText(folder["block.json"]!) as JsonObject; + + const code = Object.fromEntries( + Object.entries(block.code as JsonObject).map(([key, entry]) => [ + key, + isJsonObject(entry) ? { ...entry, source: ANCHOR } : ANCHOR, + ]), + ); + + const [parsed] = documentSchema.parse([ + { + id: "retry", + type: "animation", + ...block, + code, + html: folder["animation.html"]!, + frames: parseJsonText(folder["frames.json"]!), + scenarios: parseJsonText(folder["scenarios.json"]!), + pins: { repositoryId: "repo", base: "base", head: "head" }, + }, + ]); + + if (parsed?.type !== "animation") throw new Error("not an animation"); + + return parsed; +} + +// The slot rides the timeline from 0 to 200px between the markers. +const clip: Block = documentSchema.parse([ + { + id: "trip", + type: "clip", + title: "Trip", + height: 160, + markers: [ + { key: "start", title: "Start" }, + { key: "end", title: "End" }, + ], + html: `
packet
`, + pins: { repositoryId: "repo", base: "base", head: "head" }, + }, +])[0]!; + +const progress: ReviewProgress = { + files: [], + lenses: [], + resolvedSelections: { + [selectionKey({ + file: "order.ts", + start: { side: "head", line: 1 }, + end: { side: "head", line: 1 }, + })]: [{ file: "order.ts", side: "head", fromLine: 1, toLine: 1 }], + }, +}; + +let canvas: ReturnType | undefined; + +afterEach(async () => { + await act(async () => canvas?.dispose()); + canvas = undefined; + document.body.replaceChildren(); +}); + +async function mount(blocks: Block[]) { + const snapshot: Snapshot = { + reviewId: REVIEW_ID, + version: 0, + title: "Render", + pins: { repositoryId: "repo", base: "base", head: "head" }, + target: { + kind: "commits", + repositoryId: "repo", + base: "base", + head: "head", + }, + document: blocks, + createdAt: "2026-10-31T00:00:00.000Z", + }; + + const container = document.createElement("div"); + container.style.cssText = "width: 860px; margin: 0 auto;"; + document.body.append(container); + + await act(async () => { + canvas = mountWhiteboard( + container, + loadFixture({ + snapshot, + progress, + commits: [], + images: {}, + traces: {}, + maps: {}, + }), + ); + }); + + expect(await settled(() => container.querySelector("iframe"))).toBeTruthy(); + + return container; +} + +const text = (container: HTMLElement) => container.textContent ?? ""; + +// Playwright captures the figure the way Desktop captures its rectangle. An +// unsaved screenshot comes back as its base64. +const capture = (figure: Element) => + page.elementLocator(figure).screenshot({ save: false }); + +/** What a render came to: the frame it showed, or why it did not. */ +interface RenderOutcome { + value?: ReviewRenderedFrame; + failure?: Error; +} + +/** Renders while flushing React, as the canvas's effects commit only inside + * act in this environment. */ +async function render(blockId: string, frame: string) { + const outcome: RenderOutcome = {}; + + const pending = renderStage(REVIEW_ID, blockId, frame).then( + (value) => { + outcome.value = value; + }, + (failure: Error) => { + outcome.failure = failure; + }, + ); + + await settled(() => "value" in outcome || "failure" in outcome, 40_000); + await pending; + + if (outcome.failure) throw outcome.failure; + + return outcome.value!; +} + +describe("renderStage", () => { + it("shows a reference animation's frame, switching scenario for it, and the figure captures as a PNG", async () => { + const reference = retryBackoff(); + const container = await mount([reference]); + const figure = container.querySelector(".animation-block")!; + + const rendered = await render("retry", "a3fail"); + + const title = reference.frames.find( + (frame) => frame.key === "a3fail", + )!.title; + + expect(rendered).toMatchObject({ + frame: { key: "a3fail", title, position: 4, count: 8 }, + scenario: "gives-up", + clipped: false, + error: null, + }); + expect(text(container)).toContain("5 / 8"); + expect(text(container)).toContain(rendered.frame.title); + + const box = figure.getBoundingClientRect(); + expect(rendered.rect).toEqual({ + x: box.left, + y: box.top, + width: box.width, + height: box.height, + }); + expect(rendered.rect.width).toBeGreaterThan(400); + expect(rendered.rect.height).toBeGreaterThan(120); + + const png = await capture(figure); + // A PNG, and not a blank one: its signature, then enough bytes to draw. + expect(png.startsWith("iVBORw0KGgo")).toBe(true); + expect(png.length).toBeGreaterThan(8_000); + + // Back within the scenario showing, a frame already shown returns at once. + const again = await render("retry", "a1"); + expect(again.frame).toMatchObject({ key: "a1", position: 0, count: 8 }); + expect(text(container)).toContain("1 / 8"); + }); + + it("lands a clip on a marker at rest", async () => { + const container = await mount([clip]); + + const rendered = await render("trip", "end"); + + expect(rendered).toMatchObject({ + frame: { key: "end", title: "End", position: 1, count: 2 }, + clipped: false, + error: null, + }); + expect(text(container)).toContain("2 / 2"); + // At rest at its end: the control offers Replay, not Pause. + expect(container.querySelector("button[aria-label='Pause']")).toBeNull(); + expect(container.querySelector("button[aria-label='Replay']")).toBeTruthy(); + }); + + it("names a stop the block does not have, and a block the board does not", async () => { + await mount([clip]); + + await expect(render("trip", "middle")).rejects.toThrow( + "middle is not a marker of the clip.", + ); + await expect(render("other", "start")).rejects.toThrow( + "other is not an animation or clip on the board showing.", + ); + }); +}); diff --git a/packages/review/app/src/animation/stage-renderer.ts b/packages/review/app/src/animation/stage-renderer.ts new file mode 100644 index 0000000000..f719b743dc --- /dev/null +++ b/packages/review/app/src/animation/stage-renderer.ts @@ -0,0 +1,46 @@ +// How an agent sees a frame it wrote: Desktop asks the block showing it to +// jump there and come to rest, then captures the figure's place in the +// window. Each stage block on a board registers here while it is mounted. + +import type { ReviewRenderedFrame } from "@dev.fast/review-protocol"; + +export interface StageRenderer { + /** Shows the stop `key` (a frame, or a clip's marker) at rest, scrolled + * into view, and reports where the figure is. */ + show(key: string): Promise; +} + +const renderers = new Map(); + +const rendererKey = (reviewId: string, blockId: string) => + `${reviewId}\n${blockId}`; + +export function registerStageRenderer( + reviewId: string, + blockId: string, + renderer: StageRenderer, +): () => void { + const key = rendererKey(reviewId, blockId); + renderers.set(key, renderer); + + return () => { + if (renderers.get(key) === renderer) renderers.delete(key); + }; +} + +/** Shows `frame` of the block and reports where to capture it. Rejects when + * the board showing has no animation or clip with that id. */ +export function renderStage( + reviewId: string, + blockId: string, + frame: string, +): Promise { + const renderer = renderers.get(rendererKey(reviewId, blockId)); + + if (!renderer) + return Promise.reject( + new Error(`${blockId} is not an animation or clip on the board showing.`), + ); + + return renderer.show(frame); +} diff --git a/packages/review/app/src/desktop-entry.tsx b/packages/review/app/src/desktop-entry.tsx index 5e54df0a37..2a0d094b25 100644 --- a/packages/review/app/src/desktop-entry.tsx +++ b/packages/review/app/src/desktop-entry.tsx @@ -27,6 +27,7 @@ import { } from "react"; import { createRoot } from "react-dom/client"; +import { renderStage } from "./animation/stage-renderer"; import { ApiCanvas } from "./api-canvas"; import { SharedCanvasQueryClient, @@ -462,6 +463,12 @@ export function mountReviewCanvas( focus() { container.focus(); }, + renderAnimation(targetId, frame) { + if (content.kind !== "api") + return Promise.reject(new Error("The canvas shows no whiteboard.")); + + return renderStage(content.reviewId, targetId, frame); + }, showFind(seed) { return ( (content.kind === "api" || content.kind === "home") && diff --git a/packages/review/instructions/animation.md b/packages/review/instructions/animation.md index fcb9a4b6ca..beddb53d0c 100644 --- a/packages/review/instructions/animation.md +++ b/packages/review/instructions/animation.md @@ -10,6 +10,7 @@ A `clip` component runs in the same stage with the same API, but it has no frame 2. Edit those files with your file tools. 3. Publish the folder: `session_edit({ sessionId, edit: { type: "update", targetId, fromFiles: "" } })`. Only the files you changed are saved. Keep editing and publishing the same folder; it records what it last published in `checkout.json`. 4. Read it back with `session_get({ sessionId, targetId })`. Its script runs only where the whiteboard is open, a moment after you publish, so the read says whether it ran there without an error, threw (with the error), or has not run yet. Fix an error and publish again. +5. Look at it: `session_render({ sessionId, targetId, frame })` shows that frame (or a clip at that marker; omitted, the first) on the whiteboard open in Desktop, as the reader sees it, and returns it as a PNG image, with the error the script threw, if any. It also writes the PNG to the path the result names. Render each frame you changed and check the drawing is what you meant: cards where you placed them, arrows between the right parts, nothing clipped or overlapping. Do not take screenshots of the screen. The whiteboard must be open in Desktop: `session_create` opens it, and `session_open` opens an existing one. For a new animation, write the same files into a folder of your own and publish with `{ type: "insert", fromFiles: "", parentId, afterId }`. If someone else changed the animation after your checkout, the publish is refused: check it out again. diff --git a/packages/review/src/review-api/README.md b/packages/review/src/review-api/README.md index bcec6bc8eb..d53ca53dbf 100644 --- a/packages/review/src/review-api/README.md +++ b/packages/review/src/review-api/README.md @@ -55,6 +55,7 @@ All paths below are relative to `/reviews-api`. | `GET /:id/history` | Saved versions with titles, timestamps and `author`: `"you"` for a person's save, else `"agent"` with the Ask `threadId` when one made it | | `GET /:id/inspect` | Agent reading view: nested text outline with IDs; `targetId` reads one component completely, `full=true` includes all content, `version` selects history. `format=json` returns raw data instead. | | `POST /:id/open` | Open the review in the attached Desktop; report an error when none is attached | +| `GET /:id/render?targetId=anim-1&frame=a2` | Show that frame (or a clip at that marker) of the review open in the attached Desktop and return it: `type`, `frame` (key, title, position, count), `scenario`, `clipped`, `error` (what the stage's script threw, or null) and `png` (base64). 409 when no Desktop shows the review; 400 for a frame the block does not have | | `POST /commands` | Apply one command; return review ID, version, and for an edit the target ID, its `type`, and — after an insert or replace — `children`: its first-level children as `{id,type}` (a container's blocks; a diagram's steps, or nodes then edges), so new components are addressable without a read. A `create` with `pullRequestUrl` returns the newest existing review for that PR instead (owner/repository matched case-insensitively) unless `operation.reuseExisting` is `false`: `created:false`, a `note`, its stored `target`, `headMoved`, and `working`/`otherReviewIds` when they apply; its target is never moved. A new review reports `created:true`. Either way the result carries `review`, the review's `GET /` catalog entry (target, origin, repository name and path). An interactive `create` also opens the new review in an attached Desktop unless `operation.open` is `false`, and reports `opened` with the open result or an `openError`; the review is saved either way | | `POST /commands {operation:{type:"lens_edit",reviewId,edit}}` | Write one file lens, credited to `activityId` when given: `insert {title,targets,afterId?}` (host id `lens-N`), `update {targetId,title?,targets?}` or `remove {targetId}`. Returns `{targetId, type:"lens", uncategorized}`, where `uncategorized` lists changed files and ranges no lens covers yet (at most 50 files) | | `POST /:id/person-edit {baseVersion,title,document}` | Save a person's whole board as their version; return `{reviewId,version,ids}`. See Person edits | diff --git a/packages/review/src/review-api/agent-cli.ts b/packages/review/src/review-api/agent-cli.ts index 2c3c9cc107..ebeffed873 100644 --- a/packages/review/src/review-api/agent-cli.ts +++ b/packages/review/src/review-api/agent-cli.ts @@ -18,7 +18,7 @@ import { } from "./agent-client.js"; import { type ReviewApiClient, ReviewApiError } from "./client.js"; import { exitOnInstallUpdate } from "./install-update.js"; -import { callPublicTool, publicTool } from "./public-tools.js"; +import { callPublicTool, publicTool, renderedImage } from "./public-tools.js"; import { RECOVERY } from "./recovery.js"; import { ASK_THREAD_ENV, @@ -212,7 +212,8 @@ export async function runReviewAgentCli(input: AgentCliInput): Promise { durationMs: Date.now() - startedAt, }); - const text = toolResultText(tool, result); + // A rendered frame is read from its file; its bytes stay out of stdout. + const text = toolResultText(tool, renderedImage(tool, result).result); input.stdout.write(text.endsWith("\n") ? text : text + "\n"); return 0; diff --git a/packages/review/src/review-api/animation-render.test.ts b/packages/review/src/review-api/animation-render.test.ts new file mode 100644 index 0000000000..9d009ac9e4 --- /dev/null +++ b/packages/review/src/review-api/animation-render.test.ts @@ -0,0 +1,317 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; + +import { devReviewHome } from "@dev.fast/trace-core"; +import { describe, expect, it } from "vitest"; + +import { type AuthoringTool, ToolText } from "./agent-client.js"; +import { type RenderRequest, renderTarget } from "./animation-render.js"; +import { ReviewApiClient } from "./client.js"; +import { createReviewApi } from "./http.js"; +import { callPublicTool, publicTool, renderedImage } from "./public-tools.js"; +import { ReviewStore } from "./store.js"; + +// A 1x1 PNG: what a Desktop would answer, base64. +const PNG = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="; + +const animation = { + type: "animation", + title: "Retry", + html: "

Retry

", + frames: [ + { key: "a1", title: "Attempt 1" }, + { key: "a2", title: "Attempt 2" }, + { key: "ok", title: "Succeeds" }, + { key: "fail", title: "Gives up" }, + ], + scenarios: [ + { key: "succeeds", label: "Succeeds", frames: ["a1", "ok"] }, + { key: "gives-up", label: "Gives up", frames: ["a1", "a2", "fail"] }, + ], +}; + +const clip = { + type: "clip", + title: "Trip", + html: "

Trip

", + markers: [ + { key: "sent", title: "Sent" }, + { key: "stored", title: "Stored" }, + ], +}; + +function openStore() { + return new ReviewStore(":memory:", { + validatePins: async () => {}, + validateSource: async () => {}, + validateResource: async () => {}, + }); +} + +async function board(store: ReviewStore) { + const { reviewId } = await store.execute({ + operation: { + type: "create", + title: "Animations", + target: { + kind: "commits", + repositoryId: "repo", + base: "base", + head: "head", + }, + }, + }); + + const ids: string[] = []; + + for (const content of [ + animation, + clip, + { type: "markdown", markdown: "x" }, + ]) { + const result = await store.execute({ + operation: { type: "edit", reviewId, edit: { type: "insert", content } }, + }); + + ids.push(result.targetId!); + } + + const [animationId, clipId, markdownId] = ids as [string, string, string]; + + return { reviewId, animationId, clipId, markdownId }; +} + +describe("renderTarget", () => { + it("takes a frame of an animation, a marker of a clip, or the first stop", async () => { + const store = openStore(); + + try { + const { reviewId, animationId, clipId, markdownId } = await board(store); + const snapshot = store.read(reviewId); + + expect(renderTarget(snapshot, animationId, "a2")).toEqual({ + type: "animation", + key: "a2", + }); + // The first scenario's first frame, not the first frame declared. + expect(renderTarget(snapshot, animationId, undefined)).toEqual({ + type: "animation", + key: "a1", + }); + expect(renderTarget(snapshot, clipId, undefined)).toEqual({ + type: "clip", + key: "sent", + }); + expect(() => renderTarget(snapshot, animationId, "a9")).toThrow( + "a9 is not a frame of " + + animationId + + ". Its frames: a1, a2, ok, fail.", + ); + expect(() => renderTarget(snapshot, clipId, "a1")).toThrow( + "Its markers: sent, stored.", + ); + expect(() => renderTarget(snapshot, markdownId, undefined)).toThrow( + "is a markdown; only an animation or a clip renders a frame.", + ); + expect(() => renderTarget(snapshot, "missing", undefined)).toThrow( + "Target not found", + ); + } finally { + store.close(); + } + }); +}); + +describe("GET /:id/render", () => { + it("asks Desktop for the stop and answers its capture", async () => { + const store = openStore(); + const requests: RenderRequest[] = []; + + const app = createReviewApi( + store, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + () => true, + async (request) => { + requests.push(request); + + return { + frame: { + key: request.frame, + title: "Attempt 2", + position: 1, + count: 3, + }, + scenario: "gives-up", + clipped: false, + error: null, + png: PNG, + }; + }, + ); + + try { + const { reviewId, animationId, clipId } = await board(store); + + const rendered = await app.request( + `/${reviewId}/render?targetId=${animationId}&frame=a2`, + ); + + expect(rendered.status).toBe(200); + expect(await rendered.json()).toEqual({ + reviewId, + targetId: animationId, + type: "animation", + frame: { key: "a2", title: "Attempt 2", position: 1, count: 3 }, + scenario: "gives-up", + clipped: false, + error: null, + png: PNG, + }); + expect(requests).toEqual([ + { reviewId, targetId: animationId, frame: "a2" }, + ]); + + // Desktop is not asked for a stop the block does not have. + const missing = await app.request( + `/${reviewId}/render?targetId=${clipId}&frame=a2`, + ); + + expect(missing.status).toBe(400); + expect(await missing.json()).toEqual({ + error: `a2 is not a marker of ${clipId}. Its markers: sent, stored.`, + }); + expect(requests).toHaveLength(1); + + const noTarget = await app.request(`/${reviewId}/render`); + expect(noTarget.status).toBe(400); + } finally { + store.close(); + } + }); + + it("refuses without a Desktop to render in", async () => { + const store = openStore(); + const app = createReviewApi(store); + + try { + const { reviewId, animationId } = await board(store); + + const response = await app.request( + `/${reviewId}/render?targetId=${animationId}`, + ); + + expect(response.status).toBe(409); + expect(await response.json()).toEqual({ + error: + "Rendering a frame needs Whiteboard Desktop, which this server cannot reach.", + }); + } finally { + store.close(); + } + }); +}); + +describe("session_render", () => { + it("is listed with animation authoring, writes the PNG beside the drafts, and hands the image to the agent", async () => { + const store = openStore(); + let enabled = false; + + const app = createReviewApi( + store, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + () => enabled, + async (request) => ({ + frame: { key: request.frame, title: "Sent", position: 0, count: 2 }, + clipped: true, + error: "boom", + png: PNG, + }), + ); + + const client = new ReviewApiClient( + { serverUrl: "http://review.test", token: "test" }, + async (url, init) => app.request(url.replace("/reviews-api", ""), init), + ); + + const catalog = async () => + (await client.read("/authoring")).map(publicTool); + + try { + const { reviewId, clipId } = await board(store); + + expect((await catalog()).map((tool) => tool.name)).not.toContain( + "session_render", + ); + enabled = true; + + const tool = (await catalog()).find( + (tool) => tool.name === "session_render", + )!; + + expect(tool.description).toContain("session_open"); + expect(tool.inputSchema.required).toEqual(["sessionId", "targetId"]); + + const result = await callPublicTool(client, tool, { + sessionId: reviewId, + targetId: clipId, + }); + + const file = path.join( + devReviewHome(), + "animation-renders", + reviewId, + clipId, + "sent.png", + ); + + expect(result).toEqual({ + sessionId: reviewId, + targetId: clipId, + type: "clip", + frame: { key: "sent", title: "Sent", position: 0, count: 2 }, + clipped: true, + error: "boom", + png: PNG, + file, + }); + expect((await readFile(file)).toString("base64")).toBe(PNG); + + // MCP shows the PNG as an image; the text keeps the rest. + const split = renderedImage(tool, result); + expect(split.image).toEqual({ data: PNG, mimeType: "image/png" }); + expect(split.result).toEqual({ + sessionId: reviewId, + targetId: clipId, + type: "clip", + frame: { key: "sent", title: "Sent", position: 0, count: 2 }, + clipped: true, + error: "boom", + file, + }); + + // Other tools' results pass through, text included. + const text = new ToolText("plain"); + expect(renderedImage(tool, text)).toEqual({ result: text }); + expect(renderedImage({ name: "session_get" }, { png: PNG })).toEqual({ + result: { png: PNG }, + }); + } finally { + store.close(); + } + }); +}); diff --git a/packages/review/src/review-api/animation-render.ts b/packages/review/src/review-api/animation-render.ts new file mode 100644 index 0000000000..dc924bf453 --- /dev/null +++ b/packages/review/src/review-api/animation-render.ts @@ -0,0 +1,106 @@ +// An agent's look at an animation it wrote: one frame of it (or a clip at +// one marker), shown by the attached Desktop as the reader sees it and +// captured as a PNG. The server checks the stop against the document; the +// Desktop shows and captures it. + +import { + type Element, + ReviewInputError, + elements, +} from "@dev.fast/whiteboard-document"; +import { z } from "zod"; + +import type { Snapshot } from "./store.js"; + +export const renderQuerySchema = z.object({ + targetId: z.string().min(1), + /** A frame key, or a clip's marker key; omitted, the first stop. */ + frame: z.string().min(1).optional(), +}); + +/** What the Desktop answers: the stop it showed, and the figure's PNG. */ +export const renderedFrameSchema = z.object({ + frame: z.object({ + key: z.string(), + title: z.string(), + position: z.number().int().nonnegative(), + count: z.number().int().positive(), + }), + scenario: z.string().optional(), + /** The figure ran past the window, so the capture cuts it. */ + clipped: z.boolean(), + /** The error the stage reported, if its script threw. */ + error: z.string().nullable(), + /** The capture, base64. */ + png: z.string().min(1), +}); + +export type RenderedFrame = z.infer; + +export interface RenderRequest { + reviewId: string; + targetId: string; + frame: string; +} + +/** The host's way to a Desktop that shows the frame and captures it. */ +export type RenderAnimation = ( + request: RenderRequest, +) => Promise; + +/** The stop a render shows: a frame of an animation, or a marker of a clip. */ +export interface RenderTarget { + type: "animation" | "clip"; + key: string; +} + +type StageBlock = Extract; + +const isStageBlock = (element: Element): element is StageBlock => + element.type === "animation" || element.type === "clip"; + +/** The stop to render: `frame` checked against the block's frames or markers, or + * the first stop of the first scenario when omitted. */ +export function renderTarget( + snapshot: Snapshot, + targetId: string, + frame: string | undefined, +): RenderTarget { + const target = elements(snapshot.document).find( + (element) => element.id === targetId, + ); + + if (!target) + throw new ReviewInputError("Target not found in this version.", 404); + + if (!isStageBlock(target)) + throw new ReviewInputError( + `${targetId} is a ${target.type}; only an animation or a clip renders a frame.`, + ); + + const noun = target.type === "clip" ? "marker" : "frame"; + + const keys = + target.type === "clip" + ? target.markers.map((marker) => marker.key) + : target.frames.map((item) => item.key); + + if (frame === undefined) { + const first = + target.type === "animation" + ? (target.scenarios?.[0]?.frames[0] ?? keys[0]) + : keys[0]; + + if (first === undefined) + throw new ReviewInputError(`${targetId} has no ${noun}s to render.`); + + return { type: target.type, key: first }; + } + + if (!keys.includes(frame)) + throw new ReviewInputError( + `${frame} is not a ${noun} of ${targetId}. Its ${noun}s: ${keys.join(", ")}.`, + ); + + return { type: target.type, key: frame }; +} diff --git a/packages/review/src/review-api/authoring-tools.ts b/packages/review/src/review-api/authoring-tools.ts index fde3260a69..27c8677b47 100644 --- a/packages/review/src/review-api/authoring-tools.ts +++ b/packages/review/src/review-api/authoring-tools.ts @@ -182,6 +182,13 @@ export function authoringTools( "GET", "/:reviewId/inspect", ), + tool( + "render", + "See an animation or clip as the reader does: Desktop shows one of its frames (frame: a frame key, or a clip's marker key; omitted, the first) on the open whiteboard and returns it as a PNG image, with the error its script threw, if any. The PNG is also written to the path the result names. After publishing, render each frame you changed and look at it, instead of taking screenshots of the screen. The whiteboard must be open in Desktop: review_create opens it, and review_open opens an existing one.", + z.strictObject({ ...review, targetId: id, frame: id.optional() }), + "GET", + "/:reviewId/render", + ), ] : []), tool("list", "List saved reviews.", z.strictObject({}), "GET", ""), diff --git a/packages/review/src/review-api/http.ts b/packages/review/src/review-api/http.ts index c223314437..fb211b05f4 100644 --- a/packages/review/src/review-api/http.ts +++ b/packages/review/src/review-api/http.ts @@ -67,6 +67,11 @@ import { type Context, Hono, type MiddlewareHandler } from "hono"; import { z } from "zod"; import { anchorQuotes } from "./anchor-quotes.js"; +import { + type RenderAnimation, + renderQuerySchema, + renderTarget, +} from "./animation-render.js"; import type { AskRecord } from "./ask-history.js"; import { authoringTools } from "./authoring-tools.js"; import { documentText } from "./document-text.js"; @@ -207,6 +212,8 @@ export function createReviewApi( /** Desktop's Ask: local agents answering questions about a selection. */ ask?: AskHost, animationsEnabled: () => boolean = () => false, + /** Shows a frame of an animation in the attached Desktop and captures it. */ + render?: RenderAnimation, ) { const app = new Hono(); app.onError((error, context) => { @@ -2085,6 +2092,32 @@ export function createReviewApi( }, ); }); + // One frame of an animation (or a clip at a marker) as the reader sees it: + // the attached Desktop shows it on the open board and captures it. + app.get("/:id/render", async (context) => { + const query = renderQuerySchema.parse(context.req.query()); + const id = context.req.param("id"); + const target = renderTarget(readReview(id), query.targetId, query.frame); + + if (!render) + throw new ReviewInputError( + "Rendering a frame needs Whiteboard Desktop, which this server cannot reach.", + 409, + ); + + const rendered = await render({ + reviewId: id, + targetId: query.targetId, + frame: target.key, + }); + + return context.json({ + reviewId: id, + targetId: query.targetId, + type: target.type, + ...rendered, + }); + }); app.get("/:id", async (context) => { const query = readQuerySchemas.get.parse(context.req.query()); diff --git a/packages/review/src/review-api/mcp.ts b/packages/review/src/review-api/mcp.ts index acc1020dc1..bbdda26ba1 100644 --- a/packages/review/src/review-api/mcp.ts +++ b/packages/review/src/review-api/mcp.ts @@ -17,7 +17,7 @@ import { authoringTools } from "./authoring-tools.js"; import type { ReviewApiClient } from "./client.js"; import { ReviewApiError } from "./client.js"; import { mcpClientAgent } from "./mcp-client-agent.js"; -import { callPublicTool, publicTool } from "./public-tools.js"; +import { callPublicTool, publicTool, renderedImage } from "./public-tools.js"; import { RECOVERY } from "./recovery.js"; import { REVIEW_STATUS_TOOL } from "./status-tool.js"; import { type ToolFailure, toolFailure } from "./tool-failure.js"; @@ -191,11 +191,14 @@ export async function serveReviewMcp( `Unknown Whiteboard tool: ${request.params.name}`, ); - const result = await callPublicTool( - client, + const { image, result } = renderedImage( tool, - request.params.arguments ?? {}, - ctx.mcpReq.signal, + await callPublicTool( + client, + tool, + request.params.arguments ?? {}, + ctx.mcpReq.signal, + ), ); const text = toolResultText(tool, result); @@ -203,6 +206,8 @@ export async function serveReviewMcp( const response: CallToolResult = { content: [ + // The frame an agent rendered, ahead of what the host says about it. + ...(image ? [{ type: "image" as const, ...image }] : []), { type: "text", text: diff --git a/packages/review/src/review-api/public-tools.ts b/packages/review/src/review-api/public-tools.ts index fd512224fe..2407d056f4 100644 --- a/packages/review/src/review-api/public-tools.ts +++ b/packages/review/src/review-api/public-tools.ts @@ -1,5 +1,9 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; + import { type JsonValue, isJsonObject, isStringValue } from "@dev.fast/json"; import { boardTitle } from "@dev.fast/review-protocol"; +import { devReviewHome } from "@dev.fast/trace-core"; import { ReviewInputError } from "@dev.fast/whiteboard-document"; import { @@ -171,6 +175,13 @@ export async function callPublicTool( if (result instanceof ToolText) return result; const published = withoutRepositoryIds(publicResult(result)); + if ( + tool.name === "session_render" && + isStringValue(sessionId) && + isStringValue(rest.targetId) + ) + return saveRenderedFrame(published, sessionId, rest.targetId); + if ( draft && isStringValue(sessionId) && @@ -189,6 +200,64 @@ export async function callPublicTool( return published; } +/** A tool result with its rendered frame, if any, taken out as an image. */ +export interface RenderedImage { + image?: { data: string; mimeType: "image/png" }; + result: JsonValue | ToolText; +} + +/** A rendered frame's PNG, for an agent that reads images from a tool + * result, and the rest of the result for its text. */ +export function renderedImage( + tool: Pick, + result: JsonValue | ToolText, +): RenderedImage { + if ( + tool.name !== "session_render" || + result instanceof ToolText || + !isJsonObject(result) || + !isStringValue(result.png) + ) + return { result }; + + const { png, ...rest } = result; + + return { image: { data: png, mimeType: "image/png" }, result: rest }; +} + +/** Writes the frame's PNG where the agent can open it with its own tools, + * beside the animation drafts, and names the file in the result. A folder + * that cannot be written leaves the image in the result alone. */ +async function saveRenderedFrame( + result: JsonValue, + sessionId: string, + targetId: string, +): Promise { + if (!isJsonObject(result) || !isStringValue(result.png)) return result; + + const key = + isJsonObject(result.frame) && isStringValue(result.frame.key) + ? result.frame.key + : "frame"; + + const file = path.join( + devReviewHome(), + "animation-renders", + ...[sessionId, targetId, `${key}.png`].map((part) => + part.replace(/[^\w.-]/g, "_"), + ), + ); + + try { + await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, Buffer.from(result.png, "base64")); + } catch { + return result; + } + + return { ...result, file }; +} + /** The read a draft compares and records against: session_get's JSON. */ const INSPECT: AuthoringTool = { name: "review_get", diff --git a/packages/review/src/server/render-relay.test.ts b/packages/review/src/server/render-relay.test.ts new file mode 100644 index 0000000000..f8e54709c5 --- /dev/null +++ b/packages/review/src/server/render-relay.test.ts @@ -0,0 +1,81 @@ +import type { JsonValue, ReviewVerbResponse } from "@dev.fast/review-protocol"; +import { describe, expect, it } from "vitest"; + +import type { ReviewDesktopVerbRelay } from "./global-verb-relay.js"; +import { relayReviewCallbacks } from "./review-server-core.js"; + +const request = { + reviewId: "8b0d2d6e-4f0a-4c0c-9a2e-1f9c1c1c1c1c", + targetId: "block-3", + frame: "a2", +}; + +function fakeRelay( + attached: boolean, + answer: (value: JsonValue) => ReviewVerbResponse, +): ReviewDesktopVerbRelay & { sent: JsonValue[] } { + const sent: JsonValue[] = []; + + return { + sent, + attached, + attach: () => true, + acceptResult: () => true, + close() {}, + onAttachedChange: () => () => {}, + async dispatch(value) { + sent.push(value); + + return answer(value); + }, + }; +} + +describe("relayReviewCallbacks.render", () => { + it("asks the attached Desktop and keeps what the agent reads", async () => { + const relay = fakeRelay(true, () => ({ + ok: true, + result: { + frame: { key: "a2", title: "Attempt 2", position: 1, count: 3 }, + scenario: "gives-up", + rect: { x: 10, y: 20, width: 300, height: 200 }, + clipped: false, + error: null, + png: "aGk=", + }, + })); + + expect(await relayReviewCallbacks(relay).render(request)).toEqual({ + frame: { key: "a2", title: "Attempt 2", position: 1, count: 3 }, + scenario: "gives-up", + clipped: false, + error: null, + png: "aGk=", + }); + expect(relay.sent).toEqual([{ name: "renderAnimation", args: request }]); + }); + + it("reports Desktop's refusal, and a missing Desktop, as 409s", async () => { + const refused = fakeRelay(true, () => ({ + ok: false, + error: "The whiteboard is not open in Desktop.", + })); + + await expect( + relayReviewCallbacks(refused).render(request), + ).rejects.toMatchObject({ + status: 409, + message: "The whiteboard is not open in Desktop.", + }); + + const detached = fakeRelay(false, () => ({ ok: true })); + + await expect( + relayReviewCallbacks(detached).render(request), + ).rejects.toMatchObject({ + status: 409, + message: expect.stringContaining("No Whiteboard Desktop is attached"), + }); + expect(detached.sent).toEqual([]); + }); +}); diff --git a/packages/review/src/server/review-server-core.ts b/packages/review/src/server/review-server-core.ts index a6cb59a5f5..c0eda2a170 100644 --- a/packages/review/src/server/review-server-core.ts +++ b/packages/review/src/server/review-server-core.ts @@ -16,6 +16,11 @@ import { readBuildCommit, readReviewPackageVersion, } from "@review/package-paths.js"; +import { + type RenderRequest, + type RenderedFrame, + renderedFrameSchema, +} from "@review/review-api/animation-render.js"; import { type AuthoringCapabilities, type ReviewApiHooks, @@ -95,7 +100,8 @@ export function createReviewServerApp(input: { mountFeedback(app); app.post("/control/result", async (context) => { const accepted = input.relay.acceptResult( - await readBoundedRequestJson(context.req.raw), + // A rendered frame's PNG rides back in a result. + await readBoundedRequestJson(context.req.raw, VERB_RESULT_MAX_BYTES), ); return serverJson(accepted ? 200 : 404, { ok: accepted }); @@ -144,6 +150,10 @@ export interface WhiteboardCoreInput { }; } +/** A verb result may carry a captured frame: a PNG of up to 1600 pixels on + * its longest edge, base64. */ +const VERB_RESULT_MAX_BYTES = 16 * 1024 * 1024; + export function createWhiteboardCore(input: WhiteboardCoreInput) { const { store, data } = input.profile; @@ -196,6 +206,7 @@ export function createWhiteboardCore(input: WhiteboardCoreInput) { agents: async () => detectAskAgents(await ask?.env), }, input.animations, + callbacks.render, ); // Desktop's sharing host mounts the publisher with the rest of sharing. @@ -247,6 +258,24 @@ export function relayReviewCallbacks( ...z.object({ softwareMapEnabled: z.boolean() }).parse(result.result), }; }, + async render(request: RenderRequest): Promise { + if (!relay.attached) + throw new ReviewInputError( + "No Whiteboard Desktop is attached, so nothing can show the animation. Open the whiteboard in Desktop and render again.", + 409, + ); + + const { reviewId, targetId, frame } = request; + + const result = await relay.dispatch({ + name: "renderAnimation", + args: { reviewId, targetId, frame }, + }); + + if (!result.ok) throw new ReviewInputError(result.error, 409); + + return renderedFrameSchema.parse(result.result); + }, }; } From c2f1fc0b0efd532fe94540286d0cb46c528d5a1d Mon Sep 17 00:00:00 2001 From: Milan Bhandari Date: Tue, 3 Nov 2026 16:06:18 -0500 Subject: [PATCH 2/2] Keep the blank line lint wants between the stage scale bounds Agent-Session: cbacb871-2558-4fe4-9272-77524bb21683 Agent-Session: 62100749-c77c-4d19-ad48-c6d4ba1e867b Agent-Session: b315a4f8-1333-461c-9be6-366ceb231b7f Agent-Session: 7bc797d9-b46c-48dc-b281-4fce85dc15ad Agent-Session: b5f9f841-35c5-4ec1-95a8-6485f64d254b Agent-Session: 32e129b9-586e-4fd2-8961-e7224c5978ad --- packages/review/app/src/animation/animation-block.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/review/app/src/animation/animation-block.tsx b/packages/review/app/src/animation/animation-block.tsx index 823454e1f4..6ed1ffcda6 100644 --- a/packages/review/app/src/animation/animation-block.tsx +++ b/packages/review/app/src/animation/animation-block.tsx @@ -92,6 +92,7 @@ const MEASURE_WAIT = 1000; // at most 2x, past which a drawing made for a column reads as a poster, and // at least 0.4x, below which its text cannot be read and the room scrolls. const STAGE_MAX_SCALE = 2; + const STAGE_MIN_SCALE = 0.4; // How long a render waits for the stage to boot, paint, show the stop asked