v0.9.1: browser agent improvements

Author: waleedlatif1

apps/desktop/src/main/browser-agent/cdp.test.ts

@@ -15,10 +15,13 @@ import {
   captureScreenshot,
   clickAt,
   consumeAgentContextMenu,
+  dragPointer,
   ensureInstrumented,
   evaluateInIsolatedFrame,
   insertText,
+  movePointer,
   PRIMARY_CLICK,
+  pointerPathSteps,
   releaseFileInput,
   resolveFileInput,
   setColorScheme,
@@ -258,6 +261,122 @@ describe('browser-agent CDP instrumentation', () => {
     ])
   })
 
+  it('stops a pointer route as soon as it is aborted', async () => {
+    const contents = new WebContentsView().webContents
+    const moves = () =>
+      vi
+        .mocked(contents.debugger.sendCommand)
+        .mock.calls.filter(([method]) => String(method).startsWith('Input.dispatchMouseEvent'))
+    const aborted = new AbortController()
+    aborted.abort()
+    await expect(
+      movePointer(contents, { via: [], durationMs: null }, { x: 5, y: 5 }, aborted.signal)
+    ).rejects.toMatchObject({ name: 'AbortError' })
+    expect(moves()).toHaveLength(0)
+
+    vi.useFakeTimers()
+    try {
+      const controller = new AbortController()
+      const route = movePointer(
+        contents,
+        { via: [{ x: 0, y: 0 }], durationMs: 5_000 },
+        { x: 500, y: 0 },
+        controller.signal
+      )
+      const settled = expect(route).rejects.toMatchObject({ name: 'AbortError' })
+      await vi.advanceTimersByTimeAsync(100)
+      const sentBeforeAbort = moves().length
+      controller.abort()
+      await settled
+      expect(moves()).toHaveLength(sentBeforeAbort)
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
+  it('sends nothing for an already-aborted drag and cancels one aborted while it settles', async () => {
+    const contents = new WebContentsView().webContents
+    const mouse = () =>
+      vi
+        .mocked(contents.debugger.sendCommand)
+        .mock.calls.filter(([method]) => method === 'Input.dispatchMouseEvent')
+        .map(([, params]) => toRecord(params).type)
+    const aborted = new AbortController()
+    aborted.abort()
+    await expect(
+      dragPointer(contents, { x: 0, y: 0 }, { x: 50, y: 0 }, undefined, aborted.signal)
+    ).rejects.toMatchObject({ name: 'AbortError' })
+    expect(contents.debugger.sendCommand).not.toHaveBeenCalled()
+
+    vi.useFakeTimers()
+    try {
+      const controller = new AbortController()
+      const drag = dragPointer(
+        contents,
+        { x: 0, y: 0 },
+        { x: 50, y: 0 },
+        undefined,
+        controller.signal
+      )
+      const settled = expect(drag).rejects.toMatchObject({ name: 'AbortError' })
+      // The default route takes 13 moves 20 ms apart, then a 120 ms settle hold.
+      await vi.advanceTimersByTimeAsync(300)
+      controller.abort()
+      await settled
+      expect(mouse().at(-1)).toBe('mouseReleased')
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
+  it('releases the button when a timed drag is aborted mid-route', async () => {
+    const contents = new WebContentsView().webContents
+    const types = () =>
+      vi
+        .mocked(contents.debugger.sendCommand)
+        .mock.calls.filter(([method]) => method === 'Input.dispatchMouseEvent')
+        .map(([, params]) => toRecord(params).type)
+    vi.useFakeTimers()
+    try {
+      const controller = new AbortController()
+      const drag = dragPointer(
+        contents,
+        { x: 0, y: 0 },
+        { x: 500, y: 0 },
+        { via: [], durationMs: 5_000 },
+        controller.signal
+      )
+      const settled = expect(drag).rejects.toMatchObject({ name: 'AbortError' })
+      await vi.advanceTimersByTimeAsync(100)
+      controller.abort()
+      await settled
+      expect(types().at(-1)).toBe('mouseReleased')
+      expect(types().filter((type) => type === 'mouseMoved').length).toBeLessThan(20)
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
+  it('keeps the default drag pace and lands exactly on every via point', () => {
+    const direct = pointerPathSteps({ x: 0, y: 0 }, { via: [], durationMs: null }, { x: 120, y: 0 })
+    expect(direct.stepDelayMs).toBe(20)
+    expect(direct.points).toHaveLength(12)
+    expect(direct.points[0]).toEqual({ x: 10, y: 0 })
+    expect(direct.points[11]).toEqual({ x: 120, y: 0 })
+
+    const routed = pointerPathSteps(
+      { x: 0, y: 0 },
+      { via: [{ x: 100, y: 0 }], durationMs: 800 },
+      { x: 100, y: 300 }
+    )
+    expect(routed.points).toContainEqual({ x: 100, y: 0 })
+    expect(routed.points[routed.points.length - 1]).toEqual({ x: 100, y: 300 })
+    expect(routed.points.length * routed.stepDelayMs).toBeCloseTo(800)
+    // The longer second segment gets about three times the steps of the first.
+    const corner = routed.points.findIndex((point) => point.x === 100 && point.y === 0)
+    expect(routed.points.length - 1 - corner).toBeGreaterThan(corner * 2)
+  })
+
   it('holds the button down for holdMs before releasing it', async () => {
     const contents = new WebContentsView().webContents
     const types = () =>

apps/desktop/src/main/browser-agent/cdp.ts

@@ -11,7 +11,7 @@
 import type { BrowserTheme } from '@sim/browser-protocol'
 import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
-import { interruptibleSleep, sleep } from '@sim/utils/helpers'
+import { interruptibleSleep } from '@sim/utils/helpers'
 import { isRecordLike } from '@sim/utils/object'
 import type { NativeImage, WebContents, WebFrameMain } from 'electron'
 
@@ -948,6 +948,91 @@ export async function moveMouse(contents: WebContents, x: number, y: number): Pr
   })
 }
 
+/** A point in CSS viewport pixels. */
+export interface ViewportPoint {
+  x: number
+  y: number
+}
+
+/** The points a pointer passes through on its way, and how long the whole movement takes. */
+export interface PointerPath {
+  via: ViewportPoint[]
+  /** Total movement time; null keeps the default brisk pace. */
+  durationMs: number | null
+}
+
+export const DIRECT_PATH: PointerPath = { via: [], durationMs: null }
+
+const DEFAULT_PATH_STEPS = 12
+const DEFAULT_PATH_STEP_MS = 20
+/** One display frame, so a timed movement looks continuous to animation-driven pages. */
+const TIMED_PATH_STEP_MS = 16
+
+/**
+ * The moves from `from` through `path.via` to `to`, and the pause after each. A direct path
+ * keeps the default 12 moves 20 ms apart. A path with via points or a duration moves one frame
+ * at a time, shares the steps across segments by length, and lands exactly on every via point.
+ */
+export function pointerPathSteps(
+  from: ViewportPoint,
+  path: PointerPath,
+  to: ViewportPoint
+): { points: ViewportPoint[]; stepDelayMs: number } {
+  const lerp = (a: ViewportPoint, b: ViewportPoint, t: number): ViewportPoint => ({
+    x: a.x + (b.x - a.x) * t,
+    y: a.y + (b.y - a.y) * t,
+  })
+  if (path.via.length === 0 && path.durationMs === null) {
+    const points: ViewportPoint[] = []
+    for (let step = 1; step <= DEFAULT_PATH_STEPS; step++) {
+      points.push(lerp(from, to, step / DEFAULT_PATH_STEPS))
+    }
+    return { points, stepDelayMs: DEFAULT_PATH_STEP_MS }
+  }
+  const vertices = [from, ...path.via, to]
+  const durationMs = path.durationMs ?? DEFAULT_PATH_STEPS * DEFAULT_PATH_STEP_MS
+  const lengths = vertices
+    .slice(1)
+    .map((vertex, index) => Math.hypot(vertex.x - vertices[index].x, vertex.y - vertices[index].y))
+  const totalLength = lengths.reduce((sum, length) => sum + length, 0)
+  const totalSteps = Math.max(lengths.length, Math.round(durationMs / TIMED_PATH_STEP_MS))
+  const points: ViewportPoint[] = []
+  lengths.forEach((length, index) => {
+    const share = totalLength > 0 ? length / totalLength : 1 / lengths.length
+    const segmentSteps = Math.max(1, Math.round(totalSteps * share))
+    for (let step = 1; step <= segmentSteps; step++) {
+      points.push(lerp(vertices[index], vertices[index + 1], step / segmentSteps))
+    }
+  })
+  return { points, stepDelayMs: durationMs / points.length }
+}
+
+/**
+ * Moves the pointer with no button pressed through `path.via` to `to`, starting from the first
+ * via point (or `to` itself for a direct move), for hover effects that follow the cursor.
+ */
+export async function movePointer(
+  contents: WebContents,
+  path: PointerPath,
+  to: ViewportPoint,
+  signal?: AbortSignal
+): Promise<void> {
+  const [start, ...rest] = [...path.via, to]
+  signal?.throwIfAborted()
+  await moveMouse(contents, start.x, start.y)
+  if (rest.length === 0) return
+  const { points, stepDelayMs } = pointerPathSteps(
+    start,
+    { via: rest.slice(0, -1), durationMs: path.durationMs },
+    to
+  )
+  for (const point of points) {
+    await interruptibleSleep(stepDelayMs, signal)
+    signal?.throwIfAborted()
+    await moveMouse(contents, point.x, point.y)
+  }
+}
+
 /** One trusted click gesture: which button, how many presses, and held modifiers. */
 export interface PointerClick {
   button: 'left' | 'right' | 'middle'
@@ -1075,11 +1160,13 @@ export async function clickAt(
  */
 export async function dragPointer(
   contents: WebContents,
-  from: { x: number; y: number },
-  to: { x: number; y: number },
-  steps = 12,
-  stepDelayMs = 20
+  from: ViewportPoint,
+  to: ViewportPoint,
+  path: PointerPath = DIRECT_PATH,
+  signal?: AbortSignal
 ): Promise<{ nativeDragIntercepted: boolean }> {
+  signal?.throwIfAborted()
+  const { points, stepDelayMs } = pointerPathSteps(from, path, to)
   const interception: DragInterception = { intercepted: false, data: null }
   dragInterceptionsByContents.set(contents, interception)
   let interceptEnabled = false
@@ -1124,17 +1211,19 @@ export async function dragPointer(
     })
     // Small first nudge so libraries with a start threshold (commonly 3-8px)
     // register the drag before the pointer sweeps across the page.
-    await dragMove(from.x + Math.sign(to.x - from.x || 1) * 4, from.y + 2)
-    await sleep(stepDelayMs)
-    const stepCount = Math.max(2, steps)
-    for (let step = 1; step <= stepCount; step++) {
-      const progress = step / stepCount
-      await dragMove(from.x + (to.x - from.x) * progress, from.y + (to.y - from.y) * progress)
-      await sleep(stepDelayMs)
+    const heading = points[0] ?? to
+    await dragMove(from.x + Math.sign(heading.x - from.x || 1) * 4, from.y + 2)
+    await interruptibleSleep(stepDelayMs, signal)
+    for (const point of points) {
+      signal?.throwIfAborted()
+      await dragMove(point.x, point.y)
+      await interruptibleSleep(stepDelayMs, signal)
     }
+    signal?.throwIfAborted()
     // Hold over the target so drop zones running enter/over animations settle
     // before the release lands.
-    await sleep(120)
+    await interruptibleSleep(120, signal)
+    signal?.throwIfAborted()
     if (interception.intercepted && interception.data) {
       await sendInput(contents, 'Input.dispatchDragEvent', {
         type: 'drop',