improvement(desktop): make saved password autofill field-aware

Author: waleedlatif1

apps/desktop/e2e/password-autofill.spec.ts

@@ -102,6 +102,7 @@ const renderer: BuildOptions = {
     server: 'src/renderer/server/index.tsx',
     offline: 'src/renderer/offline/index.tsx',
     dialog: 'src/renderer/dialog/index.tsx',
+    'credential-picker': 'src/renderer/credential-picker/index.tsx',
   },
   outdir: 'dist/renderer',
   bundle: true,
@@ -146,6 +147,11 @@ async function run(): Promise<void> {
       entryPoints: ['src/preload/shell.ts'],
       outfile: 'dist/shell-preload.cjs',
     })
+    const credentialPickerPreloadCtx = await context({
+      ...common,
+      entryPoints: ['src/preload/credential-picker.ts'],
+      outfile: 'dist/credential-picker-preload.cjs',
+    })
     const mainCtx = await context({
       ...common,
       entryPoints: ['src/main/index.ts'],
@@ -169,11 +175,17 @@ async function run(): Promise<void> {
       browserPreloadCtx.watch(),
       rendererCtx.watch(),
       shellPreloadCtx.watch(),
+      credentialPickerPreloadCtx.watch(),
     ])
     return
   }
   await Promise.all([
     build(renderer),
+    build({
+      ...common,
+      entryPoints: ['src/preload/credential-picker.ts'],
+      outfile: 'dist/credential-picker-preload.cjs',
+    }),
     build({ ...common, entryPoints: ['src/preload/shell.ts'], outfile: 'dist/shell-preload.cjs' }),
     build({ ...common, entryPoints: ['src/main/index.ts'], outfile: 'dist/main.cjs' }),
     build({ ...common, entryPoints: ['src/preload/index.ts'], outfile: 'dist/preload.cjs' }),

apps/desktop/scripts/build.ts

@@ -1116,7 +1116,14 @@ describe('executeTool', () => {
       | MenuItemConstructorOptions[]
       | undefined
     const labels = template?.filter((item) => item.type !== 'separator').map((item) => item.label)
-    expect(labels).toEqual(['Find in Page', 'Zoom (110%)', 'Import Passwords', 'Browser Settings'])
+    expect(labels).toEqual([
+      'Find in Page',
+      'Zoom (110%)',
+      'Fill Saved Password',
+      'Passwords',
+      'Import Passwords',
+      'Browser Settings',
+    ])
 
     const settings = template?.find((item) => item.label === 'Browser Settings')
     const openSettings = settings?.click as (() => void) | undefined

apps/desktop/src/main/browser-agent/driver.test.ts

@@ -69,11 +69,13 @@ import {
   setFocusedInputValue,
   typeIntoElement,
 } from '@/main/browser-agent/page-functions'
+import { isPanelVisible, panelWindow } from '@/main/browser-agent/panel'
 import { withPostActionObservation } from '@/main/browser-agent/post-action-observation'
 import * as session from '@/main/browser-agent/session'
 import { checkAgentUrl } from '@/main/browser-agent/url-guard'
 import { clearCredentials, fillCoordinator, initFillCoordinator } from '@/main/browser-credentials'
 import type { ConfigStore } from '@/main/config'
+import { trackInputActivity } from '@/main/input-activity'
 
 const logger = createLogger('BrowserAgentDriver')
 
@@ -506,6 +508,7 @@ function pushTabsState(): void {
 
 /** Instruments a fresh tab: CDP dialog handling + page-state pushes. */
 function instrumentTab(contents: WebContents): void {
+  trackInputActivity(contents)
   const scopeId = session.browserScopeIdForContents(contents) ?? session.getBrowserScopeId()
   const inScope =
     <Args extends unknown[]>(fn: (...args: Args) => void) =>
@@ -625,6 +628,35 @@ export function initDriver(
   // leaving the old chain head in place would queue the new session's first
   // tool call behind a promise nothing can ever settle.
   initFillCoordinator({
+    pickerHost: (contents, bounds) => {
+      const scopeId = session.getActiveBrowserScopeId()
+      const window = panelWindow()
+      if (!scopeId || !window || window.isDestroyed() || !window.isVisible() || !isPanelVisible())
+        return null
+      return session.withBrowserScope(scopeId, () => {
+        const tab = session.activeTab()
+        if (!tab || tab.view.webContents !== contents || !tab.view.getVisible()) return null
+        const panel = tab.view.getBounds()
+        const content = window.getContentBounds()
+        const zoom = contents.getZoomFactor()
+        if (
+          bounds.x + bounds.width <= 0 ||
+          bounds.y + bounds.height <= 0 ||
+          bounds.x * zoom >= panel.width ||
+          bounds.y * zoom >= panel.height
+        )
+          return null
+        return {
+          window,
+          anchor: {
+            x: content.x + panel.x + bounds.x * zoom,
+            y: content.y + panel.y + bounds.y * zoom,
+            width: bounds.width * zoom,
+            height: bounds.height * zoom,
+          },
+        }
+      })
+    },
     getActiveContents: (scopeId) => {
       const activeScopeId = session.getActiveBrowserScopeId()
       if (!activeScopeId) return null
@@ -646,6 +678,7 @@ export function initDriver(
   })
   session.initSession(
     {
+      onPanelGeometryChanged: () => fillCoordinator()?.dismissPicker(),
       onSessionClosed: () => {
         driverCallbacks?.onSessionStatus(false, session.getBrowserScopeId())
       },
@@ -748,6 +781,14 @@ export function showToolbarMenu(
       ],
     },
     { type: 'separator' },
+    {
+      label: 'Fill Saved Password',
+      enabled: pageAvailable,
+      click: () => {
+        void fillCoordinator()?.showChooser(ownerWindow, anchor, resolved)
+      },
+    },
+    { label: 'Passwords', click: () => sendCommand('passwords') },
     { label: 'Import Passwords', click: () => sendCommand('import') },
     { type: 'separator' },
     { label: 'Browser Settings', click: () => sendCommand('browser-settings') },

apps/desktop/src/main/browser-agent/driver.ts

@@ -45,6 +45,8 @@ export interface PanelHost {
   restoreActiveScope: () => void
   /** Lets the session drop focus tracking for a view that is no longer attached. */
   onViewDetached: (view: WebContentsView | null) => void
+  /** Invalidates field-anchored UI when the page moves, hides, or detaches. */
+  onGeometryChanged?: () => void
 }
 
 let host: PanelHost = {
@@ -292,6 +294,7 @@ function detachAttachedView(): void {
   occludableFrame = null
   unbindHostResize()
   host.onViewDetached(view)
+  host.onGeometryChanged?.()
 
   if (!view || !win) return
   try {
@@ -428,11 +431,13 @@ export function layout(): void {
     lastAppliedBounds = boundsKey
     occludableFrame = null
     active.view.setBounds(bounds)
+    host.onGeometryChanged?.()
   }
   const visible = !panelOccluded
   if (lastAppliedVisibility !== visible) {
     lastAppliedVisibility = visible
     active.view.setVisible(visible)
+    host.onGeometryChanged?.()
     if (visible && !active.view.webContents.isDestroyed()) {
       // invalidate() recomposites the LAST frame — which is blank when the
       // page finished loading while this view was hidden and background

apps/desktop/src/main/browser-agent/panel.ts

@@ -4014,8 +4014,9 @@ describe('browser-agent session', () => {
     const retainedDownload = mockDownloadItem({ filename: 'retained.bin', totalBytes: 100 })
 
     startMockDownload(suspendedContents, suspendedDownload)
-    startMockDownload(retainedContents, retainedDownload)
+    await vi.waitFor(() => expect(getFreeDiskBytes).toHaveBeenCalledOnce())
     await vi.waitFor(() => expect(suspendedDownload.item.setSavePath).toHaveBeenCalledOnce())
+    startMockDownload(retainedContents, retainedDownload)
     await vi.waitFor(() => expect(retainedDownload.item.resume).toHaveBeenCalledOnce())
     onDownloadsChanged.mockClear()
 

apps/desktop/src/main/browser-agent/session.test.ts

@@ -139,6 +139,8 @@ export interface BrowserDownloadSettings {
 }
 
 export interface AgentSessionEvents {
+  /** The native page moved or its visibility changed. */
+  onPanelGeometryChanged?: () => void
   /** The browser session ended (all tabs gone). */
   onSessionClosed: () => void
   /** A newly created tab's WebContents, for the driver to instrument. */
@@ -859,6 +861,7 @@ export function initSession(
   browserSessionPersistence = persistence ?? null
   browserDownloadSettings = downloadSettings ?? null
   initPanel({
+    onGeometryChanged: () => events?.onPanelGeometryChanged?.(),
     getMainWindow: () => getMainWindow(),
     activeTab: () => {
       const scopeId = getActiveBrowserScopeId()