feat(mothership): answer a workflow tool's run still running past the worker's wait

A run_workflow* call held the turn until its run settled, up to the execution
limit, while a synchronous workflows run can already detach after
--wait-timeout (#9464). Sim now declares workflow-tool-wait-timeout and honors
the waitTimeoutMs a workflow tool frame names (synced from the mothership
contract):

- Server-run call: the handler gives the run its execution id up front and,
  once the wait passes, answers {status: running, executionId, workflowId}.
  The run goes on in-process; the tool execution keeps holding it, so its
  lease and a Stop through the call's abort signal still reach it.
- Browser-run call: once the wait passes and a browser holds the call's
  execution claim, the call is settled background, the detach a browser sends
  on pagehide. The browser's run goes on, still bound to the call, so a Stop
  cancels it.
- A background workflow completion now carries status: running beside its
  ids, the shape the worker waits on.

A frame without waitTimeoutMs, or one with a malformed value, holds the call
as before.

Author: waleedlatif1

apps/sim/app/api/copilot/confirm/route.test.ts

@@ -1036,7 +1036,7 @@ describe('Copilot Confirm API Route', () => {
       status: 'background',
       message: 'Workflow execution is continuing in the background.',
       timestamp: expect.any(String),
-      data: { workflowId: 'workflow-1', executionId: 'unbound-execution' },
+      data: { workflowId: 'workflow-1', executionId: 'unbound-execution', status: 'running' },
     })
   })
 

apps/sim/lib/mothership/chat/payload.test.ts

@@ -34,6 +34,7 @@ import {
   ChatPayloadSchema,
   TOOL_DEADLINE_CAPABILITY,
   WORKFLOW_RUN_WAIT_TIMEOUT_CAPABILITY,
+  WORKFLOW_TOOL_WAIT_TIMEOUT_CAPABILITY,
 } from '@/lib/mothership/generated/protocol'
 import { searchUsersV2Tool } from '@/tools/github/search_users'
 import { gmailListLabelsV2Tool } from '@/tools/gmail/list_labels'
@@ -430,6 +431,7 @@ describe('buildCopilotRequestPayload', () => {
         'workflow-tool-pickup',
         TOOL_DEADLINE_CAPABILITY,
         WORKFLOW_RUN_WAIT_TIMEOUT_CAPABILITY,
+        WORKFLOW_TOOL_WAIT_TIMEOUT_CAPABILITY,
       ])
     }
   )
@@ -514,6 +516,7 @@ describe('buildCopilotRequestPayload', () => {
       expect(payload.clientCapabilities).toEqual([
         TOOL_DEADLINE_CAPABILITY,
         WORKFLOW_RUN_WAIT_TIMEOUT_CAPABILITY,
+        WORKFLOW_TOOL_WAIT_TIMEOUT_CAPABILITY,
       ])
       if (owner === 'organization') expect(payload).not.toHaveProperty('integrationTools')
     }
@@ -854,6 +857,7 @@ describe('Assistant payload', () => {
     expect(payload.clientCapabilities).toEqual([
       TOOL_DEADLINE_CAPABILITY,
       WORKFLOW_RUN_WAIT_TIMEOUT_CAPABILITY,
+      WORKFLOW_TOOL_WAIT_TIMEOUT_CAPABILITY,
     ])
     expect(payload.mode).toBe('assistant')
     expect(payload.assistantSearch).toEqual({ source: 'slack', documentIds: ['document-1'] })

apps/sim/lib/mothership/chat/payload.ts

@@ -38,6 +38,7 @@ import {
   type ModelSelection,
   TOOL_DEADLINE_CAPABILITY,
   WORKFLOW_RUN_WAIT_TIMEOUT_CAPABILITY,
+  WORKFLOW_TOOL_WAIT_TIMEOUT_CAPABILITY,
 } from '@/lib/mothership/generated/protocol'
 import type { VfsSnapshotV1 } from '@/lib/mothership/generated/vfs-snapshot-v1'
 import { getToolEntry } from '@/lib/mothership/tool-executor/router'
@@ -562,13 +563,15 @@ export async function buildCopilotRequestPayload(params: BuildPayloadParams): Pr
       : {}),
     // The mounted chat view executes client-routed workflow tools (run panel UX), so the
     // UI declares that capability explicitly; headless callers omit it and the server runs
-    // those tools immediately. The deadline and run-wait tokens are this server's own: it
-    // bounds every handed call by the deadline its frame names, and its CLI lets a
-    // synchronous `workflows run` outlast `--wait-timeout`.
+    // those tools immediately. The deadline and wait tokens are this server's own: it
+    // bounds every handed call by the deadline its frame names, its CLI lets a synchronous
+    // `workflows run` outlast `--wait-timeout`, and a workflow tool's run outlasts the
+    // `waitTimeoutMs` its frame names.
     clientCapabilities: [
       ...(isAssistant ? [] : ['workflow-tool-pickup']),
       TOOL_DEADLINE_CAPABILITY,
       WORKFLOW_RUN_WAIT_TIMEOUT_CAPABILITY,
+      WORKFLOW_TOOL_WAIT_TIMEOUT_CAPABILITY,
     ],
   }
 }

apps/sim/lib/mothership/generated/protocol.ts

@@ -297,6 +297,14 @@ export interface StreamToolReplay {
  */
 export interface StreamToolDeadline {
   deadlineMs?: number | undefined;
+  /**
+   * For a workflow tool (`run_workflow`, `run_workflow_until_block`, `run_from_block`,
+   * `run_block`) on a Sim that declares {@link WORKFLOW_TOOL_WAIT_TIMEOUT_CAPABILITY}: how long
+   * Sim waits for the call's run to settle once it starts waiting on it. A run still going then
+   * keeps going and the call settles with a {@link WorkflowToolRunningResult}. Absent = held
+   * until the run settles.
+   */
+  waitTimeoutMs?: number | undefined;
 }
 
 /**
@@ -322,6 +330,24 @@ export const WORKSPACE_FILE_COPY_CAPABILITY = "workspace-file-copy";
  */
 export const WORKFLOW_RUN_WAIT_TIMEOUT_CAPABILITY = "workflow-run-wait-timeout";
 
+/**
+ * The `clientCapabilities` token a Sim sends when it honors `StreamToolDeadline.waitTimeoutMs`
+ * on a workflow tool call: a run that outlasts it continues and the call settles with a
+ * {@link WorkflowToolRunningResult}, so the worker waits on the run in slices.
+ */
+export const WORKFLOW_TOOL_WAIT_TIMEOUT_CAPABILITY = "workflow-tool-wait-timeout";
+
+/**
+ * A workflow tool call's output when its run outlasted the call's `waitTimeoutMs`: the run is
+ * still going under `executionId`, readable as a run of `workflowId` (`workflows runs wait`).
+ */
+export interface WorkflowToolRunningResult {
+  status: "running";
+  executionId: string;
+  workflowId: string;
+  message?: string | undefined;
+}
+
 /** POST /api/mothership — the chat request sim sends. */
 export interface ChatRequest extends StreamResponseReceipt {
   fileOwnerProtocolVersion?: 1 | undefined;
@@ -367,8 +393,9 @@ export interface ChatRequest extends StreamResponseReceipt {
    * grace windows and run tools server-side immediately. ABSENT = older/unknown caller —
    * dispatch keeps its conservative grace (deploy-skew safe: a stale tab that predates
    * this field still gets waited on). Known capability: "workflow-tool-pickup". Sim also
-   * declares its own {@link TOOL_DEADLINE_CAPABILITY}, {@link WORKSPACE_FILE_COPY_CAPABILITY} and
-   * {@link WORKFLOW_RUN_WAIT_TIMEOUT_CAPABILITY} here.
+   * declares its own {@link TOOL_DEADLINE_CAPABILITY}, {@link WORKSPACE_FILE_COPY_CAPABILITY},
+   * {@link WORKFLOW_RUN_WAIT_TIMEOUT_CAPABILITY} and {@link WORKFLOW_TOOL_WAIT_TIMEOUT_CAPABILITY}
+   * here.
    */
   clientCapabilities?: string[] | undefined;
   /**

apps/sim/lib/mothership/request/handlers/handlers.test.ts

@@ -732,6 +732,35 @@ describe('sse-handlers tool lifecycle', () => {
       expect(context.toolCalls.get('takeover-1')?.deadlineAt).toBeLessThan(before + 60_000)
     })
 
+    it('hands a workflow tool the wait its frame names, and no other tool', async () => {
+      executeTool.mockResolvedValue({ success: true, output: {} })
+      const frame = (toolCallId: string, toolName: string, waitTimeoutMs: unknown) => {
+        const event = deadlineFrame(900_000, {
+          toolCallId,
+          toolName,
+          arguments: { workflowId: 'workflow-1' },
+        })
+        return { ...event, payload: { ...event.payload, waitTimeoutMs } } as StreamEvent
+      }
+      await sseHandlers.tool(frame('run-1', 'run_workflow', 240_000), context, execContext, {
+        interactive: false,
+      })
+      await sseHandlers.tool(frame('audio-2', 'generate_audio', 240_000), context, execContext, {
+        interactive: false,
+      })
+      await sseHandlers.tool(frame('run-2', 'run_workflow', -5), context, execContext, {
+        interactive: false,
+      })
+      await Promise.all(context.pendingToolPromises.values())
+      const handed = (toolCallId: string) =>
+        executeTool.mock.calls.find(
+          ([, , toolContext]) => toolContext?.toolCallId === toolCallId
+        )?.[2]
+      expect(handed('run-1')?.workflowRunWaitMs).toBe(240_000)
+      expect(handed('audio-2')).not.toHaveProperty('workflowRunWaitMs')
+      expect(handed('run-2')).not.toHaveProperty('workflowRunWaitMs')
+    })
+
     /** Runs a desktop-local read whose frame names `deadlineMs`, recording the emitted events. */
     async function runLocalRead(deadlineMs: number) {
       const onEvent = vi.fn()

apps/sim/lib/mothership/request/handlers/tool.ts

@@ -44,6 +44,7 @@ import { sealClientToolContext } from '@/lib/mothership/request/tools/client-com
 import {
   capToToolDeadline,
   frameToolDeadlineAt,
+  frameWorkflowWaitMs,
   TOOL_DEADLINE_BEFORE_START_MESSAGE,
   TOOL_DEADLINE_EXCEEDED_MESSAGE,
   toDeadlineDate,
@@ -742,6 +743,8 @@ async function handleCallPhase(
   if (parentToolCallId) toolCall.parentToolCallId = parentToolCallId
   if (data.execName) toolCall.execName = data.execName
   toolCall.deadlineAt ??= frameToolDeadlineAt(data.deadlineMs, Date.now())
+  if (isWorkflowToolName(toolName))
+    toolCall.waitTimeoutMs ??= frameWorkflowWaitMs(data.waitTimeoutMs)
 
   const readPath = typeof args?.path === 'string' ? args.path : undefined
   if (toolName === 'read' && readPath?.startsWith('internal/')) return
@@ -1110,6 +1113,9 @@ async function dispatchToolExecution(
               options.clientToolPickupExpected === false
                 ? 0
                 : COPILOT_WORKFLOW_TOOL_CLIENT_GRACE_MS,
+            ...(toolCall.waitTimeoutMs !== undefined
+              ? { waitTimeoutMs: toolCall.waitTimeoutMs }
+              : {}),
             abortSignal: options.abortSignal,
             registry: execContext.resolvedSecretTraceRegistry,
             runOnServer: (boundExecutionId) => {

apps/sim/lib/mothership/request/tools/client.test.ts

@@ -729,7 +729,8 @@ describe('workflow client tool completion', () => {
     expect(completion).toEqual({
       status: 'background',
       message: 'Workflow execution is continuing in the background.',
-      data: { workflowId: 'workflow-1', executionId: 'execution-1' },
+      // A WorkflowToolRunningResult: the worker waits on the run by these ids.
+      data: { workflowId: 'workflow-1', executionId: 'execution-1', status: 'running' },
     })
     expect(registry.isComplete()).toBe(true)
     expect(getTrustedWorkflowToolExecution).not.toHaveBeenCalled()

apps/sim/lib/mothership/request/tools/deadline.ts

@@ -20,6 +20,20 @@ export function frameToolDeadlineAt(deadlineMs: unknown, receivedAt: number): nu
   return receivedAt + deadlineMs - TOOL_DEADLINE_SETTLE_MARGIN_MS
 }
 
+/**
+ * How long a workflow tool call waits for its run before settling still running, from the
+ * frame's `waitTimeoutMs`. Undefined when the frame names none or a malformed value: the call
+ * holds until its run settles.
+ */
+export function frameWorkflowWaitMs(waitTimeoutMs: unknown): number | undefined {
+  return typeof waitTimeoutMs === 'number' &&
+    Number.isSafeInteger(waitTimeoutMs) &&
+    waitTimeoutMs > 0 &&
+    waitTimeoutMs <= MAX_FRAME_DEADLINE_MS
+    ? waitTimeoutMs
+    : undefined
+}
+
 /** What the model learns when a call's deadline passed before the call could start. */
 export const TOOL_DEADLINE_BEFORE_START_MESSAGE =
   'This tool call ran out of its time limit before it could start, so it did not run. Nothing changed; it is safe to try again with a smaller request.'

apps/sim/lib/mothership/request/tools/executor.ts

@@ -369,13 +369,17 @@ class ToolExecutionTimeoutError extends Error {
 
 /** Builds the per-call context from the turn-scoped execution context. */
 export function buildToolExecutionContext(
-  toolCall: Pick<ToolCallState, 'id' | 'parentToolCallId' | 'params' | 'targetWorkspaceId'>,
+  toolCall: Pick<
+    ToolCallState,
+    'id' | 'parentToolCallId' | 'params' | 'targetWorkspaceId' | 'waitTimeoutMs'
+  >,
   execContext: ExecutionContext
 ): ExecutionContext {
   return {
     ...execContext,
     toolCallId: toolCall.id,
     ...(toolCall.targetWorkspaceId ? { targetWorkspaceId: toolCall.targetWorkspaceId } : {}),
+    ...(toolCall.waitTimeoutMs !== undefined ? { workflowRunWaitMs: toolCall.waitTimeoutMs } : {}),
     resolvedSecretTraceRegistry: execContext.resolvedSecretTraceRegistry?.forkForInputPaths([]),
     ...(toolCall.parentToolCallId ? { parentToolCallId: toolCall.parentToolCallId } : {}),
   }

apps/sim/lib/mothership/request/tools/workflow-client-fallback.test.ts

@@ -8,8 +8,13 @@ import {
 } from '@sim/testing/mocks/mothership-client-tool-waiter.mock'
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 
-const { recordDegraded } = vi.hoisted(() => ({
+const { recordDegraded, settleClientToolCall } = vi.hoisted(() => ({
   recordDegraded: vi.fn(),
+  settleClientToolCall: vi.fn(),
+}))
+
+vi.mock('@/lib/mothership/request/tools/client-settlement.server', () => ({
+  settleClientToolCall,
 }))
 
 vi.mock('@/lib/mothership/request/metrics', () => ({ recordDegraded }))
@@ -24,6 +29,8 @@ vi.mock('@/lib/mothership/async-runs/repository', () => mothershipAsyncRunsMock)
 import { raceWorkflowToolClientPickup } from '@/lib/mothership/request/tools/workflow-client-fallback'
 
 const claimWorkflowToolExecution = mothershipAsyncRunsMockFns.mockClaimWorkflowToolExecution
+const getAsyncToolCall = mothershipAsyncRunsMockFns.mockGetAsyncToolCall
+const getClaimedWorkflowExecutionId = mothershipAsyncRunsMockFns.mockGetClaimedWorkflowExecutionId
 
 const GRACE_MS = 30_000
 const TIMEOUT_MS = 3_600_000
@@ -216,3 +223,86 @@ describe('raceWorkflowToolClientPickup', () => {
     expect(params.runOnServer).not.toHaveBeenCalled()
   })
 })
+
+describe('a browser-run workflow call past its wait', () => {
+  const WAIT_MS = 240_000
+  beforeEach(() => {
+    vi.useFakeTimers()
+    waiterSignals = []
+    settleClientToolCall.mockReset()
+    getAsyncToolCall.mockReset()
+    getClaimedWorkflowExecutionId.mockImplementation((claimedBy?: string | null) =>
+      claimedBy?.startsWith('workflow:') ? claimedBy.slice('workflow:'.length) : undefined
+    )
+    pendingUntilAborted()
+    // A browser already holds the call, so the server keeps waiting on it.
+    claimWorkflowToolExecution.mockResolvedValue(null)
+  })
+  afterEach(() => {
+    vi.useRealTimers()
+  })
+
+  const browserClaim = {
+    status: 'running',
+    claimedBy: 'workflow:execution-1',
+    clientWorkflowExecutionId: 'execution-1',
+  }
+
+  it('settles the call running in the background by the browser run ids', async () => {
+    getAsyncToolCall.mockResolvedValue(browserClaim)
+    const params = baseParams({ waitTimeoutMs: WAIT_MS })
+    void raceWorkflowToolClientPickup(params as never)
+
+    await vi.advanceTimersByTimeAsync(WAIT_MS - 1)
+    expect(settleClientToolCall).not.toHaveBeenCalled()
+    await vi.advanceTimersByTimeAsync(1)
+
+    expect(settleClientToolCall).toHaveBeenCalledWith({
+      toolCallId: 'tool-1',
+      status: 'background',
+      message: 'Workflow execution is continuing in the background.',
+      data: { workflowId: 'workflow-1', executionId: 'execution-1', status: 'running' },
+      executionId: 'execution-1',
+      guard: { kind: 'open' },
+    })
+    // The browser's run goes on; only the server's wait ended.
+    expect(params.runOnServer).not.toHaveBeenCalled()
+  })
+
+  it('re-reads a call no browser has claimed yet until one does', async () => {
+    getAsyncToolCall
+      .mockResolvedValueOnce({
+        status: 'running',
+        claimedBy: null,
+        clientWorkflowExecutionId: null,
+      })
+      .mockResolvedValue(browserClaim)
+    void raceWorkflowToolClientPickup(baseParams({ waitTimeoutMs: WAIT_MS }) as never)
+
+    await vi.advanceTimersByTimeAsync(WAIT_MS)
+    expect(settleClientToolCall).not.toHaveBeenCalled()
+    await vi.advanceTimersByTimeAsync(5_000)
+    expect(settleClientToolCall).toHaveBeenCalledWith(
+      expect.objectContaining({ status: 'background', executionId: 'execution-1' })
+    )
+  })
+
+  it('leaves a call that settled first alone', async () => {
+    waitForWorkflowToolCompletion.mockResolvedValue({ status: 'success', data: { ok: true } })
+    const outcome = await raceWorkflowToolClientPickup(
+      baseParams({ waitTimeoutMs: WAIT_MS }) as never
+    )
+    await vi.advanceTimersByTimeAsync(WAIT_MS * 2)
+    expect(outcome.winner).toBe('client')
+    expect(getAsyncToolCall).not.toHaveBeenCalled()
+    expect(settleClientToolCall).not.toHaveBeenCalled()
+  })
+
+  it('holds a call that names no wait', async () => {
+    getAsyncToolCall.mockResolvedValue(browserClaim)
+    void raceWorkflowToolClientPickup(baseParams() as never)
+    await vi.advanceTimersByTimeAsync(TIMEOUT_MS / 2)
+    expect(getAsyncToolCall).not.toHaveBeenCalled()
+    expect(settleClientToolCall).not.toHaveBeenCalled()
+  })
+})