Skip to content

Commit 6943fde

Browse files
committed
improvement(sandbox): verify the workbench CLI inside each command's launch
Every warm workbench call ran a separate CLI integrity check command (ownership write, launch, stop receipt, settle) before the user's command, about a third of a warm call's fixed cost. - The session supervisor's PID 1 now runs the same check as the ordinary user, after recording itself and before starting the command; a mismatch starts nothing and is reported on the stop receipt the call already fetches - A refused launch is repaired under the session lock and relaunched once; a second refusal fails the call - The check runs with an isolated interpreter instead of the user's login shell, never follows links or blocks on non-regular files, and treats an unreadable artifact as stale so repair overwrites it - A reused machine runs no separate check command; a new machine still installs the CLI first - Gated live test against real E2B covers refusal, repair, hooks, FIFOs, Stop during a slow check, and the warm-call cost
1 parent 3fa33cf commit 6943fde

10 files changed

Lines changed: 675 additions & 65 deletions

File tree

‎apps/sim/lib/execution/remote-sandbox/e2b-session.test.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ import {
5151
stopE2BSessionProcess,
5252
} from '@/lib/execution/remote-sandbox/e2b'
5353
import { observeSandboxExecution } from '@/lib/execution/remote-sandbox/observer'
54+
import { SessionCliStaleError } from '@/lib/execution/remote-sandbox/session-cli'
5455

5556
setEnv({ E2B_API_KEY: 'test', MOTHERSHIP_E2B_TEMPLATE_ID: 'mothership-template' })
5657

@@ -643,6 +644,36 @@ describe('E2B session recovery', () => {
643644
stderr: 'slow frame\n',
644645
})
645646
})
647+
it.each([
648+
['stale', SessionCliStaleError],
649+
['failed', Error],
650+
] as const)(
651+
'rejects a launch whose receipt reports a %s CLI, whatever its process returned',
652+
async (cli, expected) => {
653+
runCommand.mockResolvedValue({
654+
pid: 9,
655+
wait: async () => ({ stdout: '', stderr: '', exitCode: 0 }),
656+
})
657+
stopCommand.mockResolvedValue({
658+
stdout: JSON.stringify({ settled: true, cli }),
659+
stderr: '',
660+
exitCode: 0,
661+
})
662+
const sandbox = await sessionSandbox('reconnected')
663+
const launched = sandbox?.runCommand('touch /tmp/marker', {
664+
timeoutMs: 1000,
665+
verifyCli: { path: '/home/user/.sim-cli/r/cli.mjs', artifacts: { a: 'b' } },
666+
})
667+
await expect(launched).rejects.toBeInstanceOf(expected)
668+
await expect(launched).rejects.toThrow(cli === 'stale' ? 'changed' : 'verification failed')
669+
const [command] = runCommand.mock.calls.at(-1) ?? []
670+
expect(command).toContain(
671+
Buffer.from(
672+
JSON.stringify({ path: '/home/user/.sim-cli/r/cli.mjs', artifacts: { a: 'b' } })
673+
).toString('base64')
674+
)
675+
}
676+
)
646677
})
647678

648679
describe('E2B session lease', () => {

‎apps/sim/lib/execution/remote-sandbox/e2b.ts‎

Lines changed: 49 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,10 @@ import {
5555
tailStreamedSandboxOutput,
5656
} from '@/lib/execution/remote-sandbox/output-limits'
5757
import { resolveSandboxDirectoryEntryPath } from '@/lib/execution/remote-sandbox/paths'
58+
import {
59+
SessionCliStaleError,
60+
SessionCliVerificationError,
61+
} from '@/lib/execution/remote-sandbox/session-cli'
5862
import { withSandboxSessionLock } from '@/lib/execution/remote-sandbox/session-lock'
5963
import {
6064
type SessionProcessIdentity,
@@ -82,6 +86,7 @@ import type {
8286
SandboxImageMaterialization,
8387
SandboxKind,
8488
SandboxProvider,
89+
SessionCliVerification,
8590
} from '@/lib/execution/remote-sandbox/types'
8691

8792
const logger = createLogger('E2BSandboxProvider')
@@ -319,11 +324,17 @@ async function listAllSandboxes(paginator: SandboxPaginator): Promise<SandboxInf
319324
return sandboxes
320325
}
321326

327+
/** What a session command's launch found when it verified the deployment CLI; absent when it matched. */
328+
interface SessionCommandReceipt {
329+
cli?: 'stale' | 'failed'
330+
}
331+
332+
/** Settles the command; the receipt is the only proof that a refused launch started nothing. */
322333
async function stopSessionCommand(
323334
sandbox: E2BSandbox,
324335
processId: string,
325336
signal = AbortSignal.timeout(10_000)
326-
): Promise<void> {
337+
): Promise<SessionCommandReceipt> {
327338
signal.throwIfAborted()
328339
const result = await sandbox.commands.run(sessionProcessStopCommand(processId), {
329340
user: 'root',
@@ -335,6 +346,7 @@ async function stopSessionCommand(
335346
if (result.exitCode !== 0 || !isRecordLike(receipt) || receipt.settled !== true) {
336347
throw new Error('Sandbox command namespace exit is unconfirmed')
337348
}
349+
return receipt.cli === 'stale' || receipt.cli === 'failed' ? { cli: receipt.cli } : {}
338350
}
339351

340352
/** Recovers the exact recorded workbench; expiry is absence, while lookup failure stays unresolved. */
@@ -413,6 +425,10 @@ class E2BSandboxHandle implements SandboxHandle {
413425
return this.sandbox.sandboxId
414426
}
415427

428+
get verifiesCliAtLaunch(): boolean {
429+
return this.sessionKey !== undefined
430+
}
431+
416432
outlives(lifetimeMs: number, fromMs: number): boolean {
417433
return (
418434
this.sessionDeadlineAtMs !== undefined &&
@@ -441,6 +457,7 @@ class E2BSandboxHandle implements SandboxHandle {
441457
javascriptPreload?: string
442458
maxOutputBytes?: number
443459
signal?: AbortSignal
460+
verifyCli?: SessionCliVerification
444461
}
445462
): Promise<SandboxCodeResult> {
446463
const runId = generateShortId(12)
@@ -467,6 +484,7 @@ class E2BSandboxHandle implements SandboxHandle {
467484
maxOutputBytes: options.maxOutputBytes,
468485
signal: options.signal,
469486
atMostOnce: true,
487+
verifyCli: options.verifyCli,
470488
envs: {
471489
...options.envs,
472490
SIM_CODE_PATH: codePath,
@@ -519,6 +537,25 @@ class E2BSandboxHandle implements SandboxHandle {
519537
command: string,
520538
options: RunCommandOptions,
521539
operation: 'code' | 'command'
540+
): Promise<SandboxCommandResult> {
541+
let refusal: SessionCommandReceipt['cli']
542+
const outcome = await this.runOwnedCommand(command, options, operation, (receipt) => {
543+
refusal = receipt.cli
544+
}).then(
545+
(result) => ({ result }),
546+
(error: unknown) => ({ error })
547+
)
548+
if (refusal === 'stale') throw new SessionCliStaleError()
549+
if (refusal === 'failed') throw new SessionCliVerificationError()
550+
if ('error' in outcome) throw outcome.error
551+
return outcome.result
552+
}
553+
554+
private async runOwnedCommand(
555+
command: string,
556+
options: RunCommandOptions,
557+
operation: 'code' | 'command',
558+
onReceipt: (receipt: SessionCommandReceipt) => void
522559
): Promise<SandboxCommandResult> {
523560
if (this.sessionKey !== undefined) options.signal?.throwIfAborted()
524561
let commandDeadlineAtMs: number | undefined
@@ -537,9 +574,9 @@ class E2BSandboxHandle implements SandboxHandle {
537574
let observedStderr = ''
538575
const processId = this.sessionKey !== undefined ? generateId() : undefined
539576
let processRecorded = false
540-
let stopping: Promise<void> | undefined
541-
const stopCommand = (): Promise<void> => {
542-
if (!processId || !processRecorded) return Promise.resolve()
577+
let stopping: Promise<SessionCommandReceipt> | undefined
578+
const stopCommand = (): Promise<SessionCommandReceipt | undefined> => {
579+
if (!processId || !processRecorded) return Promise.resolve(undefined)
543580
stopping ??= stopSessionCommand(this.sandbox, processId).catch((error: unknown) => {
544581
logger.warn('Failed to stop sandbox process', {
545582
sandboxId: this.sandboxId,
@@ -595,7 +632,12 @@ class E2BSandboxHandle implements SandboxHandle {
595632
commandDeadlineAtMs = Date.now() + processOptions.timeoutMs
596633
started = await this.sandbox.commands.run(
597634
processId
598-
? sessionProcessCommand(processId, prepared.command, options.rootUser)
635+
? sessionProcessCommand(
636+
processId,
637+
prepared.command,
638+
options.rootUser,
639+
options.verifyCli
640+
)
599641
: prepared.command,
600642
{
601643
...processOptions,
@@ -753,11 +795,12 @@ class E2BSandboxHandle implements SandboxHandle {
753795
options.signal?.removeEventListener('abort', onAbort)
754796
/** The named SDK process can exit before its namespace; only the kernel receipt settles it. */
755797
if (processId && processRecorded) {
756-
await stopCommand().catch((error: unknown) => {
798+
const receipt = await stopCommand().catch((error: unknown) => {
757799
reportUnsettledSandboxProcess(processId)
758800
throw error
759801
})
760802
await settleSandboxProcess(processId)
803+
if (receipt) onReceipt(receipt)
761804
}
762805
}
763806
}

‎apps/sim/lib/execution/remote-sandbox/index.ts‎

Lines changed: 88 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,12 @@ import {
6060
ensureSessionSandbox,
6161
SESSION_SANDBOX_IDLE_MS,
6262
} from '@/lib/execution/remote-sandbox/session'
63-
import { sessionCommandPath } from '@/lib/execution/remote-sandbox/session-cli'
63+
import {
64+
ensureSessionCli,
65+
SessionCliStaleError,
66+
sessionCliVerification,
67+
sessionCommandPath,
68+
} from '@/lib/execution/remote-sandbox/session-cli'
6469
import {
6570
readSessionSecretProvenance,
6671
recordSessionFileInput,
@@ -87,6 +92,7 @@ import type {
8792
SandboxProviderId,
8893
SandboxSessionRequest,
8994
SandboxShellExecutionRequest,
95+
SessionCliVerification,
9096
} from '@/lib/execution/remote-sandbox/types'
9197

9298
export type {
@@ -259,6 +265,42 @@ async function withSandboxExecutionBudget<T>(
259265
}
260266
}
261267

268+
/**
269+
* Launches a session command that verifies the deployment CLI as it starts. A launch that found
270+
* the CLI changed started nothing, so the CLI is repaired under the session lock and the command
271+
* launched once more; a second refusal means something keeps changing it, and the call fails.
272+
*/
273+
async function launchWithVerifiedCli<T>(
274+
sandbox: SandboxHandle,
275+
lease: SandboxLease,
276+
session: SandboxSessionRequest | undefined,
277+
signal: AbortSignal,
278+
phases: SandboxPhaseTimer,
279+
launch: (verifyCli: SessionCliVerification | undefined) => Promise<T>
280+
): Promise<T> {
281+
const cli = lease.session && sandbox.verifiesCliAtLaunch ? session?.cli : undefined
282+
if (!session || !cli) return launch(undefined)
283+
const verifyCli = sessionCliVerification(cli)
284+
try {
285+
return await launch(verifyCli)
286+
} catch (error) {
287+
if (!(error instanceof SessionCliStaleError)) throw error
288+
}
289+
const endCli = phases.start('cli')
290+
await withSandboxSessionLock(session.key, signal, (lockSignal) =>
291+
ensureSessionCli(sandbox, cli, lockSignal, remainingSandboxBudgetMs(signal))
292+
)
293+
endCli()
294+
try {
295+
return await launch(verifyCli)
296+
} catch (error) {
297+
if (error instanceof SessionCliStaleError) {
298+
throw new Error('Workbench CLI installation could not be verified')
299+
}
300+
throw error
301+
}
302+
}
303+
262304
function throwIfSandboxTimedOut(result: { timedOut?: boolean }): void {
263305
if (result.timedOut) throw new DOMException('timeout', 'AbortError')
264306
}
@@ -1008,16 +1050,27 @@ async function executeInSandboxWithinBudget(
10081050
Object.keys(privateInputFiles.environment).length > 0
10091051

10101052
let execution: SandboxCodeResult
1011-
const runTimeoutMs = codeBudgetMs(req, signal)
1053+
let runTimeoutMs = 0
10121054
try {
10131055
const endRun = phases.start('run')
1014-
execution = await sandbox.runCode(code, {
1015-
timeoutMs: runTimeoutMs,
1016-
javascriptPreload: buildJavaScriptRuntimeBindingsSource(req.runtimeBindings ?? []),
1017-
maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES,
1056+
execution = await launchWithVerifiedCli(
1057+
sandbox,
1058+
lease,
1059+
req.session,
10181060
signal,
1019-
...(hasExecutionEnvironment ? { envs: executionEnvironment } : {}),
1020-
})
1061+
phases,
1062+
(verifyCli) => {
1063+
runTimeoutMs = codeBudgetMs(req, signal)
1064+
return sandbox.runCode(code, {
1065+
timeoutMs: runTimeoutMs,
1066+
javascriptPreload: buildJavaScriptRuntimeBindingsSource(req.runtimeBindings ?? []),
1067+
maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES,
1068+
signal,
1069+
...(hasExecutionEnvironment ? { envs: executionEnvironment } : {}),
1070+
verifyCli,
1071+
})
1072+
}
1073+
)
10211074
endRun()
10221075
} catch (error) {
10231076
throwIfAborted(signal)
@@ -1212,24 +1265,36 @@ async function executeShellInSandboxWithinBudget(
12121265
endPrepare()
12131266

12141267
let result: SandboxCommandResult
1215-
const runTimeoutMs = codeBudgetMs(req, signal)
1268+
let runTimeoutMs = 0
12161269
try {
12171270
const endRun = phases.start('run')
1218-
result = await sandbox.runCommand(code, {
1219-
envs: {
1220-
...selected?.envs,
1221-
...envs,
1222-
...req.session?.envs,
1223-
PATH: sessionCommandPath(req.session, selected?.envs?.PATH ?? SANDBOX_SYSTEM_PATH),
1224-
...privateInputFiles.environment,
1225-
...(req.session && req.outputSandboxDir ? { SIM_OUTPUT_DIR: req.outputSandboxDir } : {}),
1226-
},
1227-
timeoutMs: runTimeoutMs,
1228-
maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES,
1271+
const commandEnvironment = {
1272+
...selected?.envs,
1273+
...envs,
1274+
...req.session?.envs,
1275+
PATH: sessionCommandPath(req.session, selected?.envs?.PATH ?? SANDBOX_SYSTEM_PATH),
1276+
...privateInputFiles.environment,
1277+
...(req.session && req.outputSandboxDir ? { SIM_OUTPUT_DIR: req.outputSandboxDir } : {}),
1278+
}
1279+
result = await launchWithVerifiedCli(
1280+
sandbox,
1281+
lease,
1282+
req.session,
12291283
signal,
1230-
rootUser: !lease.session,
1231-
atMostOnce: true,
1232-
})
1284+
phases,
1285+
(verifyCli) => {
1286+
runTimeoutMs = codeBudgetMs(req, signal)
1287+
return sandbox.runCommand(code, {
1288+
envs: commandEnvironment,
1289+
timeoutMs: runTimeoutMs,
1290+
maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES,
1291+
signal,
1292+
rootUser: !lease.session,
1293+
atMostOnce: true,
1294+
verifyCli,
1295+
})
1296+
}
1297+
)
12331298
endRun()
12341299
} catch (error) {
12351300
throwIfAborted(signal)

‎apps/sim/lib/execution/remote-sandbox/phase-timing.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
/**
22
* Phases of one sandbox execution, in the order they run. `lease` spans the whole acquisition
3-
* (session lock included); for a session, `lookup` and `cli` are the parts of it spent finding or
4-
* creating the machine and verifying its deployment tooling.
3+
* (session lock included); for a session, `lookup` is the part of it spent finding or creating the
4+
* machine. `cli` is time spent installing or repairing deployment tooling: on a new machine, or
5+
* after a launch refused a changed CLI. A reused machine verifies it inside `run`.
56
*/
67
type SandboxPhase =
78
| 'resolve'

0 commit comments

Comments
 (0)