diff --git a/Dockerfile.sandbox b/Dockerfile.sandbox index 5cffc99d..b028908c 100644 --- a/Dockerfile.sandbox +++ b/Dockerfile.sandbox @@ -73,15 +73,18 @@ RUN echo "Installing playwright=${PLAYWRIGHT_VERSION} chromium (cachebust=${TOOL # /usr/local/lib/node_modules for the NODE_PATH fallback above to work. ENV npm_config_prefix=/opt/agent-tools -# The exec user is an unknown numeric uid, so verify as uid 1000 (also absent -# from /etc/passwd) instead of root. A root-only check would not catch the -# EACCES failures this image exists to prevent. -RUN echo "Verifying guest toolchain as an unprivileged uid (cachebust=${TOOLS_CACHEBUST})" && \ +# The runtime exec user is a numeric host uid with no /etc/passwd entry, so the +# toolchain is verified as uid 4242, which is absent from this image's passwd +# database. uid 1000 would not do: it is the base image's `node` user, so it +# would exercise the known-user path and pass even if the real one were broken. +# sudo is checked separately as that known user, because sudo refuses unknown +# uids by design and the Manager provisions their passwd entry at runtime. +RUN echo "Verifying guest toolchain as an unknown uid (cachebust=${TOOLS_CACHEBUST})" && \ rm -rf /home/ocm-agent && \ mkdir -p /home/ocm-agent && \ chmod 1777 /home/ocm-agent && \ - setpriv --reuid=1000 --regid=1000 --clear-groups sh -c "set -e; pnpm --version; bun --version; bunx --version; uv --version; uvx --version; playwright --version; gh --version; jq --version; rg --version; npm install -g --silent cowsay && cowsay -t verify-ok | head -1; sudo -n apt-get update -qq && sudo -n apt-get install -y -qq --no-install-recommends bc && bc --version | head -1; uv venv /tmp/uv-check >/dev/null; uv pip install --python /tmp/uv-check/bin/python idna >/dev/null; pip3 install --user --force-reinstall --no-deps --quiet idna; rm -rf /tmp/uv-check" && \ - npm uninstall -g --silent cowsay && \ + setpriv --reuid=4242 --regid=4242 --clear-groups sh -c "set -e; pnpm --version; bun --version; bunx --version; uv --version; uvx --version; playwright --version; gh --version; jq --version; rg --version; npm install -g --silent cowsay && cowsay -t verify-ok | head -1; git config --global user.email verify@ocm.local; uv venv /tmp/uv-check >/dev/null; uv pip install --python /tmp/uv-check/bin/python idna >/dev/null; pip3 install --user --force-reinstall --no-deps --quiet idna; rm -rf /tmp/uv-check" && \ + setpriv --reuid=1000 --regid=1000 --clear-groups sh -c "set -e; sudo -n apt-get update -qq; sudo -n apt-get install -y -qq --no-install-recommends bc; bc --version | head -1" && \ rm -rf /home/ocm-agent /opt/agent-tools /var/lib/apt/lists/* && \ mkdir -p /home/ocm-agent /opt/agent-tools/bin && \ chmod 1777 /home/ocm-agent /opt/agent-tools /opt/agent-tools/bin diff --git a/backend/src/services/sandbox/command.ts b/backend/src/services/sandbox/command.ts index 54a2a7ea..6089473b 100644 --- a/backend/src/services/sandbox/command.ts +++ b/backend/src/services/sandbox/command.ts @@ -124,6 +124,27 @@ export function buildSandboxPullArgs(): string[] { return ['pull', ENV.SANDBOX.IMAGE] } +export function buildSandboxPingArgs(): string[] { + return ['ping', WORKSPACE_SANDBOX_NAME, '-q'] +} + +export function buildSandboxVerifyProvisionArgs(): string[] { + const uid = resolveSandboxExecUserUid() + if (uid === null) return [] + return [ + 'exec', + WORKSPACE_SANDBOX_NAME, + '--no-tty', + '-q', + '-u', + '0:0', + '--', + '/bin/sh', + '-c', + `getent passwd ${uid} >/dev/null`, + ] +} + export function sandboxMountRoots(): string[] { return [getReposPath(), getScheduleWorktreesPath()] } @@ -155,6 +176,8 @@ export function buildSandboxCreateArgs(): string[] { getReposPath(), '--entrypoint', '/usr/bin/env', + '--shell', + '/bin/sh', ENV.SANDBOX.IMAGE, '--', 'sleep', @@ -314,6 +337,7 @@ function parseSandboxCreateArgs(args: string[]): { mountDirs: string[] workdir: string entrypoint: string[] + shell: string image: string cmd: string[] } { @@ -325,6 +349,7 @@ function parseSandboxCreateArgs(args: string[]): { let user = '' let workdir = '' let entrypoint: string[] = [] + let shell = '' let image = '' let cmd: string[] = [] for (let i = 1; i < args.length; i++) { @@ -351,12 +376,13 @@ function parseSandboxCreateArgs(args: string[]): { case '--mount-dir': if (value !== undefined) mountDirs.push(value); i += 1; break case '-w': workdir = value ?? ''; i += 1; break case '--entrypoint': if (value !== undefined) entrypoint = [value]; i += 1; break + case '--shell': if (value !== undefined) shell = value; i += 1; break case '-d': break default: if (image === '' && !token.startsWith('-')) image = token } } - return { name, labels, memory, cpus, user, mountDirs, workdir, entrypoint, image, cmd } + return { name, labels, memory, cpus, user, mountDirs, workdir, entrypoint, shell, image, cmd } } export function buildCanonicalSandboxSpec(): Record { @@ -392,7 +418,7 @@ export function buildCanonicalSandboxSpec(): Record { }, runtime: { workdir: args.workdir, - shell: null, + shell: args.shell, scripts: {}, entrypoint: args.entrypoint, cmd: args.cmd, @@ -480,8 +506,11 @@ export function buildSandboxProvisionArgs(): string[] { '/bin/sh', '-c', `getent group ${gid} >/dev/null 2>&1 || echo 'ocm-exec:x:${gid}:' >> /etc/group; ` + - `getent passwd ${uid} >/dev/null 2>&1 || echo 'ocm-exec:x:${uid}:${gid}:Manager sandbox exec user:/home/ocm-agent:/bin/sh' >> /etc/passwd; ` + - `grep -q '^ocm-exec:' /etc/shadow || echo 'ocm-exec:*:19000:0:99999:7:::' >> /etc/shadow`, + `getent passwd ${uid} >/dev/null 2>&1 || { ` + + `echo 'ocm-exec:x:${uid}:${gid}:Manager sandbox exec user:/home/ocm-agent:/bin/sh' >> /etc/passwd; ` + + `grep -q '^ocm-exec:' /etc/shadow || echo 'ocm-exec:*:19000:0:99999:7:::' >> /etc/shadow; ` + + `}; ` + + `getent passwd ${uid} >/dev/null || exit 1`, ] } diff --git a/backend/src/services/sandbox/runtime.ts b/backend/src/services/sandbox/runtime.ts index 9fb5292b..269b6121 100644 --- a/backend/src/services/sandbox/runtime.ts +++ b/backend/src/services/sandbox/runtime.ts @@ -15,11 +15,13 @@ import { buildSandboxCreateArgs, buildSandboxInspectArgs, buildSandboxListArgs, + buildSandboxPingArgs, buildSandboxProvisionArgs, buildSandboxPullArgs, buildSandboxRemoveArgs, buildSandboxStartArgs, buildSandboxStopManagedArgs, + buildSandboxVerifyProvisionArgs, resolveExpectedSandboxNetworkPolicy, resolveSandboxRuntimeTmpfsSizeMib, resolveSandboxWorkDirectory, @@ -33,6 +35,9 @@ const SANDBOX_LS_TIMEOUT_MS = 15000 const SANDBOX_STOP_TIMEOUT_MS = 30000 const SANDBOX_PROVISION_ATTEMPTS = 3 const SANDBOX_PROVISION_RETRY_DELAY_MS = 2000 +const SANDBOX_PROVISION_RETRY_ATTEMPTS = 5 +const SANDBOX_AGENT_PING_TIMEOUT_MS = 10000 +const SANDBOX_AGENT_POLL_DELAY_MS = 1000 const SANDBOX_RUNTIME_TMPFS_GUEST = path.resolve('/tmp') export type SandboxShellPlan = @@ -59,6 +64,9 @@ export function resetSandboxRuntimeState(): void { shutdownRequested = false stopInProgress = false canonicalSandboxSpecMemo = null + provisionRetryGeneration += 1 + backgroundProvisionRetry = null + inFlightProvision = null } function memoizedCanonicalSandboxSpec(): Record { @@ -133,7 +141,7 @@ async function bootWorkspaceSandbox(): Promise { await createWorkspaceSandbox() } else { await startWorkspaceSandbox() - await provisionSandboxExecUserPasswd() + await provisionSandboxExecUser() const runningAttestation = await attestWorkspaceSandbox(true) if (!runningAttestation.trusted) { logger.warn( @@ -166,7 +174,7 @@ async function bootWorkspaceSandboxFromListing(): Promise { await createWorkspaceSandbox() } else if (!entry.running) { await startWorkspaceSandbox() - await provisionSandboxExecUserPasswd() + await provisionSandboxExecUser() const runningAttestation = await attestWorkspaceSandbox(true) if (!runningAttestation.trusted) { logger.warn( @@ -551,23 +559,69 @@ async function cacheSandboxImage(): Promise { logger.info(`Sandbox guest image ${ENV.SANDBOX.IMAGE} is cached`) } +async function pingSandboxAgent(): Promise { + try { + const result = (await executeCommand([sandboxExecutablePath(), ...buildSandboxPingArgs()], { + ignoreExitCode: true, + silent: true, + timeout: SANDBOX_AGENT_PING_TIMEOUT_MS, + })) as string | { exitCode: number; stdout: string; stderr: string } + return typeof result === 'string' || result.exitCode === 0 + } catch { + return false + } +} + +async function waitForSandboxAgent(): Promise { + const deadline = Date.now() + ENV.SANDBOX.START_TIMEOUT_MS + for (;;) { + if (await pingSandboxAgent()) return true + if (Date.now() >= deadline) return false + await new Promise((resolve) => setTimeout(resolve, SANDBOX_AGENT_POLL_DELAY_MS)) + } +} + +async function provisionSandboxExecUser(): Promise { + try { + await ensureSandboxExecUserProvisioned() + return + } catch (error) { + logger.warn( + `Sandbox exec user provisioning failed, so sudo will not work inside the guest yet; retrying in the background: ${error instanceof Error ? error.message : String(error)}`, + ) + } + scheduleBackgroundProvisionRetry() +} + async function provisionSandboxExecUserPasswd(): Promise { const provisionArgs = buildSandboxProvisionArgs() if (provisionArgs.length === 0) { return } + if (!(await waitForSandboxAgent())) { + throw new Error('the sandbox agent did not become reachable before the exec user could be provisioned') + } let lastError: unknown = null + const deadline = Date.now() + ENV.SANDBOX.START_TIMEOUT_MS for (let attempt = 0; attempt < SANDBOX_PROVISION_ATTEMPTS; attempt++) { if (attempt > 0) { + if (Date.now() >= deadline) break await new Promise((resolve) => setTimeout(resolve, SANDBOX_PROVISION_RETRY_DELAY_MS)) } try { await executeCommand([sandboxExecutablePath(), ...provisionArgs], { - timeout: SANDBOX_LS_TIMEOUT_MS, + timeout: ENV.SANDBOX.START_TIMEOUT_MS, }) + const verifyArgs = buildSandboxVerifyProvisionArgs() + if (verifyArgs.length > 0) { + await executeCommand([sandboxExecutablePath(), ...verifyArgs], { + timeout: SANDBOX_LS_TIMEOUT_MS, + }) + } return } catch (error) { lastError = error + if (Date.now() >= deadline) break } } throw new Error( @@ -575,11 +629,69 @@ async function provisionSandboxExecUserPasswd(): Promise { ) } +let backgroundProvisionRetry: Promise | null = null +let provisionRetryGeneration = 0 +let inFlightProvision: Promise | null = null + +function ensureSandboxExecUserProvisioned(): Promise { + if (inFlightProvision) { + return inFlightProvision + } + inFlightProvision = provisionSandboxExecUserPasswd().finally(() => { + inFlightProvision = null + }) + return inFlightProvision +} + +export function backgroundProvisionRetryForTests(): Promise | null { + return backgroundProvisionRetry +} + +export function provisionSandboxExecUserForTests(): Promise { + return ensureSandboxExecUserProvisioned() +} + +function scheduleBackgroundProvisionRetry(): void { + const generation = provisionRetryGeneration + if (backgroundProvisionRetry !== null) return + logger.info('Sandbox exec user provisioning background retry scheduled') + backgroundProvisionRetry = (async () => { + for (let attempt = 1; attempt <= SANDBOX_PROVISION_RETRY_ATTEMPTS; attempt++) { + await new Promise((resolve) => setTimeout(resolve, SANDBOX_PROVISION_RETRY_DELAY_MS)) + if (generation !== provisionRetryGeneration) { + logger.info('Sandbox exec user provisioning background retry cancelled by a runtime state reset') + return + } + if (shutdownRequested) { + logger.info('Sandbox exec user provisioning background retry cancelled by shutdown') + return + } + try { + await ensureSandboxExecUserProvisioned() + logger.info(`Sandbox exec user provisioned by the background retry (attempt ${attempt})`) + return + } catch (error) { + logger.warn( + `Sandbox exec user provisioning background retry attempt ${attempt} failed: ${error instanceof Error ? error.message : String(error)}`, + ) + if (generation !== provisionRetryGeneration || shutdownRequested) return + } + } + logger.warn( + 'Sandbox exec user provisioning retries were exhausted; sudo will not work inside the guest until the next boot cycle', + ) + })().finally(() => { + if (generation === provisionRetryGeneration) { + backgroundProvisionRetry = null + } + }) +} + async function createWorkspaceSandbox(): Promise { await executeCommand([sandboxExecutablePath(), ...buildSandboxCreateArgs()], { timeout: ENV.SANDBOX.START_TIMEOUT_MS, }) - await provisionSandboxExecUserPasswd() + await provisionSandboxExecUser() const attestation = await attestWorkspaceSandbox(true) if (!attestation.trusted) { throw new Error(`newly created sandbox ${WORKSPACE_SANDBOX_NAME} failed attestation: ${attestation.reason}`) @@ -665,8 +777,15 @@ export class SandboxRuntimeService { if (getProcessIdentityAttestationError() !== null) return await cacheSandboxImage() await ensureWorkspaceSandbox() - await provisionSandboxExecUserPasswd() - logger.info('Workspace sandbox is running and its exec user is provisioned') + try { + await ensureSandboxExecUserProvisioned() + logger.info('Workspace sandbox is running and its exec user is provisioned') + } catch (error) { + logger.warn( + `Workspace sandbox is running but its exec user is not provisioned yet; retrying in the background: ${error instanceof Error ? error.message : String(error)}`, + ) + scheduleBackgroundProvisionRetry() + } } isEnabled(): boolean { @@ -727,6 +846,8 @@ export class SandboxRuntimeService { private async stopManagedSandbox(): Promise { stopInProgress = true + provisionRetryGeneration += 1 + backgroundProvisionRetry = null try { await this.runManagedSandboxStop() } finally { diff --git a/backend/src/utils/process.ts b/backend/src/utils/process.ts index aea1cd84..ed39b931 100644 --- a/backend/src/utils/process.ts +++ b/backend/src/utils/process.ts @@ -43,7 +43,8 @@ export async function executeCommand( const proc: ChildProcess = spawn(command || '', cmdArgs, { cwd: options.cwd, shell: false, - env: effectiveEnv + env: effectiveEnv, + stdio: ['ignore', 'pipe', 'pipe'], }) let stdout = '' diff --git a/backend/test/routes/internal-sandbox.test.ts b/backend/test/routes/internal-sandbox.test.ts index e7930b11..76cd5f36 100644 --- a/backend/test/routes/internal-sandbox.test.ts +++ b/backend/test/routes/internal-sandbox.test.ts @@ -11,73 +11,37 @@ import { createOpenCodeClient } from '../../src/services/opencode/client' import { allMigrations } from '../../src/db/migrations' import { getOrCreateInternalToken } from '../../src/services/internal-token' import { migrate } from '../../src/db/migration-runner' -import { resolveSandboxExecUser, resolveSandboxRuntimeTmpfsSizeMib, WORKSPACE_SANDBOX_NAME } from '../../src/services/sandbox/command' +import { + buildCanonicalSandboxSpec, + resolveExpectedSandboxNetworkPolicy, + resolveSandboxRuntimeTmpfsSizeMib, + WORKSPACE_SANDBOX_NAME, +} from '../../src/services/sandbox/command' import { executeCommand } from '../../src/utils/process' import { detectSandboxCapability } from '../../src/services/sandbox/capability' -import { getReposPath, getScheduleWorktreesPath, ENV } from '@opencode-manager/shared/config/env' +import { forceProcessAttestation } from '../../src/services/opencode/process-identity' +import { getReposPath, ENV } from '@opencode-manager/shared/config/env' import type { ScheduleWorktreeManager } from '../../src/services/schedule-worktree' function trustedRunningInspect(): { exitCode: number; stdout: string; stderr: string } { - const memoryMatch = /^(\d+(?:\.\d+)?)([gGmM])?$/.exec(ENV.SANDBOX.MEMORY) - const memoryMib = memoryMatch - ? memoryMatch[2] === undefined || memoryMatch[2] === 'M' || memoryMatch[2] === 'm' - ? Math.floor(Number(memoryMatch[1])) - : Math.floor(Number(memoryMatch[1]) * 1024) - : 0 - const bindMount = (host: string) => ({ - type: 'Bind', - host, - guest: host, - options: { readonly: false, noexec: false, nosuid: false, nodev: false }, - stat_virtualization: 'strict', - host_permissions: 'private', - follow_root_symlinks: false, - quota_mib: null, - }) + const canonical = buildCanonicalSandboxSpec() + const { memory_mib: memoryMib } = canonical.resources as { memory_mib: number } const config = { - name: WORKSPACE_SANDBOX_NAME, - image: { Oci: { reference: ENV.SANDBOX.IMAGE, root_disk: { kind: 'managed', size_mib: 4096 } } }, - resources: { cpus: ENV.SANDBOX.CPUS, memory_mib: memoryMib, max_cpus: ENV.SANDBOX.CPUS, max_memory_mib: memoryMib }, - runtime: { - workdir: getReposPath(), - shell: null, - scripts: {}, - entrypoint: ['/usr/bin/env'], - cmd: ['sleep', 'infinity'], - hostname: null, - user: resolveSandboxExecUser(), - log_level: null, - metrics_sample_interval_ms: 1000, - disable_metrics_sample: false, - }, - env: [], - labels: { 'ocm.managed': 'true', 'ocm.net': ENV.SANDBOX.NET }, - rlimits: [], + ...canonical, mounts: [ - bindMount(getReposPath()), - bindMount(getScheduleWorktreesPath()), - { type: 'Tmpfs', guest: '/tmp', size_mib: resolveSandboxRuntimeTmpfsSizeMib(memoryMib), options: { readonly: false, noexec: false, nosuid: false, nodev: false } }, + ...(canonical.mounts as unknown[]), + { + type: 'Tmpfs', + guest: '/tmp', + size_mib: resolveSandboxRuntimeTmpfsSizeMib(memoryMib), + options: { readonly: false, noexec: false, nosuid: false, nodev: false }, + }, ], - patches: [], network: { enabled: true, ports: [], - policy: { - default_egress: 'deny', - default_ingress: 'allow', - rules: [ - { direction: 'egress', destination: { group: 'host' }, protocols: ['udp', 'tcp'], ports: [{ start: 53, end: 53 }], action: 'allow' }, - { direction: 'egress', destination: { group: 'public' }, protocols: [], ports: [], action: 'allow' }, - ], - }, - max_connections: null, - trust_host_cas: false, + policy: resolveExpectedSandboxNetworkPolicy(ENV.SANDBOX.NET), }, - init: null, - pull_policy: 'IfMissing', - security_profile: 'default', - lifecycle: { ephemeral: false, max_duration_secs: null, idle_timeout_secs: null }, - manifest_digest: 'sha256:0000000000000000000000000000000000000000000000000000000000000000', } return { exitCode: 0, @@ -120,6 +84,7 @@ describe('internal sandbox routes', () => { mockExecuteCommand.mockClear() mockDetectSandboxCapability.mockReset() mockDetectSandboxCapability.mockReturnValue({ available: true, msbVersion: 'msb 0.3.1' }) + forceProcessAttestation(true) db = new Database(':memory:') migrate(db, allMigrations) const openCodeClient = createOpenCodeClient() @@ -135,6 +100,7 @@ describe('internal sandbox routes', () => { }) afterEach(() => { + forceProcessAttestation(null) db.close() rmSync(repoDir, { recursive: true, force: true }) }) diff --git a/backend/test/services/sandbox/command.test.ts b/backend/test/services/sandbox/command.test.ts index 1f7ce444..de6248c5 100644 --- a/backend/test/services/sandbox/command.test.ts +++ b/backend/test/services/sandbox/command.test.ts @@ -149,6 +149,7 @@ describe('sandbox command builders', () => { expect(runtime.user).toBe(resolveSandboxExecUser()) expect(runtime.cmd).toEqual(['sleep', 'infinity']) expect(runtime.entrypoint).toEqual(['/usr/bin/env']) + expect(runtime.shell).toBe('/bin/sh') expect(spec.patches).toEqual([]) expect(network.enabled).toBe(true) expect(network.ports).toEqual([]) @@ -268,9 +269,10 @@ describe('sandbox command builders', () => { const script = args[args.indexOf('-c') + 1] expect(script).toContain("getent group 1002 >/dev/null 2>&1 || echo 'ocm-exec:x:1002:' >> /etc/group") expect(script).toContain( - "getent passwd 1001 >/dev/null 2>&1 || echo 'ocm-exec:x:1001:1002:Manager sandbox exec user:/home/ocm-agent:/bin/sh' >> /etc/passwd", + "getent passwd 1001 >/dev/null 2>&1 || { echo 'ocm-exec:x:1001:1002:Manager sandbox exec user:/home/ocm-agent:/bin/sh' >> /etc/passwd; grep -q '^ocm-exec:' /etc/shadow || echo 'ocm-exec:*:19000:0:99999:7:::' >> /etc/shadow; }", ) - expect(script).toContain("grep -q '^ocm-exec:' /etc/shadow || echo 'ocm-exec:*:19000:0:99999:7:::' >> /etc/shadow") + expect(script).toContain('getent passwd 1001 >/dev/null || exit 1') + expect(script).toContain('getent passwd 1001 >/dev/null || exit 1') }) it('returns no provisioning args when the exec user does not resolve to numeric uid and gid', () => { diff --git a/backend/test/services/sandbox/runtime.test.ts b/backend/test/services/sandbox/runtime.test.ts index 73b7ca8e..1b7668b3 100644 --- a/backend/test/services/sandbox/runtime.test.ts +++ b/backend/test/services/sandbox/runtime.test.ts @@ -8,7 +8,7 @@ import { migrate } from '../../../src/db/migration-runner' import { allMigrations } from '../../../src/db/migrations' import { SettingsService } from '../../../src/services/settings' import { buildSandboxInspectArgs, resolveSandboxExecUser, resolveSandboxRuntimeTmpfsSizeMib, sandboxExecutablePath, WORKSPACE_SANDBOX_NAME } from '../../../src/services/sandbox/command' -import { SandboxRuntimeService, resetSandboxRuntimeState, stopWorkspaceSandboxOnShutdown } from '../../../src/services/sandbox/runtime' +import { SandboxRuntimeService, backgroundProvisionRetryForTests, provisionSandboxExecUserForTests, resetSandboxRuntimeState, stopWorkspaceSandboxOnShutdown } from '../../../src/services/sandbox/runtime' import { executeCommand } from '../../../src/utils/process' import { detectSandboxCapability } from '../../../src/services/sandbox/capability' import { logger } from '../../../src/utils/logger' @@ -125,7 +125,7 @@ describe('SandboxRuntimeService', () => { resources: { cpus: ENV.SANDBOX.CPUS, memory_mib: memoryMib(), max_cpus: ENV.SANDBOX.CPUS, max_memory_mib: memoryMib() }, runtime: { workdir: reposRoot, - shell: null, + shell: '/bin/sh', scripts: {}, entrypoint: ['/usr/bin/env'], cmd: ['sleep', 'infinity'], @@ -409,6 +409,102 @@ describe('SandboxRuntimeService', () => { expect(mockExecuteCommand.mock.calls.find((call) => call[0].includes('exec'))).toBeDefined() }) + it('waits for the guest agent before provisioning the exec user', async () => { + enableEnforcement() + let pingCalls = 0 + let inspectCalls = 0 + mockExecuteCommand.mockImplementation(async (args: string[]) => { + if (args.includes('ping')) { + pingCalls += 1 + if (pingCalls === 1) { + return { exitCode: 1, stdout: '', stderr: 'agent unreachable' } + } + return { exitCode: 0, stdout: '', stderr: '' } + } + if (args.includes('inspect')) { + inspectCalls += 1 + if (inspectCalls === 1) { + return { exitCode: 1, stdout: '', stderr: 'no such sandbox' } + } + return inspectedRunningSandbox() + } + if (args.includes('ls')) { + return { exitCode: 0, stdout: '[]', stderr: '' } + } + return { exitCode: 0, stdout: '', stderr: '' } + }) + + const plan = await service.planShell(repoADir) + + expect(plan).toEqual({ mode: 'sandbox', workdir: repoADir }) + expect(pingCalls).toBeGreaterThanOrEqual(2) + const pingIndex = mockExecuteCommand.mock.calls.findIndex((call) => call[0].includes('ping')) + const provisionIndex = mockExecuteCommand.mock.calls.findIndex((call) => call[0].includes('exec')) + expect(pingIndex).toBeLessThan(provisionIndex) + }) + + it('keeps the sandbox usable when exec user provisioning fails', async () => { + enableEnforcement() + let inspectCalls = 0 + mockExecuteCommand.mockImplementation(async (args: string[]) => { + if (args.includes('exec')) { + throw new Error('Command timed out after 30000ms') + } + if (args.includes('inspect')) { + inspectCalls += 1 + if (inspectCalls === 1) { + return { exitCode: 1, stdout: '', stderr: 'no such sandbox' } + } + return inspectedRunningSandbox() + } + if (args.includes('ls')) { + return { exitCode: 0, stdout: '[]', stderr: '' } + } + return { exitCode: 0, stdout: '', stderr: '' } + }) + + const plan = await service.planShell(repoADir) + + expect(plan).toEqual({ mode: 'sandbox', workdir: repoADir }) + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('sudo will not work inside the guest')) + await backgroundProvisionRetryForTests() + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining('provisioning retries were exhausted'), + ) + }, 60000) + + it('provisions the exec user via the background retry after an initial failure', async () => { + enableEnforcement() + let planDone = false + let inspectCalls = 0 + mockExecuteCommand.mockImplementation(async (args: string[]) => { + if (args.includes('exec')) { + if (!planDone) { + throw new Error('Command timed out after 30000ms') + } + return { exitCode: 0, stdout: '', stderr: '' } + } + if (args.includes('inspect')) { + inspectCalls += 1 + if (inspectCalls === 1) { + return { exitCode: 1, stdout: '', stderr: 'no such sandbox' } + } + return inspectedRunningSandbox() + } + if (args.includes('ls')) { + return { exitCode: 0, stdout: '[]', stderr: '' } + } + return { exitCode: 0, stdout: '', stderr: '' } + }) + + const plan = await service.planShell(repoADir) + planDone = true + + expect(plan).toEqual({ mode: 'sandbox', workdir: repoADir }) + await backgroundProvisionRetryForTests() + expect(logger.info).toHaveBeenCalledWith(expect.stringContaining('provisioned by the background retry')) + }, 30000) + it('never touches msb at boot when the sandbox preference is off', async () => { mockExecuteCommand.mockImplementation(async () => { throw new Error('msb must not be invoked when sandboxing is disabled') @@ -488,6 +584,8 @@ describe('SandboxRuntimeService', () => { stderr: '', }) .mockResolvedValueOnce({ exitCode: 0, stdout: '', stderr: '' }) + .mockResolvedValueOnce({ exitCode: 0, stdout: '', stderr: '' }) + .mockResolvedValueOnce({ exitCode: 0, stdout: '', stderr: '' }) .mockResolvedValueOnce(inspectedRunningSandbox()) const directory = repoADir @@ -2318,6 +2416,51 @@ describe('SandboxRuntimeService', () => { expect(mockExecuteCommand.mock.calls.filter((call) => call[0].includes('inspect'))).toHaveLength(1) }) + it('shares one in-flight attempt when exec user provisioning is requested concurrently', async () => { + enableEnforcement() + mockExecuteCommand.mockImplementation(async (args: string[]) => { + if (args.includes('inspect')) return inspectedRunningSandbox() + if (args.includes('ls')) return { exitCode: 0, stdout: '[]', stderr: '' } + return { exitCode: 0, stdout: '', stderr: '' } + }) + + const first = provisionSandboxExecUserForTests() + const second = provisionSandboxExecUserForTests() + + expect(second).toBe(first) + await Promise.all([first, second]) + + const provisionCalls = mockExecuteCommand.mock.calls.filter( + (call) => call[0].includes('exec') && call[0].some((arg: string) => arg.includes('/etc/passwd')), + ) + expect(provisionCalls).toHaveLength(1) + }) + + it('cancels a scheduled provisioning retry when the sandbox is stopped by a toggle', async () => { + enableEnforcement() + let inspectCalls = 0 + mockExecuteCommand.mockImplementation(async (args: string[]) => { + if (args.includes('exec')) throw new Error('Command timed out after 30000ms') + if (args.includes('inspect')) { + inspectCalls += 1 + if (inspectCalls === 1) return { exitCode: 1, stdout: '', stderr: 'no such sandbox' } + return inspectedRunningSandbox() + } + if (args.includes('ls')) return { exitCode: 0, stdout: '[]', stderr: '' } + return { exitCode: 0, stdout: '', stderr: '' } + }) + + await service.planShell(repoADir) + const retry = backgroundProvisionRetryForTests() + expect(retry).not.toBeNull() + + await service.stopWorkspaceSandboxForToggle() + expect(backgroundProvisionRetryForTests()).toBeNull() + + await retry + expect(logger.info).toHaveBeenCalledWith(expect.stringContaining('background retry cancelled')) + }, 30000) + it('attempts the toggle stop even when sandbox capability is unavailable', async () => { mockDetectSandboxCapability.mockReturnValue({ available: false, reason: '/dev/kvm is not available' }) mockExecuteCommand.mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' }) @@ -2416,6 +2559,8 @@ describe('SandboxRuntimeService', () => { .mockResolvedValueOnce({ exitCode: 0, stdout: '[]', stderr: '' }) .mockResolvedValueOnce({ exitCode: 0, stdout: '', stderr: '' }) .mockResolvedValueOnce({ exitCode: 0, stdout: '', stderr: '' }) + .mockResolvedValueOnce({ exitCode: 0, stdout: '', stderr: '' }) + .mockResolvedValueOnce({ exitCode: 0, stdout: '', stderr: '' }) .mockResolvedValueOnce(inspectedRunningSandbox()) const failed = await service.planShell(repoADir) @@ -2675,7 +2820,7 @@ describe('SandboxRuntimeService', () => { it('removes and recreates a sandbox whose runtime shell differs from the canonical spec', async () => { const runtime = realInspectConfig().runtime as Record await assertRecreateForInspectMutation( - realInspectConfig({ runtime: { ...runtime, shell: '/bin/sh' } }), + realInspectConfig({ runtime: { ...runtime, shell: '/bin/bash' } }), 'runtime.shell', ) }) diff --git a/docker-compose.sandbox.yml b/docker-compose.sandbox.yml index 7daa857e..03512e8f 100644 --- a/docker-compose.sandbox.yml +++ b/docker-compose.sandbox.yml @@ -16,7 +16,7 @@ services: cap_add: - NET_ADMIN environment: - - SANDBOX_IMAGE=${SANDBOX_IMAGE:-docker.io/cstechdev/ocm-sandbox@sha256:74a9f12e1c1768e36bc159c4c7efb70ab9c47da5f46f93763ddb282dd58ad79c} + - SANDBOX_IMAGE=${SANDBOX_IMAGE:-docker.io/cstechdev/ocm-sandbox@sha256:10e2ca3c93883441538b5f6008caee28ec88ec842cbdbdfed0b7a459d80f1583} - SANDBOX_MEMORY=${SANDBOX_MEMORY:-4G} - SANDBOX_CPUS=${SANDBOX_CPUS:-2} - SANDBOX_EXEC_USER=${SANDBOX_EXEC_USER:-${PUID:-1000}} diff --git a/docs/configuration/docker.md b/docs/configuration/docker.md index fcee1fff..2707a31d 100644 --- a/docs/configuration/docker.md +++ b/docs/configuration/docker.md @@ -330,7 +330,7 @@ services: cap_add: - NET_ADMIN environment: - - SANDBOX_IMAGE=${SANDBOX_IMAGE:-docker.io/cstechdev/ocm-sandbox@sha256:74a9f12e1c1768e36bc159c4c7efb70ab9c47da5f46f93763ddb282dd58ad79c} + - SANDBOX_IMAGE=${SANDBOX_IMAGE:-docker.io/cstechdev/ocm-sandbox@sha256:10e2ca3c93883441538b5f6008caee28ec88ec842cbdbdfed0b7a459d80f1583} - SANDBOX_MEMORY=${SANDBOX_MEMORY:-4G} - SANDBOX_CPUS=${SANDBOX_CPUS:-2} - SANDBOX_EXEC_USER=${SANDBOX_EXEC_USER:-${PUID:-1000}} diff --git a/docs/configuration/environment.md b/docs/configuration/environment.md index db306987..a4d868f4 100644 --- a/docs/configuration/environment.md +++ b/docs/configuration/environment.md @@ -116,7 +116,7 @@ Sandboxed agent commands run inside a microVM managed by `msb` (see [Agent Sandb |----------|-------------|---------| | `MSB_PATH` | Path to the `msb` executable | `msb` | | `MSB_LIBKRUNFW_PATH` | Path to the `libkrunfw` firmware library used by `msb` (set in the container image) | `/opt/microsandbox/lib/libkrunfw.so` | -| `SANDBOX_IMAGE` | OCI image the microVM boots from. Digest-pinned by default so a rebuilt guest image is actually adopted; see [Sandbox Guest Image](../features/sandboxing.md#sandbox-guest-image) for what the default ships and how to build your own | `docker.io/cstechdev/ocm-sandbox@sha256:74a9f12e…` | +| `SANDBOX_IMAGE` | OCI image the microVM boots from. Digest-pinned by default so a rebuilt guest image is actually adopted; see [Sandbox Guest Image](../features/sandboxing.md#sandbox-guest-image) for what the default ships and how to build your own | `docker.io/cstechdev/ocm-sandbox@sha256:10e2ca3c…` | | `SANDBOX_MEMORY` | MicroVM memory (e.g. `4G`) | `4G` | | `SANDBOX_CPUS` | MicroVM CPU count | `2` | | `SANDBOX_EXEC_USER` | Guest identity sandboxed commands run as: a numeric `uid`, a numeric `uid:gid`, or a guest username. A numeric uid must match the Manager's effective uid (`PUID`); the compose overlay defaults it to `${PUID:-1000}`. A guest username is resolved to the Manager's effective `uid:gid` so writes to the mounted project roots always succeed. When a configured numeric identity cannot write the workspace, enforcement is reported unavailable | `${PUID:-1000}` via the overlay, otherwise `node` | diff --git a/docs/features/sandboxing.md b/docs/features/sandboxing.md index 89688ecb..e42fa01f 100644 --- a/docs/features/sandboxing.md +++ b/docs/features/sandboxing.md @@ -193,7 +193,7 @@ It is `node:24` (Debian 12, `buildpack-deps` based), so the compile toolchain is The guest runs every command as a numeric host uid that has no `/etc/passwd` entry in the image, so the uncorrected default `HOME` is `/` and anything that writes per-user state — the pnpm store, uv and pip caches, `git config`, `gh` config — dies with `EACCES` on the first run. The image therefore sets `HOME=/home/ocm-agent` (mode 1777), prewarms corepack into a world-readable `COREPACK_HOME`, and puts a world-writable `/opt/agent-tools/bin` on `PATH` for `uv tool` and global package-manager installs. Image `ENV` reaches `msb exec` commands verbatim, including for unknown uids. The image build verifies the whole toolchain as an unprivileged uid so a root-only regression fails the build instead of the agent. -`sudo` also needs the exec user to exist in the guest, which the image cannot know at build time. The Manager provisions the `/etc/passwd`, `/etc/group`, and `/etc/shadow` entries for the exec uid through an idempotent root exec (verified by `getent`, so repeats are no-ops) at three points: when the workspace sandbox is created, when a stopped sandbox is started, and at Manager startup. Without the entries, `sudo` refuses with "you do not exist in the passwd database" or a PAM "account validation failure". A microVM created from an older image reference is replaced automatically when the digest pin changes, so it picks up both sudo and the fixed toolchain without manual cleanup. +`sudo` also needs the exec user to exist in the guest, which the image cannot know at build time. The Manager provisions the `/etc/passwd`, `/etc/group`, and `/etc/shadow` entries for the exec uid through an idempotent root exec (verified by `getent`, so repeats are no-ops) at three points: when the workspace sandbox is created, when a stopped sandbox is started, and at Manager startup. A freshly created microVM does not accept `msb exec` until its guest agent is up, so provisioning first waits on `msb ping` (bounded by `SANDBOX_START_TIMEOUT_MS`) instead of racing the boot. Without the entries, `sudo` refuses with "you do not exist in the passwd database" or a PAM "account validation failure". Provisioning is deliberately non-fatal: if it fails, the Manager logs a warning and the sandbox still runs commands — only `sudo` is unavailable. A microVM created from an older image reference is replaced automatically when the digest pin changes, so it picks up both sudo and the fixed toolchain without manual cleanup. Chromium launches headless as the non-root exec user without extra flags. If your host kernel restricts user namespaces so Chromium's own sandbox fails, pass `--no-sandbox` — the microVM is already the isolation boundary. diff --git a/shared/src/config/defaults.ts b/shared/src/config/defaults.ts index 6e2c63c9..696b741f 100644 --- a/shared/src/config/defaults.ts +++ b/shared/src/config/defaults.ts @@ -33,7 +33,7 @@ export const DEFAULTS = { SANDBOX: { MSB_PATH: 'msb', - IMAGE: 'docker.io/cstechdev/ocm-sandbox@sha256:74a9f12e1c1768e36bc159c4c7efb70ab9c47da5f46f93763ddb282dd58ad79c', + IMAGE: 'docker.io/cstechdev/ocm-sandbox@sha256:10e2ca3c93883441538b5f6008caee28ec88ec842cbdbdfed0b7a459d80f1583', MEMORY: '4G', CPUS: 2, EXEC_USER: 'node',