Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -126,8 +126,10 @@ cut.

Copy `.env.example` to `.env` and set `CODEAPI_BRIDGE_TOKEN` to a private value
of at least 32 bytes (generate one with `openssl rand -hex 32`). The API exposes
bridge routes even with the default HTTP sandbox backend, so hardened mode
requires this enrollment credential. Compose defaults to
bridge routes when configured through the remote-bridge backend, paired auth,
dynamic workers, or a bridge token. Hardened deployments with none of these
configured leave bridge routes disabled and do not require a bridge token.
Enabled bridges still require this enrollment credential. Compose defaults to
`CODEAPI_BRIDGE_AUTH_MODE=paired` and `CODEAPI_BRIDGE_DYNAMIC_WORKERS=true`.
To restrict pairing to a fixed worker, set `CODEAPI_BRIDGE_DYNAMIC_WORKERS=false`
and `CODEAPI_BRIDGE_WORKER_ID` to its ID. Keep the token outside workspaces and
Expand Down
4 changes: 4 additions & 0 deletions packages/code/src/native-process-child.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,10 @@ process.on('message', async (raw: unknown) => {
error instanceof WorkspaceToolError
? error.mutationMayHaveCommitted
: true,
requiresQuarantine:
error instanceof WorkspaceToolError
? error.requiresQuarantine
: true,
});
} finally {
active = undefined;
Expand Down
36 changes: 35 additions & 1 deletion packages/code/src/native-process.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -179,13 +179,47 @@ test('executor cancellation targets the active request and preserves mutation ce
ok: false,
code: 'EXECUTION_ABORTED',
mutation: true,
requiresQuarantine: false,
});
await assert.rejects(
execution,
(error: unknown) =>
error instanceof WorkspaceToolError &&
error.code === 'EXECUTION_ABORTED' &&
error.mutationMayHaveCommitted,
error.mutationMayHaveCommitted &&
!error.requiresQuarantine,
);
await sandbox.close();
});

test('executor ignores a cleanup exemption on non-cancellation failures', async () => {
let dispatched!: () => void;
const dispatch = new Promise<void>((resolve) => {
dispatched = resolve;
});
const fake = fixture(() => dispatched());
const sandbox = new NativeProcessWorkspaceCommandSandbox(
{ workspaceRoot: '/workspace' },
fake.fork,
);
const execution = sandbox.execute(request);
await dispatch;
const command = fake.messages.find((message) => message.type === 'execute')!;
fake.child.emit('message', {
id: command.id,
ok: false,
code: 'COMMAND_UNAVAILABLE',
mutation: true,
requiresQuarantine: false,
});

await assert.rejects(
execution,
(error: unknown) =>
error instanceof WorkspaceToolError &&
error.code === 'COMMAND_UNAVAILABLE' &&
error.mutationMayHaveCommitted &&
error.requiresQuarantine,
);
await sandbox.close();
});
Expand Down
5 changes: 5 additions & 0 deletions packages/code/src/native-process.ts
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ export class NativeProcessWorkspaceCommandSandbox
ok?: unknown;
result?: unknown;
mutation?: unknown;
requiresQuarantine?: unknown;
code?: unknown;
errorMessage?: unknown;
fatal?: unknown;
Expand All @@ -156,6 +157,9 @@ export class NativeProcessWorkspaceCommandSandbox
message.code === 'REGISTRATION_INVALID'
? message.code
: 'COMMAND_UNAVAILABLE';
const processTerminationConfirmed =
code === 'EXECUTION_ABORTED' &&
message.requiresQuarantine === false;
pending.reject(
new WorkspaceToolError(
typeof message.errorMessage === 'string' &&
Expand All @@ -164,6 +168,7 @@ export class NativeProcessWorkspaceCommandSandbox
: 'Native executor request failed',
code,
pending.mutation && message.mutation !== false,
pending.mutation && !processTerminationConfirmed,
),
);
}
Expand Down
7 changes: 5 additions & 2 deletions packages/code/src/native-sandbox.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1030,7 +1030,8 @@ test('reports cancellation after command start as a potentially committed mutati
(error: unknown) =>
error instanceof WorkspaceToolError &&
error.code === 'EXECUTION_ABORTED' &&
error.mutationMayHaveCommitted === true,
error.mutationMayHaveCommitted === true &&
error.requiresQuarantine === false,
);
});

Expand Down Expand Up @@ -1157,7 +1158,9 @@ test('cleans allocated command state exactly once on every execution exit', asyn
error.code === (outcome.startsWith('abort')
? 'EXECUTION_ABORTED'
: 'COMMAND_UNAVAILABLE') &&
error.mutationMayHaveCommitted === (outcome === 'abort-after-spawn'),
error.mutationMayHaveCommitted === (outcome === 'abort-after-spawn') &&
error.requiresQuarantine ===
(outcome === 'abort-after-spawn' && process.platform === 'win32'),
);
}
assert.equal(
Expand Down
4 changes: 4 additions & 0 deletions packages/code/src/native-sandbox.ts
Original file line number Diff line number Diff line change
Expand Up @@ -681,6 +681,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox
'Workspace command execution aborted',
'EXECUTION_ABORTED',
true,
// POSIX commands run in a detached process group, so its
// observed close follows a group-wide SIGKILL. The Windows
// fallback cannot yet prove descendant termination.
this.platform === 'win32',
),
);
return;
Expand Down
2 changes: 1 addition & 1 deletion packages/code/src/worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1562,7 +1562,7 @@ export class BridgeWorker {
!(
error instanceof WorkspaceToolError &&
this.options.workspaceTools?.mutationFailuresAreAtomic === true &&
!error.mutationMayHaveCommitted
!error.requiresQuarantine
))
) {
ambiguousWorkspaceMutationError = error;
Expand Down
70 changes: 70 additions & 0 deletions packages/code/src/workspace-worker.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1410,6 +1410,76 @@ test('worker clears quarantine after a composed command is cleanly rejected', as
assert.deepEqual(lifecycle, ['arm', 'execute', 'settle', 'clear']);
});

test('worker clears quarantine after a command cancellation confirms process termination', async () => {
const lifecycle: string[] = [];
const baseCapabilities = {
protocolVersion: 1 as const,
operations: ['read_file' as const],
workspaces: [{ id: 'primary', operations: ['read_file' as const] }],
};
const workspaceTools = new SandboxWorkspaceTools({
workspaceTools: {
capabilities: baseCapabilities,
mutationFailuresAreAtomic: true,
async execute() { throw new Error('base executor must not run'); },
},
commandWorkspaces: ['primary'],
commandSandbox: {
mutationFailuresAreAtomic: true,
async execute() {
lifecycle.push('execute');
throw new WorkspaceToolError(
'Workspace command execution aborted',
'EXECUTION_ABORTED',
true,
false,
);
},
},
});
const worker = new BridgeWorker({
codeApiUrl: 'https://code.example/v1',
token: 'worker-secret',
workerId: 'vm-1',
incarnationId,
sandboxEndpoint: 'http://127.0.0.1:2000/api/v2',
capabilities: {
statefulWorkspace: true,
sandboxProfile: 'nsjail',
runtimes: ['bash'],
workspaceTools: workspaceTools.capabilities,
},
workspaceTools,
workspaceMutationQuarantine: mutationQuarantine(
() => lifecycle.push('quarantine'),
() => lifecycle.push('arm'),
() => lifecycle.push('clear'),
),
fetchImpl: async () => {
lifecycle.push('settle');
return Response.json({ protocolVersion: 1, accepted: true });
},
});

await worker.executeAndSettle({
protocolVersion: 1,
assignmentId: 'assignment-command-cancelled-cleanly',
workerId: 'vm-1',
incarnationId,
generation: 4,
leaseToken: 'lease-token-that-is-long-enough-for-testing',
expiresAt: new Date(Date.now() + 5_000).toISOString(),
executionKind: 'workspace_tool',
request: {
protocolVersion: 1,
operation: 'execute_command',
workspaceId: 'primary',
command: 'sleep 30',
},
});
assert.deepEqual(lifecycle, ['arm', 'execute', 'settle', 'clear']);
});

test('worker retains quarantine when an atomic executor cannot confirm durability', async () => {
const lifecycle: string[] = [];
const workspaceCapabilities = {
Expand Down
2 changes: 2 additions & 0 deletions packages/code/src/workspace.ts
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,8 @@ export class WorkspaceToolError extends Error {
message: string,
public readonly code: WorkspaceToolErrorCode,
public readonly mutationMayHaveCommitted = false,
/** Retain the durable mutation guard when process or write settlement is uncertain. */
public readonly requiresQuarantine = mutationMayHaveCommitted,
) {
super(message);
this.name = 'WorkspaceToolError';
Expand Down
9 changes: 9 additions & 0 deletions service/src/bridge/enabled.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import { env } from '../config';

/** API-only deployments can serve bridges without selecting that worker backend. */
export function isBridgeEnabled(): boolean {
return env.SANDBOX_BACKEND === 'remote-bridge'
|| env.BRIDGE_AUTH_MODE === 'paired'
|| env.BRIDGE_DYNAMIC_WORKERS
|| env.BRIDGE_TOKEN.length > 0;
}
2 changes: 2 additions & 0 deletions service/src/bridge/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { env } from '../config';
import { RedisBridgePairingStore } from './pairing';
import { createBridgeRouter } from './router';
import { RedisBridgeStore } from './store';
import { isBridgeEnabled } from './enabled';

export const bridgeStore = new RedisBridgeStore(
connection,
Expand All @@ -13,6 +14,7 @@ export const bridgeStore = new RedisBridgeStore(
export const bridgePairings = new RedisBridgePairingStore(connection);

export default createBridgeRouter({
enabled: isBridgeEnabled(),
store: bridgeStore,
pairings: bridgePairings,
authMode: env.BRIDGE_AUTH_MODE,
Expand Down
17 changes: 17 additions & 0 deletions service/src/bridge/router.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,23 @@ afterEach(async () => {
});

describe('paired bridge HTTP API', () => {
test('disabled bridges expose no HTTP routes', async () => {
const app = express();
app.use('/v1/bridge', createBridgeRouter({
enabled: false,
store: new RedisBridgeStore(redis),
pairings: new RedisBridgePairingStore(redis),
authMode: 'static',
adminToken: '',
}));
server = createServer(app);
await new Promise<void>((resolve) => server?.listen(0, '127.0.0.1', resolve));
const address = server.address();
if (address == null || typeof address === 'string') throw new Error('Expected TCP listener');
const response = await fetch(`http://127.0.0.1:${address.port}/v1/bridge/workers/test/status`);
expect(response.status).toBe(404);
});

test('reports authenticated worker readiness without exposing identity or binding data', async () => {
const store = new RedisBridgeStore(redis);
const app = express();
Expand Down
2 changes: 2 additions & 0 deletions service/src/bridge/router.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ const PRINCIPAL_TYPES = new Set<BridgePrincipalType>([
export type BridgeAuthMode = 'static' | 'paired';

export interface BridgeRouterOptions {
enabled?: boolean;
store: RedisBridgeStore;
pairings: RedisBridgePairingStore;
authMode: BridgeAuthMode;
Expand Down Expand Up @@ -130,6 +131,7 @@ function isSettlement(value: unknown): value is CodeBridgeSettlement {

export function createBridgeRouter(options: BridgeRouterOptions): Router {
const router = Router();
if (options.enabled === false) return router;

const configuredWorker = (workerId: string): boolean =>
options.allowDynamicWorkers === true ||
Expand Down
50 changes: 50 additions & 0 deletions service/src/bridge/store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import { BridgeWorkspaceSlots } from './slots';

const PREFIX = 'codeapi:bridge:v1';
const POLL_INTERVAL_MS = 100;
const CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS = 5_000;
const DEFAULT_WORKER_TTL_SECONDS = 60;
const DEFAULT_REDIS_COMMAND_TIMEOUT_MS = 1_000;

Expand Down Expand Up @@ -1901,6 +1902,55 @@ export class RedisBridgeStore {
// settlement before returning an error, even when the caller is gone.
pollError = error;
}
const workspaceRequest =
assignment.executionKind === 'workspace_tool' &&
isWorkspaceToolRequest(assignment.request)
? assignment.request
: undefined;
const cancelledMutation =
signal.aborted &&
workspaceRequest != null &&
(workspaceRequest.operation === 'write_file' ||
workspaceRequest.operation === 'edit_file' ||
workspaceRequest.operation === 'execute_command');
if (cancelledMutation) {
try {
// Keep the acknowledged assignment available long enough for the
// worker to terminate its process tree and commit a clean rejection.
// Closing it first makes that rejection impossible to acknowledge and
// leaves the worker's durable mutation guard armed.
await this.cancel(assignment.assignmentId, assignment);
// Rejected settlements remain valid after the execution deadline.
// Give Stop its own grace so a near-timeout cancellation is not
// misclassified as an ambiguous timeout.
const cancellationDeadlineAtMs =
Date.now() + CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS;
let cancellationPollMs = POLL_INTERVAL_MS;
while (Date.now() < cancellationDeadlineAtMs) {
const raw = await boundedCommand(
this.redis.get(settlementKey(assignment.assignmentId)),
Math.max(
1,
Math.min(
this.redisCommandTimeoutMs,
cancellationDeadlineAtMs - Date.now(),
),
),
'Bridge cancelled workspace settlement poll',
);
if (raw != null) return JSON.parse(raw) as CodeBridgeSettlement;
await delay(
Math.min(
cancellationPollMs,
Math.max(0, cancellationDeadlineAtMs - Date.now()),
),
);
cancellationPollMs = Math.min(cancellationPollMs * 2, 500);
}
} catch (error) {
pollError ??= error;
}
}
const closeKeys = [
assignmentKey(assignment.assignmentId),
settlementKey(assignment.assignmentId),
Expand Down
Loading