diff --git a/README.md b/README.md index e52a549c..b35bd09d 100644 --- a/README.md +++ b/README.md @@ -976,13 +976,22 @@ on the host: unless it or a parent closes it, as Bun and Node do for the processes they start. A cancelled gate waits until every process holding it has exited, including one outside the group that the signal never reached. -- Windows owns only the command itself. - -A nested invocation runs immediately inside the admitted command, and a gate -inside another Quest scope, such as `quest worktree run`, leaves ownership to -that scope: an owner of its own would take the command out of the enclosing -one's reach. Internal compiler or test worker parallelism remains the command's -responsibility. +- Windows runs the command under a small Bun stub in a Job Object that kills + every process left in it once the stub exits, and lets none break away. It + owns every descendant, including detached ones, and stops them once the + command exits or the gate is killed. Windows has no signals to forward: + Ctrl-C reaches every process on the console, and killing the gate + terminates the job. + +A nested invocation runs immediately inside the admitted command. A gate inside +another Quest scope, such as `quest worktree run`, runs its command in a child +cgroup of its own within that scope, which systemd delegates to the scope's +processes, so the gate owns its own subtree and the enclosing scope still +reaches it; a cancelled gate there waits until that child cgroup is empty. +A scope started by an older Quest, without delegation, keeps +ownership instead: a scope of the gate's own would take the command out of +the enclosing one's reach. Internal compiler or test worker parallelism remains +the command's responsibility. This repository's `just check`, `just test`, `just worker-check`, and `just worker-test` run through the gate. @@ -1283,6 +1292,8 @@ is empty, and a second signal kills everything at once. A run inside another Quest scope, such as a gate's, binds its own scope to that one, so the enclosing owner's kill stops it too, and leaves a repeated signal's kill to that owner: it may be the copy of SIGTERM that Just forwards. +The scope is delegated, so a `quest gate` inside it owns its command through a +child cgroup of the scope. Work handed to another service manager, such as `systemd-run` or a container daemon, escapes the scope. If the supervisor itself is killed, its command keeps running and its scratch is kept; the next `quest worktree` command drops the run diff --git a/quest/m2/README.md b/quest/m2/README.md index 849e470f..11510053 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -16,5 +16,4 @@ additional login providers. This milestone does not gate the public release. - [ChatGPT hosted runs](/quest/m2/chatgpt-hosted.md) - conditional on provider approval and current terms - [ChatGPT approval](/quest/m2/chatgpt-waitlist.md) - resolve hosted-use approval before implementation - [Worker dev setup](/quest/m2/worker-dev.md) - per-server cookies, seeded raw output, static assets, one fake GitHub, each test run alone -- [Gate ownership everywhere](/quest/m2/gate-ownership.md) - delegated sub-cgroups for nested gates and Windows Job Objects - [Delete the legacy hold labels](/quest/m2/legacy-labels.md) - once every gated repository pins quest past #326, only `quest:active` and `quest:kix` are holds diff --git a/quest/m2/gate-ownership.md b/quest/m2/gate-ownership.md deleted file mode 100644 index a6ee8bed..00000000 --- a/quest/m2/gate-ownership.md +++ /dev/null @@ -1,19 +0,0 @@ -# [M] Own gated descendants everywhere - -## Goal - -A gate inside a `quest worktree run` scope owns its own subtree, and on -Windows a gate owns its command's descendants, not just the command. - -## Plan - -Decided 2026-10-28 (user, /quest-plan of the follow-ups from a 25-PR /quest-complete run), from [moq-dev/quest#261](https://github.com/moq-dev/quest/pull/261), -which made the gate own descendants through systemd scopes on Linux and a -process group on macOS. - -- **Delegated sub-cgroups.** Set `Delegate=yes` on `quest worktree run` - scopes, so a nested gate can create a child cgroup instead of deferring to - the enclosing scope. Today it defers, because a new scope would let - `cgroup.kill` on the outer scope leave the gated command running. -- **Windows Job Objects.** Assign the command to a Job Object with - kill-on-close, so stopping the gate stops every descendant. diff --git a/scripts/gate-check.sh b/scripts/gate-check.sh index 1eb83c59..f04d78b2 100755 --- a/scripts/gate-check.sh +++ b/scripts/gate-check.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash # Check that an installed CLI's gate admits a lone command, however loaded the -# host, and owns its command's descendants. Inside another gate or Quest scope -# the enclosing owner holds them instead, so CI runs this on its own rather than -# from `just test`. +# host, and owns its command's descendants. Inside another gate or Quest scope, +# the enclosing owner holds them or delivers their signals instead, so a signal +# sent to the gate alone never reaches its command: CI runs this on its own +# rather than from `just test`. set -euo pipefail binary=${1:?usage: gate-check.sh /absolute/path/to/quest} diff --git a/src/cli/gate.ts b/src/cli/gate.ts index db2e7cdf..aea9246f 100644 --- a/src/cli/gate.ts +++ b/src/cli/gate.ts @@ -9,7 +9,8 @@ import { dirname, join, relative, resolve, sep } from "node:path"; import type { Readable } from "node:stream"; import { setTimeout as sleep } from "node:timers/promises"; import { type Cancellation, cancelled, follow, forwards, GROUP, interrupts, received, requested } from "./interrupt"; -import { enclosingScope, systemdScopes } from "./supervise"; +import { stubbed } from "./job"; +import { delegated, nest, questScope, systemdScopes } from "./supervise"; /** * One admitted command: a unique id, the wall-clock time it was admitted, and the process ID of @@ -491,6 +492,24 @@ export const inside: Owner = plain(true); export const scope: Owner = (command, env, cancel) => systemdScopes.run(`quest-gate-${randomUUID()}.scope`, process.cwd(), command, { env, cancel, within: null }); +/** + * Inside a delegated Quest scope, run the command in a new child of `parent`, this process's cgroup, + * which holds every descendant like a scope of its own and stays within the enclosing scope's reach. + * That scope's owner signals every process in it, so this leaves a received signal and the kill at + * its second one to that owner, as `inside` does, and lets the child's processes finish their own + * cleanup before it kills what is left. + */ +export const nested = + (parent: string): Owner => + (command, env, cancel) => + nest(parent, `quest-gate-${randomUUID()}`, process.cwd(), command, { env, cancel }); + +/** + * On Windows, run the command under a stub in a kill-on-close Job Object that forbids breakaway + * (see job.ts), so every descendant stops once the command exits or the gate stops the stub. + */ +export const job: Owner = (command, env, cancel) => direct(...stubbed(command, env), cancel); + /** * Run the command as the leader of a new session, whose process group holds every descendant that * does not move to another group or session. Once the command exits, the rest of its group is @@ -559,23 +578,20 @@ export const group: Owner = (command, env, cancel) => /** The descriptor on which the group owner hands its command the token (see `group`). */ const TOKEN = 3; -/** Whether this process already runs in a Quest scope, which then holds whatever it starts. */ -export function scoped(self = "/proc/self/cgroup"): boolean { - return enclosingScope(self) !== undefined; -} - /** - * The strongest owner this host offers. A nested call runs inside its admitted command, and a call - * inside a Quest scope inside that scope: moving into an owner of its own would take its command out - * of reach of the enclosing one, which also signals it when it is Quest's (see `inside`). Linux - * owns descendants through a systemd scope, and settles for a process group, like macOS, without a - * systemd user manager. Windows owns only the command. + * The strongest owner this host offers. A nested call runs inside its admitted command, which an + * enclosing Quest owner also signals when it is Quest's group or a Quest scope (see `inside`). A + * call inside a Quest scope owns a child cgroup of its own there once the scope is delegated, and + * otherwise runs inside the scope: a scope of its own would take its command out of reach of the + * enclosing one. Linux owns descendants through a systemd scope, and settles for a process group, + * like macOS, without a systemd user manager. Windows owns them through a Job Object. */ export function owner(env = process.env, cgroup = "/proc/self/cgroup"): Owner { - if (process.platform === "win32") return direct; - // A Quest scope around this process holds the command too, since it shares the cgroup. - const enclosed = process.platform === "linux" && scoped(cgroup); - if (env[ADMITTED] === "1" || enclosed) return env[GROUP] === "1" || enclosed ? inside : direct; + // A Quest scope around this process holds the command too, since it shares the cgroup or an ancestor. + const enclosing = process.platform === "linux" ? questScope(cgroup) : undefined; + if (env[ADMITTED] === "1") return env[GROUP] === "1" || enclosing ? inside : direct; + if (process.platform === "win32") return job; + if (enclosing) return delegated(enclosing.unit) ? nested(enclosing.group) : inside; if (process.platform !== "linux") return group; try { systemdScopes.check(); diff --git a/src/cli/job.ts b/src/cli/job.ts new file mode 100644 index 00000000..361ac1fb --- /dev/null +++ b/src/cli/job.ts @@ -0,0 +1,69 @@ +// Windows process ownership for the gate. Node offers no Job Object API and libuv's own job lets +// grandchildren break away silently, so a small Bun program joins a job of its own before it starts +// the command: everything the command starts is created inside that job and cannot break away. + +/** + * Runs as ` -e STUB`, with `BUN_BE_BUN=1` so a compiled Quest runs it as Bun would. The command + * and the caller's `BUN_BE_BUN` arrive as JSON in `QUEST_GATE_JOB`, since Bun's `process.argv` for + * `-e` differs between platforms; the stub restores the caller's environment before the command + * starts. + * + * On Windows it creates a job that kills every process left in it once its last handle closes, and + * joins it. Only the stub holds that handle, so the job ends with the stub: when the command exits, + * when the gate terminates the stub, or when the gate dies and libuv's own kill-on-close job ends + * the stub. Once in a job that forbids breakaway, a process stays in it, and so does everything it + * starts, including detached processes and those that break away from libuv's nested job. + * + * The stub waits out console signals, which reach the command directly, so the command's own + * handling runs. It exits with the command's status, or 127 when the command cannot start, as a + * shell would, and 128 plus the signal's number for a command a signal stopped. Without a job it + * warns and still runs the command, as `direct` would. + * + * Joined onto one line of single-quoted strings, so it stays one plain command-line argument. + */ +export const STUB = [ + "const { spawn } = require('node:child_process');", + "const { keep: original, command: [file, ...args] } = JSON.parse(process.env.QUEST_GATE_JOB);", + "delete process.env.QUEST_GATE_JOB;", + "if (original === null) delete process.env.BUN_BE_BUN; else process.env.BUN_BE_BUN = original;", + "if (process.platform === 'win32') {", + " try {", + " const { dlopen, FFIType, ptr } = require('bun:ffi');", + " const handle = FFIType.u64;", + " const { symbols: kernel } = dlopen('kernel32.dll', {", + " CreateJobObjectW: { args: [FFIType.ptr, FFIType.ptr], returns: handle },", + " SetInformationJobObject: { args: [handle, FFIType.i32, FFIType.ptr, FFIType.u32], returns: FFIType.i32 },", + " AssignProcessToJobObject: { args: [handle, handle], returns: FFIType.i32 },", + " GetCurrentProcess: { args: [], returns: handle },", + " GetLastError: { args: [], returns: FFIType.u32 },", + " });", + // JOBOBJECT_EXTENDED_LIMIT_INFORMATION is 144 bytes on 64-bit Windows; its LimitFlags sit at + // offset 16. Class 9 is JobObjectExtendedLimitInformation, and 0x2000 is + // JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE. No breakaway flag is set, so none is allowed. + " const limits = new BigUint64Array(18);", + " new DataView(limits.buffer).setUint32(16, 0x2000, true);", + " const job = kernel.CreateJobObjectW(null, null);", + " if (!job || !kernel.SetInformationJobObject(job, 9, ptr(limits), 144) || !kernel.AssignProcessToJobObject(job, kernel.GetCurrentProcess()))", + " throw new Error('Windows error ' + kernel.GetLastError());", + " } catch (error) {", + " process.stderr.write('quest gate: cannot own descendants of the command: ' + error.message + String.fromCharCode(10));", + " }", + "}", + "for (const name of ['SIGINT', 'SIGBREAK', 'SIGHUP']) process.on(name, () => {});", + "const child = spawn(file, args, { stdio: 'inherit' });", + "child.once('error', (error) => {", + " process.stderr.write('quest gate: ' + error.message + String.fromCharCode(10));", + " process.exit(127);", + "});", + "child.once('exit', (code, signal) => process.exit(code ?? 128 + require('node:os').constants.signals[signal]));", +] + .map((line) => line.trim()) + .join(" "); + +/** The stub's command line and environment for running `command` with `env` in a job of its own. */ +export function stubbed(command: string[], env: NodeJS.ProcessEnv): [string[], NodeJS.ProcessEnv] { + return [ + [process.execPath, "-e", STUB], + { ...env, BUN_BE_BUN: "1", QUEST_GATE_JOB: JSON.stringify({ keep: env.BUN_BE_BUN ?? null, command }) }, + ]; +} diff --git a/src/cli/main.ts b/src/cli/main.ts index 3316e689..992c5546 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -122,9 +122,11 @@ export function main(argv: string[], out: Output): number | Promise { "that lands PRs should set, the command waits for memory but never for load.\n\n" + "Once the command exits, or is cancelled and exits, its remaining descendants are killed; a second " + "signal kills them all at once. Linux with a systemd 254+ user manager owns every descendant through " + - "a systemd user scope. macOS, and Linux without one, own the command's process group in a new session " + - "without a controlling terminal, so a descendant that leaves the group or session escapes. Windows " + - "owns only the command. Exits with the command's status, or 128 plus the signal's number once cancelled.", + "a systemd user scope, or inside a Quest scope through a child cgroup of its own. macOS, and Linux " + + "without one, own the command's process group in a new session without a controlling terminal, so a " + + "descendant that leaves the group or session escapes. Windows owns every descendant through a " + + "kill-on-close Job Object. Exits with the command's status, or 128 plus the signal's number once " + + "cancelled.", ) .option("--quick", `Start a cheap command after at most ${QUICK_MS / 1000} seconds of waiting`) .argument("", "Executable to run") @@ -743,7 +745,8 @@ export function main(argv: string[], out: Output): number | Promise { .summary("Run a command in a checkout, then release it once every process stops") .description( "Run a command in the checkout, restoring it first when released. The command and its descendants " + - "run in a new systemd user scope; once the command exits, descendants still in the scope are killed. " + + "run in a new, delegated systemd user scope; once the command exits, descendants still in the scope " + + "are killed. A `quest gate` inside it owns its command through a child cgroup of the scope. " + "When no other supervised run remains, the scratch declared at creation is pruned. Put the whole job, " + "such as an agent session or a script that installs, builds, and tests, in one run.\n\n" + "A supervisor that crashes leaves its run recorded; the next worktree command drops it once its scope " + diff --git a/src/cli/supervise.ts b/src/cli/supervise.ts index 61100b17..c459b0ee 100644 --- a/src/cli/supervise.ts +++ b/src/cli/supervise.ts @@ -1,11 +1,24 @@ // Linux process ownership for supervised checkout commands and gated ones. Each run gets its own // systemd user scope, named once and never reused, so its cgroup holds every descendant that does -// not ask another service manager to start it. Other hosts fail explicitly; only the gate settles -// for a process group there (see `group` in gate.ts). +// not ask another service manager to start it. The scope is delegated, so a gate inside it owns its +// command through a child cgroup of its own (see `nest`). Other hosts fail explicitly; only the +// gate settles for a process group there (see `group` in gate.ts). import { type ChildProcessWithoutNullStreams, execFile, execFileSync, spawn } from "node:child_process"; -import { closeSync, existsSync, constants as fs, openSync, readFileSync, watch, writeSync } from "node:fs"; +import { + closeSync, + existsSync, + constants as fs, + mkdirSync, + openSync, + readdirSync, + readFileSync, + rmdirSync, + watch, + writeSync, +} from "node:fs"; import { constants, hostname } from "node:os"; -import { join } from "node:path"; +// Cgroup paths are POSIX paths, also in the tests that parse them on Windows. +import { join } from "node:path/posix"; import type { Readable, Writable } from "node:stream"; import { type Cancellation, cancelled, follow, forwards, interrupts, received, requested } from "./interrupt"; @@ -178,14 +191,6 @@ export function kill(group: string): void { } } -/** The Quest scope, a gate's or a supervised run's, that this process runs in, if any. */ -export function enclosingScope(self = "/proc/self/cgroup"): string | undefined { - const unified = readFileSync(self, "utf8") - .split("\n") - .find((line) => line.startsWith("0::")); - return unified?.match(/\/(quest-[a-z]+-[0-9a-f-]+\.scope)$/)?.[1]; -} - /** Whether the cgroup still holds a process, or `otherwise` when that cannot be read. */ function populatedOr(group: string, otherwise: boolean): boolean { try { @@ -251,132 +256,314 @@ export const systemdScopes: Scopes = { }, run(unit, cwd, command, options = {}) { - return new Promise((resolve, reject) => { - const own = options.cancel ? undefined : interrupts(); - const cancel = options.cancel ?? own!; - const pipe = options.spawned ? "pipe" : "inherit"; - // Once the enclosing owner kills everything it holds, systemd kills this scope too. - const enclosing = options.within === undefined ? enclosingScope() : options.within; - const bound = enclosing ? [`--property=BindsTo=${enclosing}`, "--property=KillSignal=SIGKILL"] : []; - const child = spawn( + // Once the enclosing owner kills everything it holds, systemd kills this scope too. + const enclosing = options.within === undefined ? questScope()?.unit : options.within; + const bound = enclosing ? [`--property=BindsTo=${enclosing}`, "--property=KillSignal=SIGKILL"] : []; + return contain( + [ "systemd-run", - [ - "--user", - "--scope", - "--quiet", - "--collect", - `--unit=${unit}`, - ...bound, - "--expand-environment=no", - "--", - "/bin/sh", - "-c", - handshake, - "quest-worktree", - ...command, - ], - { - cwd, - env: options.env, - stdio: [pipe, pipe, pipe, "pipe", "pipe"], - }, - ); - options.spawned?.(child as ChildProcessWithoutNullStreams); - const report = child.stdio[3] as Readable; - const admission = (child.stdio as unknown as (Writable | null)[])[4]!; - let group: string | undefined; - let admitted = false; - let interrupted = 0; - let failure: Error | undefined; - // Set once every process in the scope was asked to stop; aborted once they no longer get time to. - let signalled = false; - const drain = new AbortController(); - const stop = () => { - if (interrupted) return; - interrupted = cancelled(cancel.signal); - // The shell reads end-of-file and never starts the command. - if (!admitted) return admission.destroy(); - if (!forwards(cancel.signal)) return; - const name = requested(cancel.signal); - signalled = true; - signalScope(unit, name, (error) => { - // The scope is gone once its last process exits, which leaves nothing to signal. - if (!error || !group || !populatedOr(group, true)) return; - process.stderr.write(`quest: cannot signal every process in ${unit}, only its command: ${error}\n`); - child.kill(name); - drain.abort(); - }); - }; - const terminate = () => { - stop(); - // The enclosing owner kills this scope at its own second signal (see `within`). - if (enclosing && received(cancel.signal)) return; - drain.abort(); - if (group) kill(group); - }; - const unfollow = follow(cancel, stop, terminate); - const cleanup = () => { - unfollow(); - own?.close(); - }; + "--user", + "--scope", + "--quiet", + "--collect", + `--unit=${unit}`, + ...bound, + // A gate inside the scope runs its command in a child cgroup of its own (see `nest`). + "--property=Delegate=yes", + "--expand-environment=no", + "--", + "/bin/sh", + "-c", + handshake, + "quest-worktree", + ], + (path) => path.endsWith(`/${unit}`), + { + name: unit, + unsupported: UNSUPPORTED, + launcher: "systemd-run", + signal: (_group, name, done) => signalScope(unit, name, done), + inside: false, + bound: Boolean(enclosing), + }, + cwd, + command, + options, + ); + }, +}; - admission.on("error", (error: NodeJS.ErrnoException) => { - if (error.code !== "EPIPE") failure = error; - }); - let text = ""; - report.setEncoding("utf8"); - report.on("data", (data: string) => { - text += data; - if (group !== undefined || !text.endsWith("\n")) return; - const path = text.slice(0, -1); - // A unique unit name can never name a later scope, so this path stays this run's. - if (!path.startsWith("/") || path.includes("\n") || !path.endsWith(`/${unit}`)) { - failure = new Error(`${UNSUPPORTED}: the command ran outside its scope (${path})`); - admission.destroy(); - return; - } - group = join("/sys/fs/cgroup", path); - if (interrupted) return; - admitted = true; - admission.end("admitted\n"); - }); +/** How `contain` names, signals, and shares its run's cgroup. */ +interface Containment { + /** The run's scope or cgroup. */ + name: string; + /** Why the command cannot run, prefixed to every failure before it starts. */ + unsupported: string; + /** The program that starts the handshake shell. */ + launcher: string; + /** Send `name` to every process in the cgroup `group`, then call `done`, with why when it could not. */ + signal(group: string, name: NodeJS.Signals, done: (error?: string) => void): void; + /** + * Whether the cgroup lies within an enclosing Quest scope, whose owner delivers each signal it + * receives to every process in the scope, this cgroup's included. A signal this process received + * then already reached them all, so it is not delivered again, and they get to finish their + * cleanup as if it had been. + */ + inside: boolean; + /** + * Whether an enclosing Quest owner kills this cgroup at its own second signal, so a repeated + * signal this process receives, which may be a copy of that owner's, leaves the kill to it. + */ + bound: boolean; +} - child.once("error", (error) => { - cleanup(); - reject( - (error as NodeJS.ErrnoException).code === "ENOENT" - ? new Error(`${UNSUPPORTED}: systemd-run is missing`) - : error, +/** + * Start the handshake shell through `launch`, admit the command once the shell reports a cgroup + * that `owns` accepts, and once the command exits, kill whatever is left in that cgroup and wait + * until it is empty (see `Scopes.run`). + */ +function contain( + launch: string[], + owns: (path: string) => boolean, + describe: Containment, + cwd: string, + command: string[], + options: RunOptions, +): Promise { + return new Promise((resolve, reject) => { + const own = options.cancel ? undefined : interrupts(); + const cancel = options.cancel ?? own!; + const pipe = options.spawned ? "pipe" : "inherit"; + const child = spawn(launch[0], [...launch.slice(1), ...command], { + cwd, + env: options.env, + stdio: [pipe, pipe, pipe, "pipe", "pipe"], + }); + options.spawned?.(child as ChildProcessWithoutNullStreams); + const report = child.stdio[3] as Readable; + const admission = (child.stdio as unknown as (Writable | null)[])[4]!; + let group: string | undefined; + let admitted = false; + let interrupted = 0; + let failure: Error | undefined; + // Set once every process in the cgroup was asked to stop; aborted once they no longer get time to. + let signalled = false; + const drain = new AbortController(); + const stop = () => { + if (interrupted) return; + interrupted = cancelled(cancel.signal); + // The shell reads end-of-file and never starts the command. + if (!admitted) return admission.destroy(); + if (!(describe.inside ? forwards(cancel.signal, true) : forwards(cancel.signal))) { + // The enclosing scope's owner already signalled every process here (see `inside`). + signalled = describe.inside; + return; + } + const name = requested(cancel.signal); + signalled = true; + describe.signal(group!, name, (error) => { + // The cgroup empties once its last process exits, which leaves nothing to signal. + if (!error || !group || !populatedOr(group, true)) return; + process.stderr.write( + `quest: cannot signal every process in ${describe.name}, only its command: ${error}\n`, ); + child.kill(name); + drain.abort(); }); - child.once("exit", async (code, signal) => { - try { - // Descendants that outlive the command are orphans of this run: stop them. - if (group) { - try { - // Every process got the signal, so let each finish its own cleanup, even one that - // outlives the wrappers that exited first, until a second signal kills them. - if (signalled) await vacated(group, drain.signal); - kill(group); - await vacated(group); - } catch (error) { - throw new Retained( - `cannot confirm that ${unit} stopped: ${error instanceof Error ? error.message : error}`, - ); - } - } - if (failure) throw failure; - if (!admitted && !interrupted) - throw new Error( - `${UNSUPPORTED}: systemd-run exited with ${code ?? signal} before the command started`, + }; + const terminate = () => { + stop(); + // The enclosing owner kills this cgroup at its own second signal (see `bound`). + if (describe.bound && received(cancel.signal)) return; + drain.abort(); + if (group) kill(group); + }; + const unfollow = follow(cancel, stop, terminate); + const cleanup = () => { + unfollow(); + own?.close(); + }; + + admission.on("error", (error: NodeJS.ErrnoException) => { + if (error.code !== "EPIPE") failure = error; + }); + let text = ""; + report.setEncoding("utf8"); + report.on("data", (data: string) => { + text += data; + if (group !== undefined || !text.endsWith("\n")) return; + const path = text.slice(0, -1); + // A unique name can never name a later cgroup, so this path stays this run's. + if (!path.startsWith("/") || path.includes("\n") || !owns(path)) { + failure = new Error(`${describe.unsupported}: the command ran outside its cgroup (${path})`); + admission.destroy(); + return; + } + group = join("/sys/fs/cgroup", path); + if (interrupted) return; + admitted = true; + admission.end("admitted\n"); + }); + + child.once("error", (error) => { + cleanup(); + reject( + (error as NodeJS.ErrnoException).code === "ENOENT" + ? new Error(`${describe.unsupported}: ${describe.launcher} is missing`) + : error, + ); + }); + child.once("exit", async (code, signal) => { + try { + // Descendants that outlive the command are orphans of this run: stop them. + if (group) { + try { + // Every process got the signal, so let each finish its own cleanup, even one that + // outlives the wrappers that exited first, until a second signal kills them. + if (signalled) await vacated(group, drain.signal); + kill(group); + await vacated(group); + } catch (error) { + throw new Retained( + `cannot confirm that ${describe.name} stopped: ${error instanceof Error ? error.message : error}`, ); - resolve(interrupted || (code ?? 128 + constants.signals[signal!])); - } catch (error) { - reject(error); - } finally { - cleanup(); + } } - }); + if (failure) throw failure; + if (!admitted && !interrupted) + throw new Error( + `${describe.unsupported}: ${describe.launcher} exited with ${code ?? signal} before the command started`, + ); + resolve(interrupted || (code ?? 128 + constants.signals[signal!])); + } catch (error) { + reject(error); + } finally { + cleanup(); + } }); - }, -}; + }); +} + +/** Remove an empty cgroup and any empty child cgroups left in it; one already gone is fine. */ +function prune(group: string): void { + let entries; + try { + entries = readdirSync(group, { withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + for (const entry of entries) if (entry.isDirectory()) prune(join(group, entry.name)); + try { + rmdirSync(group); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } +} + +/** + * Send `signal` to every process in the cgroup and its children, until a pass finds none it has not + * signalled yet, as systemd does for a unit, then call `done`. + */ +function signalGroup(group: string, signal: NodeJS.Signals, done: (error?: string) => void): void { + const sent = new Set(); + const pass = (dir: string): boolean => { + let found = false; + let entries; + try { + entries = readdirSync(dir, { withFileTypes: true }); + for (const line of readFileSync(join(dir, "cgroup.procs"), "utf8").split("\n")) { + const pid = Number(line); + if (!line || sent.has(pid)) continue; + sent.add(pid); + found = true; + try { + process.kill(pid, signal); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } + } + } catch (error) { + // The cgroup was removed while it was read. + if (["ENOENT", "ENODEV"].includes((error as NodeJS.ErrnoException).code ?? "")) return found; + throw error; + } + for (const entry of entries) if (entry.isDirectory() && pass(join(dir, entry.name))) found = true; + return found; + }; + try { + while (pass(group)); + } catch (error) { + return done(error instanceof Error ? error.message : String(error)); + } + done(); +} + +/** Matches a Quest scope's unit as a component of a cgroup path. */ +const QUEST_SCOPE = /\/(quest-[a-z]+-[0-9a-f-]+\.scope)(?=\/|$)/g; + +/** + * This process's unified cgroup and the innermost Quest scope holding it, which may be the cgroup + * itself or an ancestor; undefined outside every Quest scope. + */ +export function questScope(self = "/proc/self/cgroup"): { unit: string; group: string } | undefined { + const path = readFileSync(self, "utf8") + .split("\n") + .find((line) => line.startsWith("0::")) + ?.slice(3); + const unit = path === undefined ? undefined : [...path.matchAll(QUEST_SCOPE)].at(-1)?.[1]; + return unit === undefined ? undefined : { unit, group: join("/sys/fs/cgroup", path!) }; +} + +/** + * Whether systemd delegated the scope's cgroup subtree, so its processes may create child cgroups. + * A scope from a Quest that predates delegation, or a manager that cannot be asked, is not. + */ +export function delegated(unit: string): boolean { + try { + return ( + execFileSync("systemctl", ["--user", "show", "--property=Delegate", "--value", unit], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }).trim() === "yes" + ); + } catch { + return false; + } +} + +/** + * Run a command in a new child cgroup of `parent`, inside a delegated Quest scope, like a scope of + * its own (see `Scopes.run`): every descendant stays in the child, which the enclosing scope's + * `cgroup.kill` also reaches, and the child is removed once empty. The handshake shell moves itself + * into the child before it reports, so nothing it starts runs outside it. + */ +export async function nest( + parent: string, + name: string, + cwd: string, + command: string[], + options: RunOptions = {}, +): Promise { + const group = join(parent, name); + const unsupported = `quest gate cannot run the command in a child cgroup of its Quest scope (${group})`; + mkdirSync(group); + let status: number; + try { + status = await contain( + // Writing 0 to cgroup.procs moves the writing process, the shell itself. + ["/bin/sh", "-c", `echo 0 >"$0/cgroup.procs" || exit 125; ${handshake}`, group], + (path) => join("/sys/fs/cgroup", path) === group, + { name: group, unsupported, launcher: "/bin/sh", signal: signalGroup, inside: true, bound: true }, + cwd, + command, + options, + ); + } catch (error) { + // The cgroup may still hold processes. + if (!(error instanceof Retained)) prune(group); + throw error; + } + prune(group); + return status; +} diff --git a/test/gate.test.ts b/test/gate.test.ts index a77176f3..e4ffc7af 100644 --- a/test/gate.test.ts +++ b/test/gate.test.ts @@ -1,4 +1,5 @@ import { spawn, spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; import { once } from "node:events"; import { chmodSync, @@ -35,11 +36,12 @@ import { QUICK_MS, RECENT_MS, SETTLE_MS, + job, scope, - scoped, } from "../src/cli/gate"; import { forwards, foregroundStat } from "../src/cli/interrupt"; import { gateBoundary, main } from "../src/cli/main"; +import { questScope } from "../src/cli/supervise"; import { created, onTerminal, run, signalled, tempDir } from "./fixture"; const GiB = 2 ** 30; @@ -578,24 +580,34 @@ test.skipIf(process.platform === "win32")("cancellation forwards the received si test("a nested call, or one inside a Quest scope, leaves its command to the enclosing owner", () => { const dir = tempDir(); - const slice = "0::/user.slice/user-1000.slice/user@1000.service/app.slice"; - const cgroup = (unit: string) => { - const file = join(dir, unit); - writeFileSync(file, `${slice}/${unit}\n`); + const slice = "/user.slice/user-1000.slice/user@1000.service/app.slice"; + const cgroup = (path: string) => { + const file = join(dir, path.replaceAll("/", "_")); + writeFileSync(file, `0::${slice}/${path}\n`); return file; }; + const unit = "quest-worktree-0f18849e-1c60-4b6e-9d2a-7a1c2b3d4e5f.scope"; const terminal = cgroup("app-terminal-42.scope"); - const worktree = cgroup("quest-worktree-0f18849e-1c60-4b6e-9d2a-7a1c2b3d4e5f.scope"); + const worktree = cgroup(unit); const gated = cgroup("quest-gate-0f18849e-1c60-4b6e-9d2a-7a1c2b3d4e5f.scope"); - expect(scoped(worktree)).toBe(true); - expect(scoped(gated)).toBe(true); - expect(scoped(terminal)).toBe(false); + // A gate's child cgroup, and one a gate inside it made, stay in the scope. + const child = cgroup(`${unit}/quest-gate-1a2b/quest-gate-3c4d`); + expect(questScope(worktree)).toEqual({ unit, group: `/sys/fs/cgroup${slice}/${unit}` }); + expect(questScope(gated)?.unit).toBe("quest-gate-0f18849e-1c60-4b6e-9d2a-7a1c2b3d4e5f.scope"); + expect(questScope(child)).toEqual({ + unit, + group: `/sys/fs/cgroup${slice}/${unit}/quest-gate-1a2b/quest-gate-3c4d`, + }); + expect(questScope(terminal)).toBeUndefined(); + expect(questScope(cgroup(`${unit}-not`))).toBeUndefined(); if (process.platform === "win32") return expect(owner({ QUEST_GATE_ADMITTED: "1" })).toBe(direct); // The enclosing owner signals the command too unless it was not one of Quest's. expect(owner({ QUEST_GATE_ADMITTED: "1" }, terminal)).toBe(direct); expect(owner({ QUEST_GATE_ADMITTED: "1", QUEST_GATE_GROUP: "1" }, terminal)).toBe(inside); if (process.platform !== "linux") return; expect(owner({ QUEST_GATE_ADMITTED: "1" }, gated)).toBe(inside); + expect(owner({ QUEST_GATE_ADMITTED: "1" }, child)).toBe(inside); + // No such scope runs, so it is not delegated. expect(owner({}, worktree)).toBe(inside); }); @@ -764,6 +776,217 @@ describe.runIf(process.platform === "linux")("a systemd scope", () => { }); }); +/** + * Run a Bun script that picks the gate's owner in a scope started by `launch`, outside any admitted + * command, and reports which one it picked; with `run`, it also runs a command through that owner + * that leaves a descendant behind and reports what became of it. + */ +async function insideScope(launch: (command: string[], env: NodeJS.ProcessEnv) => Promise, run: boolean) { + const dir = tempDir(); + const script = join(dir, "inside.ts"); + const result = join(dir, "result.json"); + const gate = fileURLToPath(new URL("../src/cli/gate.ts", import.meta.url)); + writeFileSync( + script, + `import { spawnSync } from "node:child_process"; + import { existsSync, readFileSync, writeFileSync } from "node:fs"; + import { direct, group, inside, job, owner, scope } from ${JSON.stringify(gate)}; + const dir = ${JSON.stringify(dir)}; + const own = owner(process.env); + const kind = new Map([[direct, "direct"], [group, "group"], [inside, "inside"], [job, "job"], [scope, "scope"]]).get(own) ?? "nested"; + let report = { kind }; + if (${run}) { + const never = new AbortController().signal; + const command = "cat /proc/self/cgroup > " + dir + "/cgroup; sleep 1000 & echo $! > " + dir + "/pid; exit 7"; + const status = await own(["/bin/sh", "-c", command], process.env, { signal: never, kill: never }); + const path = readFileSync(dir + "/cgroup", "utf8").trim().slice(3); + const pid = readFileSync(dir + "/pid", "utf8").trim(); + const ps = spawnSync("ps", ["-o", "stat=", "-p", pid], { encoding: "utf8" }); + const stopped = ps.status !== 0 || ps.stdout.trim().startsWith("Z"); + report = { kind, status, path, stopped, removed: !existsSync("/sys/fs/cgroup" + path) }; + } + writeFileSync(${JSON.stringify(result)}, JSON.stringify(report));`, + ); + const env = { ...process.env }; + delete env.QUEST_GATE_ADMITTED; + expect(await launch([process.execPath, script], env)).toBe(0); + return JSON.parse(readFileSync(result, "utf8")); +} + +describe.runIf(process.platform === "linux")("a gate inside a Quest scope", () => { + test("owns a child cgroup of a delegated scope and removes it", { timeout: 30_000 }, async () => { + const report = await insideScope((command, env) => scope(command, env, cancellation().cancel), true); + expect(report).toMatchObject({ kind: "nested", status: 7, stopped: true, removed: true }); + // The child sits inside the enclosing scope, so that scope's cgroup.kill still reaches it. + expect(report.path).toMatch(/\/quest-gate-[0-9a-f-]+\.scope\/quest-gate-[0-9a-f-]+$/); + }); + + test("leaves its command to a scope that is not delegated", { timeout: 30_000 }, async () => { + // A scope started as a Quest before delegation would have. + const unit = `quest-worktree-${randomUUID()}.scope`; + const launch = (command: string[], env: NodeJS.ProcessEnv) => + new Promise((resolve, reject) => { + const child = spawn( + "systemd-run", + ["--user", "--scope", "--quiet", "--collect", `--unit=${unit}`, "--", ...command], + { env, stdio: "inherit" }, + ); + child.once("error", reject); + child.once("exit", (code) => resolve(code ?? 1)); + }); + expect(await insideScope(launch, false)).toEqual({ kind: "inside" }); + }); + + /** + * A Bun script that runs `command` through the owner a gate picks, outside any admitted command, + * and exits with its status. It follows its own signals as the gate does, unless `abort` names a + * file whose creation makes it cancel the run itself, without a signal. + */ + function nestedGate(dir: string, command: string[], abort?: string): string[] { + const script = join(dir, "nested.ts"); + const gate = fileURLToPath(new URL("../src/cli/gate.ts", import.meta.url)); + const interrupt = fileURLToPath(new URL("../src/cli/interrupt.ts", import.meta.url)); + writeFileSync( + script, + `import { existsSync } from "node:fs"; + import { direct, inside, owner } from ${JSON.stringify(gate)}; + import { cancelled, interrupts } from ${JSON.stringify(interrupt)}; + const own = owner(process.env); + if (own === direct || own === inside) process.exit(90); + const signals = interrupts(); + const stop = new AbortController(); + const abort = ${JSON.stringify(abort ?? null)}; + const cancel = abort ? { signal: stop.signal, kill: new AbortController().signal } : signals; + if (abort) { + const poll = setInterval(() => { + if (!existsSync(abort)) return; + clearInterval(poll); + stop.abort(new Error("cancelled by its caller")); + }, 25); + } + const status = await own(${JSON.stringify(command)}, process.env, cancel); + signals.close(); + process.exit(cancel.signal.aborted ? cancelled(cancel.signal) : status);`, + ); + return [process.execPath, script]; + } + + const outside = () => { + const env = { ...process.env }; + delete env.QUEST_GATE_ADMITTED; + return env; + }; + + test("lets a phase finish its cleanup once the enclosing scope signals it", { timeout: 30_000 }, async () => { + const dir = tempDir(); + const leaf = signalled(dir, ["SIGTERM"], 200); + const { stop, cancel } = cancellation(); + const result = scope(nestedGate(dir, phase(dir, leaf.script)), outside(), cancel); + await created(leaf.ready); + stop.abort("SIGTERM"); + expect(await result).toBe(128 + constants.signals.SIGTERM); + // Delivered once, by the enclosing scope, and the nested gate did not kill the leaf once Just exited. + expect(leaf.log()).toEqual(["SIGTERM"]); + expect(existsSync(leaf.done)).toBe(true); + }); + + test("cancelled by its caller, signals every process in its child cgroup", { timeout: 30_000 }, async () => { + const dir = tempDir(); + const leaf = signalled(dir, ["SIGTERM"], 200); + const abort = join(dir, "abort"); + // The shell waits on the leaf in the background, so only a signal to the whole cgroup reaches it. + const command = ["/bin/sh", "-c", `${quote(process.execPath)} ${quote(leaf.script)} & wait`]; + const result = scope(nestedGate(dir, command, abort), outside(), cancellation().cancel); + await created(leaf.ready); + writeFileSync(abort, ""); + expect(await result).toBe(128 + constants.signals.SIGTERM); + expect(leaf.log()).toEqual(["SIGTERM"]); + expect(existsSync(leaf.done)).toBe(true); + }); +}); + +/** Whether a process is still running, on any platform. */ +function running(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +/** A Bun command that starts a detached descendant, records its PID, then runs `rest`. */ +function detached(dir: string, rest: string) { + const pid = join(dir, "descendant"); + const start = + "const c = require('node:child_process').spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], " + + "{ detached: true, stdio: 'ignore' }); c.unref(); " + + `require('node:fs').writeFileSync(${JSON.stringify(pid)}, String(c.pid)); `; + return { + command: [process.execPath, "-e", start + rest], + pid: () => Number(readFileSync(pid, "utf8")), + }; +} + +describe("the Windows job stub", () => { + test("passes the exact exit status and restores the caller's environment", async () => { + const script = + "process.exit(process.env.BUN_BE_BUN === process.env.EXPECTED && !('QUEST_GATE_JOB' in process.env) ? 7 : 1)"; + const env: NodeJS.ProcessEnv = { ...process.env, EXPECTED: "keep" }; + delete env.BUN_BE_BUN; + const command = [process.execPath, "-e", script]; + expect(await job(command, { ...env, BUN_BE_BUN: "keep" }, cancellation().cancel)).toBe(7); + const unset = "process.exit(process.env.BUN_BE_BUN === undefined ? 7 : 1)"; + expect(await job([process.execPath, "-e", unset], env, cancellation().cancel)).toBe(7); + }); + + test("passes the command's arguments as running it directly would", async () => { + // Compared with `direct`, since Bun's process.argv for a script passed with -e differs on Windows. + const dir = tempDir(); + const args = ["a b", 'say "hi"', "back\\slash\\", ""]; + const script = "require('node:fs').writeFileSync(process.env.OUT, JSON.stringify(process.argv.slice(1)))"; + const argv = async (owner: Owner, out: string) => { + const env = { ...process.env, OUT: join(dir, out) }; + expect(await owner([process.execPath, "-e", script, ...args], env, cancellation().cancel)).toBe(0); + return JSON.parse(readFileSync(env.OUT, "utf8")); + }; + const expected = await argv(direct, "direct"); + if (process.platform !== "win32") expect(expected).toEqual(args); + expect(await argv(job, "job")).toEqual(expected); + }); + + test("reports a command that cannot start as a shell would", async () => { + expect(await job([join(tempDir(), "missing")], process.env, cancellation().cancel)).toBe(127); + }); +}); + +describe.runIf(process.platform === "win32")("a job object", () => { + test("is the gate's owner outside an admitted command", () => { + expect(owner({})).toBe(job); + }); + + test("keeps the exact exit status and stops a detached descendant", { timeout: 30_000 }, async () => { + const { command, pid } = detached(tempDir(), "process.exit(7);"); + expect(await job(command, process.env, cancellation().cancel)).toBe(7); + await until(() => !running(pid()), "the detached descendant to stop"); + }); + + test("killing stops the command and a detached descendant", { timeout: 30_000 }, async () => { + const dir = tempDir(); + const ready = join(dir, "ready"); + const { command, pid } = detached( + dir, + `require('node:fs').writeFileSync(${JSON.stringify(ready)}, ''); setInterval(() => {}, 1000);`, + ); + const { kill, cancel } = cancellation(); + const result = job(command, process.env, cancel); + await until(() => existsSync(ready), "the command to start"); + kill.abort(); + expect(await result).not.toBe(0); + await until(() => !running(pid()), "the detached descendant to stop"); + }); +}); + describe.skipIf(process.platform === "win32")("a process group", () => { test("delivers SIGINT once to a command under a nested gate", { timeout: 30_000 }, async () => { const dir = tempDir();