Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions packages/cli/src/commands/preview.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { afterEach, describe, expect, it, vi } from "vitest";
import { runCommand } from "citty";
import {
default as previewCommand,
drainEmbeddedPreviewResources,
foregroundPreviewReadyPayload,
handlePreviewKillAll,
handlePreviewList,
Expand Down Expand Up @@ -493,3 +494,33 @@ describe("waitForStudioChildClose", () => {
expect(signalTarget.off).toHaveBeenCalledTimes(2);
});
});

describe("embedded preview resource drain", () => {
it("closes child registration before cancelling and awaiting renders", async () => {
const order: string[] = [];
let finishRenderDrain!: () => void;
const renderDrain = new Promise<void>((resolve) => {
finishRenderDrain = resolve;
});
const draining = drainEmbeddedPreviewResources({
beginProcessDrain: () => order.push("processes"),
disposeRenders: async () => {
order.push("renders:start");
await renderDrain;
order.push("renders:end");
},
closeThumbnailBrowser: async () => {
order.push("thumbnail");
},
drainBrowserPool: async () => {
order.push("browsers");
},
});
await Promise.resolve();

expect(order).toEqual(["processes", "renders:start", "thumbnail"]);
finishRenderDrain();
await draining;
expect(order).toEqual(["processes", "renders:start", "thumbnail", "renders:end", "browsers"]);
});
});
26 changes: 21 additions & 5 deletions packages/cli/src/commands/preview.ts
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,17 @@ interface BrowserLaunchOptions {
browserNoGpu?: boolean;
}

export async function drainEmbeddedPreviewResources(input: {
beginProcessDrain: () => void;
disposeRenders: () => Promise<void>;
closeThumbnailBrowser: () => Promise<void>;
drainBrowserPool: () => Promise<void>;
}): Promise<void> {
input.beginProcessDrain();
await Promise.allSettled([input.disposeRenders(), input.closeThumbnailBrowser()]);
await input.drainBrowserPool().catch(() => {});
}

interface StudioLaunchOptions extends BrowserLaunchOptions {
projectName?: string;
autoProxy?: boolean;
Expand Down Expand Up @@ -1566,7 +1577,7 @@ async function runEmbeddedMode(
// Compute everything that may throw before acquiring the fs.watch handle.
// Once createStudioServer returns, every subsequent exit path must close it.
const serverBuildSignature = await loadPreviewServerBuildSignature();
const { app, watcher } = createStudioServer({
const { app, watcher, dispose } = createStudioServer({
projectDir: dir,
projectName: pName,
autoProxy: options?.autoProxy,
Expand All @@ -1584,6 +1595,7 @@ async function runEmbeddedMode(
options?.browserGpuMode,
);
} catch (err: unknown) {
await dispose().catch(() => {});
watcher.close();
reportPreviewFailure(
Boolean(options?.json),
Expand All @@ -1598,6 +1610,7 @@ async function runEmbeddedMode(
// createStudioServer acquires an fs.watch handle before port discovery.
// Reuse owns no local server, so release that handle before returning or
// the otherwise-finished CLI process remains alive indefinitely.
await dispose().catch(() => {});
watcher.close();
const url = `http://localhost:${result.port}`;
if (options?.json) {
Expand Down Expand Up @@ -1685,10 +1698,13 @@ async function runEmbeddedMode(
// Kill ffmpeg first (sync, fast), then drain browsers (async, slower).
const cleanup = async () => {
const { closeThumbnailBrowser } = await import("../server/studioServer.js");
const { drainBrowserPool, killTrackedProcesses } = await import("@hyperframes/engine");
killTrackedProcesses();
await closeThumbnailBrowser().catch(() => {});
await drainBrowserPool().catch(() => {});
const { beginTrackedProcessDrain, drainBrowserPool } = await import("@hyperframes/engine");
await drainEmbeddedPreviewResources({
beginProcessDrain: beginTrackedProcessDrain,
disposeRenders: dispose,
closeThumbnailBrowser,
drainBrowserPool,
});
};

cleanup()
Expand Down
7 changes: 5 additions & 2 deletions packages/cli/src/server/studioServer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,8 @@ export interface StudioServerOptions {
export interface StudioServer {
app: Hono;
watcher: ProjectWatcher;
/** Cancel and await every render owned by this server. */
dispose(): Promise<void>;
/** Exposed for tests: the adapter handed to the shared studio API (carries
* the resolved `autoProxy` flag the preview routes read). */
adapter: PreviewApiAdapter;
Expand Down Expand Up @@ -474,7 +476,7 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {

// Run render asynchronously, mutating the state object
const startTime = Date.now();
(async () => {
const completion = (async () => {
let renderJob: RenderJob | undefined;
const removeCancelledOutput = () => {
// User-initiated cancel: not a failure. Remove any output so the
Expand Down Expand Up @@ -567,6 +569,7 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
}
}
})();
state.completion = completion;

return state;
},
Expand Down Expand Up @@ -983,5 +986,5 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
return c.html(html);
});

return { app, watcher, adapter };
return { app, watcher, adapter, dispose: () => api.dispose() };
}
36 changes: 35 additions & 1 deletion packages/cli/src/utils/orphanCleanup.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import { describe, it, expect } from "vitest";
import { describe, it, expect, vi } from "vitest";
import { spawn } from "node:child_process";
import {
isProcessDescendant,
killOwnedOrphanedFfmpegProcesses,
killProcessTree,
killOrphanedProcesses,
processIdentity,
Expand Down Expand Up @@ -72,6 +73,39 @@ describe("process-tree ownership", () => {
});
});

describe("owned FFmpeg orphan cleanup", () => {
it("kills only the ownership-verified PID list", () => {
const kill = vi.fn();
const records = [
{ pid: 41, identity: "linux:one" },
{ pid: 42, identity: "linux:two" },
];

expect(
killOwnedOrphanedFfmpegProcesses(
records,
kill,
(pid) => records.find((record) => record.pid === pid)?.identity ?? null,
),
).toBe(2);
expect(kill.mock.calls.map(([pid]) => pid)).toEqual([41, 42]);
expect(kill.mock.calls.every(([, , stillOwned]) => stillOwned())).toBe(true);
});

it("does not kill when the PID birth identity changed after discovery", () => {
const kill = vi.fn();

expect(
killOwnedOrphanedFfmpegProcesses(
[{ pid: 41, identity: "linux:original" }],
kill,
() => "linux:reused",
),
).toBe(0);
expect(kill).not.toHaveBeenCalled();
});
});

describe.skipIf(!IS_UNIX)("killProcessTree", () => {
it("kills a process and all its children", async () => {
// Spawn a parent that spawns two sleeping children
Expand Down
135 changes: 45 additions & 90 deletions packages/cli/src/utils/orphanCleanup.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,23 @@
import { execFileSync, execSync } from "node:child_process";
import { readFileSync } from "node:fs";
import {
findOwnedOrphanedFfmpegProcesses,
type OwnedFfmpegProcess,
processIdentity,
processParentPid,
} from "@hyperframes/engine/process-tracker";

export { processIdentity };

/**
* Find and kill orphaned Chrome processes from previous crashed sessions.
* Find and kill orphaned Chrome and HyperFrames-owned FFmpeg processes from
* previous crashed sessions.
* Targets both chrome-headless-shell (production/CI) and Google Chrome
* launched by Puppeteer (dev mode). Puppeteer Chrome is identified by the
* `puppeteer_dev_chrome_profile` marker in its user-data-dir argument.
*
* An orphan is a process whose PPID=1 (reparented to init/launchd after
* FFmpeg recovery additionally requires a private process-tracker record with
* a matching birth identity, so an unrelated same-user encoder is never
* selected by name. An orphan is a process whose PPID=1 (reparented to init/launchd after
* its parent died). We kill the orphan's entire subtree so child helper
* processes (GPU, renderer, network, etc.) are also cleaned up.
*
Expand All @@ -26,7 +36,27 @@ export function killOrphanedProcesses(): number {
}

killed += killOrphansByName("puppeteer_dev_chrome_profile");
killed += killOwnedOrphanedFfmpegProcesses();

return killed;
}

export function killOwnedOrphanedFfmpegProcesses(
records: OwnedFfmpegProcess[] = findOwnedOrphanedFfmpegProcesses(),
kill: (
pid: number,
signal?: NodeJS.Signals,
stillOwned?: () => boolean,
) => void = killProcessTree,
identityForPid: (pid: number) => string | null = processIdentity,
): number {
let killed = 0;
for (const record of records) {
const stillOwned = () => identityForPid(record.pid) === record.identity;
if (!stillOwned()) continue;
kill(record.pid, "SIGTERM", stillOwned);
killed++;
}
return killed;
}

Expand All @@ -44,7 +74,12 @@ export function killOrphanedProcesses(): number {
* ignore, and leaving a preview server alive is the worse failure here. Do not
* pass SIGTERM expecting a clean shutdown on Windows.
*/
export function killProcessTree(pid: number, signal: NodeJS.Signals = "SIGTERM"): void {
export function killProcessTree(
pid: number,
signal: NodeJS.Signals = "SIGTERM",
stillOwned: () => boolean = () => true,
): void {
if (!stillOwned()) return;
if (process.platform === "win32") {
try {
execFileSync("taskkill", windowsProcessTreeKillArgs(pid), {
Expand All @@ -60,8 +95,10 @@ export function killProcessTree(pid: number, signal: NodeJS.Signals = "SIGTERM")

const descendants = getDescendants(pid);
const allPids = [...descendants.reverse(), pid];
const identities = new Map(allPids.map((candidate) => [candidate, processIdentity(candidate)]));

for (const p of allPids) {
if (!stillOwned()) return;
try {
process.kill(p, signal);
} catch {
Expand All @@ -72,7 +109,10 @@ export function killProcessTree(pid: number, signal: NodeJS.Signals = "SIGTERM")
// Escalate to SIGKILL after a short grace period for any survivors.
if (signal !== "SIGKILL") {
setTimeout(() => {
if (!stillOwned()) return;
for (const p of allPids) {
const identity = identities.get(p);
if (!identity || processIdentity(p) !== identity) continue;
try {
process.kill(p, "SIGKILL");
} catch {
Expand All @@ -87,85 +127,8 @@ export function windowsProcessTreeKillArgs(pid: number): string[] {
return ["/PID", String(pid), "/T", "/F"];
}

/**
* Return a process birth token suitable for detecting PID reuse. The token is
* diagnostic state only: callers must still prove the live server is a
* descendant before treating a saved wrapper as the owned process-tree root.
*/
export function processIdentity(pid: number): string | null {
if (!Number.isInteger(pid) || pid <= 0) return null;
try {
if (process.platform === "win32") {
const created = execFileSync(
"powershell.exe",
[
"-NoProfile",
"-NonInteractive",
"-Command",
`$p = Get-CimInstance Win32_Process -Filter 'ProcessId = ${pid}' -ErrorAction SilentlyContinue; if ($p) { $p.CreationDate.ToFileTimeUtc() }`,
],
{
encoding: "utf8",
timeout: 2000,
stdio: ["pipe", "pipe", "ignore"],
windowsHide: true,
},
).trim();
return created ? `windows:${created}` : null;
}

if (process.platform === "linux") {
const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
const fields = stat
.slice(stat.lastIndexOf(") ") + 2)
.trim()
.split(/\s+/);
const startTicks = fields[19]; // field 22 overall; fields starts at process state (3)
return startTicks ? `linux:${startTicks}` : null;
}

const started = execFileSync("ps", ["-o", "lstart=", "-p", String(pid)], {
encoding: "utf8",
timeout: 2000,
}).trim();
return started ? `posix:${started}` : null;
} catch {
return null;
}
}

type ParentPidLookup = (pid: number) => number | null;

function processParentPid(pid: number): number | null {
try {
const output =
process.platform === "win32"
? execFileSync(
"powershell.exe",
[
"-NoProfile",
"-NonInteractive",
"-Command",
`$p = Get-CimInstance Win32_Process -Filter 'ProcessId = ${pid}' -ErrorAction SilentlyContinue; if ($p) { $p.ParentProcessId }`,
],
{
encoding: "utf8",
timeout: 2000,
stdio: ["pipe", "pipe", "ignore"],
windowsHide: true,
},
)
: execFileSync("ps", ["-o", "ppid=", "-p", String(pid)], {
encoding: "utf8",
timeout: 2000,
});
const parentPid = Number(output.trim());
return Number.isInteger(parentPid) && parentPid > 0 ? parentPid : null;
} catch {
return null;
}
}

/**
* Prove that `childPid` currently belongs to the process tree rooted at
* `ancestorPid`. The walk fails closed on missing, invalid, or cyclic process
Expand Down Expand Up @@ -254,13 +217,5 @@ function getUid(): string | null {
}

function isOrphan(pid: number): boolean {
try {
const ppid = execSync(`ps -p ${pid} -o ppid=`, {
encoding: "utf-8",
timeout: 2000,
}).trim();
return ppid === "1";
} catch {
return false;
}
return processParentPid(pid) === 1;
}
6 changes: 6 additions & 0 deletions packages/engine/package-subpaths.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,12 @@
"types": "./dist/utils/shaderTransitions.d.ts",
"environments": ["browser", "bun", "node"]
},
"./process-tracker": {
"source": "./src/utils/processTracker.ts",
"runtime": "./dist/utils/processTracker.js",
"types": "./dist/utils/processTracker.d.ts",
"environments": ["bun", "node"]
},
"./package.json": {
"source": "./package.json",
"runtime": "./package.json",
Expand Down
Loading
Loading