fix(desktop): stop stale dev runners from crash-looping Electron

.config/mise/tasks/dev/desktop/reset

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+#MISE description="Stop this checkout's desktop dev runner and delete its Electron bundle so the next start rebuilds it"
+#MISE depends=["dev:desktop:stop"]
+set -euo pipefail
+cd "$MISE_PROJECT_ROOT"
+rm -rf apps/desktop/.electron-runtime
+echo "[desktop-dev] removed apps/desktop/.electron-runtime"

.config/mise/tasks/dev/desktop/stop

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+#MISE description="Stop this checkout's desktop dev runner"
+set -euo pipefail
+cd "$MISE_PROJECT_ROOT"
+node apps/desktop/scripts/stop-dev-electron.mjs

apps/desktop/package.json

@@ -7,6 +7,7 @@
   "scripts": {
     "ensure:electron": "node scripts/ensure-electron-runtime.mjs",
     "start": "node scripts/start-electron.mjs",
+    "dev:stop": "node scripts/stop-dev-electron.mjs",
     "typecheck": "tsc-rs --noEmit",
     "test": "vp test run --passWithNoTests",
     "smoke-test": "node scripts/smoke-test.mjs"

apps/desktop/scripts/dev-electron.mjs

@@ -8,6 +8,8 @@ import {
   resolveDevProtocolClient,
   resolveElectronLaunchCommand,
 } from "./electron-launcher.mjs";
+import { acquireRunnerRecord, releaseRunnerRecord } from "./dev-runner-control.mjs";
+import { createCrashLoopBreaker } from "./dev-runner-guards.mjs";
 import { waitForResources } from "./wait-for-resources.mjs";
 
 const devServerUrl = process.env.VITE_DEV_SERVER_URL?.trim();
@@ -49,10 +51,31 @@ const watchedDirectories = [
 const forcedShutdownTimeoutMs = 1_500;
 const restartDebounceMs = 120;
 const childTreeGracePeriodMs = 1_200;
+const orphanCheckIntervalMs = 1_000;
 const remoteDebuggingPort = process.env.T3CODE_DESKTOP_REMOTE_DEBUGGING_PORT?.trim();
 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone dev script has no Effect runtime.
 const hostPlatform = NodeOS.platform();
 
+const initialParentPid = process.ppid;
+
+// Two runners in one checkout rebuild the same launcher bundle concurrently and kill each
+// other's Electron processes, so a new runner replaces the previous one.
+const claim = await acquireRunnerRecord(desktopDir);
+if (!claim.claimed) {
+  const reason =
+    claim.blocker.action === "survived"
+      ? "did not exit after SIGKILL"
+      : "may still own this checkout but its working directory could not be verified";
+  console.error(
+    `[desktop-dev] runner ${claim.blocker.pid} ${reason}. ` +
+      "Stop it, or delete .electron-runtime/runner.json, then retry.",
+  );
+  process.exit(1);
+}
+for (const stoppedPid of claim.stoppedPids) {
+  console.log(`[desktop-dev] stopped previous runner ${stoppedPid} for this checkout`);
+}
+
 NodeChildProcess.execFileSync(
   process.execPath,
   [NodePath.join(desktopDir, "scripts/build-browser-secret.mjs")],
@@ -66,6 +89,11 @@ await waitForResources({
   tcpPort: port,
 });
 
+if (process.ppid !== initialParentPid) {
+  releaseRunnerRecord(desktopDir);
+  process.exit(0);
+}
+
 const childEnv = { ...process.env };
 delete childEnv.ELECTRON_RUN_AS_NODE;
 const devProtocolClient = resolveDevProtocolClient();
@@ -80,6 +108,8 @@ let currentApp = null;
 let restartQueue = Promise.resolve();
 const expectedExits = new WeakSet();
 const watchers = [];
+const failedApps = new WeakSet();
+const crashLoopBreaker = createCrashLoopBreaker();
 
 function killChildTreeByPid(pid, signal) {
   if (hostPlatform === "win32" || typeof pid !== "number") {
@@ -129,23 +159,38 @@ function startApp() {
       currentApp = null;
     }
 
-    if (!shuttingDown) {
-      scheduleRestart();
-    }
+    restartAfterFailure(app);
   });
 
   app.once("exit", (code, signal) => {
     if (currentApp === app) {
       currentApp = null;
     }
 
-    const exitedAbnormally = signal !== null || code !== 0;
-    if (!shuttingDown && !expectedExits.has(app) && exitedAbnormally) {
-      scheduleRestart();
+    if (signal !== null || code !== 0) {
+      restartAfterFailure(app);
     }
   });
 }
 
+// A spawn error can be followed by an exit event for the same child, so each child counts once.
+function restartAfterFailure(app) {
+  if (shuttingDown || expectedExits.has(app) || failedApps.has(app)) {
+    return;
+  }
+  failedApps.add(app);
+
+  if (crashLoopBreaker.recordAbnormalExit(Date.now()).tripped) {
+    console.error(
+      "[desktop-dev] Electron keeps crashing right after launch, so restarts are paused. " +
+        "Fix the problem and rebuild to try again, or run `mise run dev:desktop:reset` " +
+        "to rebuild the Electron bundle.",
+    );
+    return;
+  }
+  scheduleRestart();
+}
+
 async function stopApp() {
   const app = currentApp;
   if (!app) {
@@ -217,6 +262,7 @@ function startWatchers() {
           return;
         }
 
+        crashLoopBreaker.rearm();
         scheduleRestart();
       },
     );
@@ -248,16 +294,26 @@ async function shutdown(exitCode) {
     watcher.close();
   }
 
+  clearInterval(orphanCheck);
   await stopApp();
   killChildTree("TERM");
   await new Promise((resolve) => {
     setTimeout(resolve, childTreeGracePeriodMs);
   });
   killChildTree("KILL");
+  // Released only after teardown, because the teardown kills every Electron tagged with this
+  // checkout and would take down a replacement runner's app.
+  releaseRunnerRecord(desktopDir);
 
   process.exit(exitCode);
 }
 
+const orphanCheck = setInterval(() => {
+  if (process.ppid !== initialParentPid) {
+    void shutdown(0);
+  }
+}, orphanCheckIntervalMs);
+
 startWatchers();
 cleanupStaleDevApps();
 startApp();

apps/desktop/scripts/dev-runner-control.mjs

@@ -0,0 +1,174 @@
+import * as NodeChildProcess from "node:child_process";
+import * as NodeFS from "node:fs";
+import * as NodeOS from "node:os";
+import * as NodePath from "node:path";
+
+import { decideTakeover, parseRunnerRecord } from "./dev-runner-guards.mjs";
+
+const stopGracePeriodMs = 4_000;
+const stopPollIntervalMs = 100;
+const maxClaimAttempts = 5;
+
+// oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone dev script has no Effect runtime.
+const hostPlatform = NodeOS.platform();
+
+function runnerRecordPath(desktopDir) {
+  return NodePath.join(desktopDir, ".electron-runtime", "runner.json");
+}
+
+function readRunnerRecordText(desktopDir) {
+  try {
+    return NodeFS.readFileSync(runnerRecordPath(desktopDir), "utf8");
+  } catch {
+    return null;
+  }
+}
+
+function readRunnerRecord(desktopDir) {
+  const text = readRunnerRecordText(desktopDir);
+  return text === null ? null : parseRunnerRecord(text);
+}
+
+function readProcessCwd(pid) {
+  if (hostPlatform === "linux") {
+    try {
+      return NodeFS.readlinkSync(`/proc/${pid}/cwd`);
+    } catch {
+      return null;
+    }
+  }
+  const result = NodeChildProcess.spawnSync("lsof", ["-a", "-p", String(pid), "-d", "cwd", "-Fn"], {
+    encoding: "utf8",
+  });
+  const line = result.stdout?.split("\n").find((entry) => entry.startsWith("n"));
+  return line ? line.slice(1) : null;
+}
+
+function inspectProcess(pid) {
+  const result = NodeChildProcess.spawnSync("ps", ["-p", String(pid), "-o", "command="], {
+    encoding: "utf8",
+  });
+  const command = result.stdout?.trim();
+  if (result.status !== 0 || !command) {
+    return null;
+  }
+  return { command, cwd: readProcessCwd(pid) };
+}
+
+function isAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    return error?.code === "EPERM";
+  }
+}
+
+async function waitForExit(pid, timeoutMs) {
+  for (let waited = 0; waited < timeoutMs; waited += stopPollIntervalMs) {
+    if (!isAlive(pid)) {
+      return true;
+    }
+    await new Promise((resolve) => setTimeout(resolve, stopPollIntervalMs));
+  }
+  return !isAlive(pid);
+}
+
+function removeRunnerRecordIfUnchanged(desktopDir, text) {
+  if (readRunnerRecordText(desktopDir) === text) {
+    NodeFS.rmSync(runnerRecordPath(desktopDir), { force: true });
+  }
+}
+
+// The runner can exit on its own between inspection and each signal.
+function signalIfAlive(pid, signal) {
+  try {
+    process.kill(pid, signal);
+  } catch (error) {
+    if (error?.code !== "ESRCH") {
+      throw error;
+    }
+  }
+}
+
+async function terminate(pid) {
+  signalIfAlive(pid, "SIGTERM");
+  if (!(await waitForExit(pid, stopGracePeriodMs))) {
+    signalIfAlive(pid, "SIGKILL");
+    return waitForExit(pid, stopGracePeriodMs);
+  }
+  return true;
+}
+
+async function clearRecordedRunner(desktopDir, selfPid) {
+  const text = readRunnerRecordText(desktopDir);
+  if (text === null) {
+    return { action: "none" };
+  }
+  const record = parseRunnerRecord(text);
+  const decision = decideTakeover({
+    record,
+    selfPid,
+    processInfo: record === null ? null : inspectProcess(record.pid),
+    desktopDir,
+  });
+  if (decision.action === "unverifiable") {
+    return decision;
+  }
+  if (decision.action === "stop" && !(await terminate(decision.pid))) {
+    return { action: "survived", pid: decision.pid };
+  }
+  removeRunnerRecordIfUnchanged(desktopDir, text);
+  return decision;
+}
+
+export async function stopRecordedRunner(desktopDir) {
+  if (hostPlatform === "win32") {
+    return { action: "unsupported" };
+  }
+  return clearRecordedRunner(desktopDir, 0);
+}
+
+// The record is published with link() so it appears atomically with its full contents and two
+// runners starting together cannot both believe they own the checkout.
+export async function acquireRunnerRecord(desktopDir, pid = process.pid) {
+  const recordPath = runnerRecordPath(desktopDir);
+  const contents = `${JSON.stringify({ pid })}\n`;
+  NodeFS.mkdirSync(NodePath.dirname(recordPath), { recursive: true });
+  if (hostPlatform === "win32") {
+    NodeFS.writeFileSync(recordPath, contents);
+    return { claimed: true, stoppedPids: [] };
+  }
+
+  const stagingPath = `${recordPath}.${pid}.tmp`;
+  NodeFS.writeFileSync(stagingPath, contents);
+  const stoppedPids = [];
+  try {
+    for (let attempt = 0; attempt < maxClaimAttempts; attempt += 1) {
+      try {
+        NodeFS.linkSync(stagingPath, recordPath);
+        return { claimed: true, stoppedPids };
+      } catch (error) {
+        if (error?.code !== "EEXIST") {
+          throw error;
+        }
+      }
+      const decision = await clearRecordedRunner(desktopDir, pid);
+      if (decision.action === "unverifiable" || decision.action === "survived") {
+        return { claimed: false, blocker: decision };
+      }
+      if (decision.action === "stop") {
+        stoppedPids.push(decision.pid);
+      }
+    }
+    throw new Error(`another runner keeps claiming ${recordPath}`);
+  } finally {
+    NodeFS.rmSync(stagingPath, { force: true });
+  }
+}
+
+export function releaseRunnerRecord(desktopDir, pid = process.pid) {
+  if (readRunnerRecord(desktopDir)?.pid === pid) {
+    NodeFS.rmSync(runnerRecordPath(desktopDir), { force: true });
+  }
+}

apps/desktop/scripts/dev-runner-guards.mjs

@@ -0,0 +1,57 @@
+const RUNNER_SCRIPT_NAME = "dev-electron.mjs";
+
+export function parseRunnerRecord(text) {
+  let parsed;
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    return null;
+  }
+  const pid = parsed?.pid;
+  return Number.isInteger(pid) && pid > 1 ? { pid } : null;
+}
+
+function isRunnerCommand(command) {
+  const [executable = "", ...args] = command.trim().split(/\s+/);
+  const script = args.find((arg) => !arg.startsWith("-"));
+  return (
+    /^node(?:\.exe)?$/.test(executable.split(/[\\/]/).at(-1)) &&
+    script !== undefined &&
+    script.split(/[\\/]/).at(-1) === RUNNER_SCRIPT_NAME
+  );
+}
+
+export function isVerifiedRunner(processInfo, desktopDir) {
+  return (
+    processInfo !== null && isRunnerCommand(processInfo.command) && processInfo.cwd === desktopDir
+  );
+}
+
+// `processInfo` is null when the pid is not alive. Only a verified runner is signaled: a pid
+// running something else was reused, so its record is stale, while a runner whose working
+// directory cannot be read may belong to this checkout and must be neither signaled nor replaced.
+export function decideTakeover({ record, selfPid, processInfo, desktopDir }) {
+  if (record === null || record.pid === selfPid || processInfo === null) {
+    return { action: "none" };
+  }
+  if (isVerifiedRunner(processInfo, desktopDir)) {
+    return { action: "stop", pid: record.pid };
+  }
+  if (isRunnerCommand(processInfo.command) && processInfo.cwd === null) {
+    return { action: "unverifiable", pid: record.pid };
+  }
+  return { action: "stale", pid: record.pid };
+}
+
+export function createCrashLoopBreaker({ maxAbnormalExits = 3, windowMs = 10_000 } = {}) {
+  let exitTimes = [];
+  return {
+    recordAbnormalExit(nowMs) {
+      exitTimes = [...exitTimes.filter((time) => nowMs - time < windowMs), nowMs];
+      return { tripped: exitTimes.length >= maxAbnormalExits };
+    },
+    rearm() {
+      exitTimes = [];
+    },
+  };
+}