diff --git a/packages/cli/src/commands/handlers/default.ts b/packages/cli/src/commands/handlers/default.ts index c23e2be27393..9e3693c00aa7 100644 --- a/packages/cli/src/commands/handlers/default.ts +++ b/packages/cli/src/commands/handlers/default.ts @@ -4,7 +4,7 @@ import { run } from "@opencode/tui" import { Commands } from "../commands" import { Runtime } from "../../framework/runtime" import { Config } from "../../config" -import { Context, Effect, FileSystem, Option, Queue, Schedule, Semaphore } from "effect" +import { Effect, FileSystem, Option, Queue, Schedule, Semaphore } from "effect" import { ServerConnection } from "../../services/server-connection" import { Updater } from "../../services/updater" import { UpdatePreflight } from "../../services/update-preflight" @@ -87,8 +87,6 @@ export default Runtime.handler(Commands, (input) => preflight.loading() const config = yield* Config.Service const npm = yield* Npm.Service - const fileSystem = yield* FileSystem.FileSystem - const runServicePromise = Effect.runPromiseWith(Context.make(FileSystem.FileSystem, fileSystem)) const context = yield* Effect.context() const runFork = Effect.runForkWith(context) const runPromise = Effect.runPromiseWith(context) @@ -106,8 +104,8 @@ export default Runtime.handler(Commands, (input) => local: requestedServer === undefined, service: service ? { - reconnect: (signal) => runServicePromise(service.reconnect(), { signal }), - restart: () => runServicePromise(service.restart()), + reconnect: (signal) => runPromise(service.reconnect(), { signal }), + restart: () => runPromise(service.restart()), } : undefined, }, diff --git a/packages/cli/src/commands/handlers/mini.ts b/packages/cli/src/commands/handlers/mini.ts index 6bcc6f5e8097..4d6d486c62f4 100644 --- a/packages/cli/src/commands/handlers/mini.ts +++ b/packages/cli/src/commands/handlers/mini.ts @@ -1,4 +1,4 @@ -import { Context, Effect, FileSystem, Option } from "effect" +import { Effect, FileSystem, Option } from "effect" import { Commands } from "../commands" import { Runtime } from "../../framework/runtime" import { ServerConnection } from "../../services/server-connection" @@ -16,14 +16,14 @@ export default Runtime.handler(Commands.commands.mini, (input) => const config = yield* Config.Service const global = yield* Global.Service const resolved = resolve(yield* config.get(), { terminalSuspend: process.platform !== "win32" }) - const fileSystem = yield* FileSystem.FileSystem - const runServicePromise = Effect.runPromiseWith(Context.make(FileSystem.FileSystem, fileSystem)) + const context = yield* Effect.context() + const runPromise = Effect.runPromiseWith(context) const service = server.service yield* Effect.promise(() => runMini({ server: { endpoint: server.endpoint, - reconnect: service ? (signal) => runServicePromise(service.reconnect(), { signal }) : undefined, + reconnect: service ? (signal) => runPromise(service.reconnect(), { signal }) : undefined, }, continue: input.continue, session: Option.getOrUndefined(input.session), @@ -36,7 +36,7 @@ export default Runtime.handler(Commands.commands.mini, (input) => demo: input.demo, tuiConfig: resolved, config: { - update: (update) => runServicePromise(config.update(update)), + update: (update) => runPromise(config.update(update)), }, paths: { home: global.home, state: global.state, log: global.log }, }), diff --git a/packages/cli/src/services/server-connection.ts b/packages/cli/src/services/server-connection.ts index 48af79aac52e..ed5976c331fb 100644 --- a/packages/cli/src/services/server-connection.ts +++ b/packages/cli/src/services/server-connection.ts @@ -59,10 +59,30 @@ export const resolve = Effect.fn("cli.server-connection.resolve")(function* (arg if (!endpoint) return { endpoint: yield* Standalone.start(), standalone: true } satisfies Resolved return { endpoint, service: managedService(options) } satisfies Resolved } - return { - endpoint: yield* resolveManaged({ ...options, onStart: args.onStart }, mismatch), - service: managedService(options), - } satisfies Resolved + // Service.ensure waits minutes for a registered service that stopped answering; say so instead of leaving the + // terminal blank. Once it starts a server, onStart reports progress and the update screen may own the terminal. + let started = false + const endpoint = yield* Effect.raceFirst( + resolveManaged( + { + ...options, + onStart: (reason, previousVersion) => { + started = true + args.onStart?.(reason, previousVersion) + }, + }, + mismatch, + ), + Effect.sleep("5 seconds").pipe( + Effect.andThen( + Effect.sync(() => { + if (!started) process.stderr.write("Waiting for the background server to respond...\n") + }), + ), + Effect.andThen(Effect.never), + ), + ) + return { endpoint, service: managedService(options) } satisfies Resolved }) function managedService(options: EnsureOptions) { diff --git a/packages/client/src/effect/service.ts b/packages/client/src/effect/service.ts index 4ec16aa7fcb0..367d0631f1f2 100644 --- a/packages/client/src/effect/service.ts +++ b/packages/client/src/effect/service.ts @@ -1,10 +1,18 @@ -import { Effect, FileSystem, Option, Schedule, Schema } from "effect" +import { Clock, Effect, FileSystem, Option, Schedule, Schema } from "effect" import type { DiscoverOptions, EnsureOptions, StopOptions } from "../service.js" import { contenderPool, spawnServiceContender } from "../service-contender.js" import { defaultEnsureTiming, ensureTiming, type EnsureTiming } from "../service-timing.js" import { matchesVersion } from "../service-version.js" import { PtyHandoff } from "../pty-handoff.js" -import { decide, fallback, headers, probeResult, same } from "../service-probe.js" +import { + decide, + fallback, + headers, + isUninterruptibleDiskSleep, + probeResult, + same, + unresponsiveOwner, +} from "../service-probe.js" export * from "../service.js" export { headers } @@ -49,7 +57,8 @@ export const incumbent = Effect.fn("service.incumbent")(function* ( export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOptions = {}) { const timing = ensureTiming(options) const pool = contenderPool(timing) - let timeouts: { readonly info: Info; readonly count: number } | undefined + const unresponsive = unresponsiveOwner(timing.unresponsiveTimeout) + let deadline = (yield* Clock.currentTimeMillis) + timing.promiseTimeout let announced = false const announce = (reason: "missing" | "version-mismatch", previousVersion?: string) => Effect.sync(() => { @@ -68,25 +77,39 @@ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOpti catch: (cause) => new Error("Failed to start server", { cause }), }) }) + let unanswered = false + const timedOut = () => + pool.failure() ?? + new Error( + unanswered + ? "Timed out waiting for the background service to respond" + : "Timed out waiting for the background service to start", + ) const found = yield* Effect.gen(function* () { + const probedAt = yield* Clock.currentTimeMillis + // An owner still within its unresponsive budget holds the deadline open, so a long budget is not cut short. + if (probedAt >= deadline && !unresponsive.waiting()) return yield* Effect.fail(timedOut()) const registration = yield* registered(options.file, timing.requestTimeout) const info = registration.info const service = registration.service - if (registration.timedOut && info !== undefined) { - timeouts = { - info, - count: timeouts !== undefined && same(timeouts.info, info) ? timeouts.count + 1 : 1, - } - if (timeouts.count >= 3) { - yield* announce("missing") - yield* Effect.logWarning("Background service is unresponsive; recovery cannot preserve persistent terminals") - yield* Effect.tryPromise(() => PtyHandoff.clear(options.file ?? fallback())) - yield* terminate(info, options, timing) - pool.evict(info.pid) - pool.recruitNow() - timeouts = undefined - } - } else timeouts = undefined + // A server this call started can block its event loop while it boots, opening a large database on a slow disk. + // The startup deadline bounds it rather than the unresponsive budget, so that boot is never killed and repeated. + const owner = registration.timedOut && info !== undefined && !pool.spawned(info.pid) ? info : undefined + unanswered = owner !== undefined + const diskSleep = owner !== undefined && (yield* Effect.promise(() => isUninterruptibleDiskSleep(owner.pid))) + const expired = unresponsive.observe(owner, probedAt, yield* Clock.currentTimeMillis, diskSleep) + let evicted = false + if (expired !== undefined) { + yield* announce("missing") + yield* Effect.logWarning("Background service is unresponsive; recovery cannot preserve persistent terminals") + yield* Effect.tryPromise(() => PtyHandoff.clear(options.file ?? fallback())) + yield* terminate(expired, options, timing) + pool.evict(expired.pid) + pool.recruitNow() + unresponsive.forget() + deadline = (yield* Clock.currentTimeMillis) + timing.promiseTimeout + evicted = true + } if (service !== undefined) { pool.serviceAnswered() const decision = decide(service, options) @@ -107,22 +130,19 @@ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOpti const failed = pool.reap() if (failed !== undefined) return yield* Effect.fail(failed) - if (pool.shouldRecruit(info !== undefined)) { + // An owner that times out still holds the port; a contender would only add load until it is evicted. + if ((!registration.timedOut || evicted) && pool.shouldRecruit(info !== undefined)) { yield* announce("missing") pool.add(yield* spawnContender) } return Option.none() }).pipe( - Effect.repeat({ - until: Option.isSome, - // Probes run sequentially, so a slow probe stretches each iteration; bound the loop by wall clock - // like the Promise variant rather than by attempt count. - schedule: Schedule.spaced(timing.pollInterval).pipe(Schedule.upTo({ duration: timing.promiseTimeout })), - }), + // Probes run sequentially, so a slow probe stretches each iteration; the loop is bounded by the wall-clock + // deadline above rather than by attempt count, like the Promise variant. + Effect.repeat({ until: Option.isSome, schedule: Schedule.spaced(timing.pollInterval) }), Effect.ensuring(Effect.sync(() => pool.releaseAll())), ) - if (Option.isNone(found)) - return yield* Effect.fail(pool.failure() ?? new Error("Timed out waiting for the background service to start")) + if (Option.isNone(found)) return yield* Effect.fail(timedOut()) return found.value.endpoint }) diff --git a/packages/client/src/promise/service.ts b/packages/client/src/promise/service.ts index 5d5d0bad4a17..028dfb6cfe60 100644 --- a/packages/client/src/promise/service.ts +++ b/packages/client/src/promise/service.ts @@ -4,7 +4,15 @@ import { contenderPool, spawnServiceContender } from "../service-contender.js" import { defaultEnsureTiming, ensureTiming, type EnsureTiming } from "../service-timing.js" import { matchesVersion } from "../service-version.js" import { PtyHandoff } from "../pty-handoff.js" -import { decide, fallback, headers, probeResult, same } from "../service-probe.js" +import { + decide, + fallback, + headers, + isUninterruptibleDiskSleep, + probeResult, + same, + unresponsiveOwner, +} from "../service-probe.js" export * from "../service.js" export { fallback, headers } @@ -27,9 +35,9 @@ export async function discover(options: DiscoverOptions = {}) { /** Ensure a healthy, compatible local service is running. */ export async function ensure(options: EnsureOptions = {}): Promise { const timing = ensureTiming(options) - const deadline = Date.now() + timing.promiseTimeout + let deadline = Date.now() + timing.promiseTimeout const pool = contenderPool(timing) - let timeouts: { readonly info: Info; readonly count: number } | undefined + const unresponsive = unresponsiveOwner(timing.unresponsiveTimeout) let announced = false const announce = (reason: "missing" | "version-mismatch", previousVersion?: string) => { @@ -48,25 +56,39 @@ export async function ensure(options: EnsureOptions = {}): Promise { } try { + let unanswered = false while (true) { - if (Date.now() >= deadline) - throw pool.failure() ?? new Error("Timed out waiting for the background service to start") + // An owner still within its unresponsive budget holds the deadline open, so a long budget is not cut short. + if (Date.now() >= deadline && !unresponsive.waiting()) + throw ( + pool.failure() ?? + new Error( + unanswered + ? "Timed out waiting for the background service to respond" + : "Timed out waiting for the background service to start", + ) + ) + const probedAt = Date.now() const registration = await registered(options.file, timing.requestTimeout) - if (registration.timedOut && registration.info !== undefined) { - timeouts = { - info: registration.info, - count: timeouts !== undefined && same(timeouts.info, registration.info) ? timeouts.count + 1 : 1, - } - if (timeouts.count >= 3) { - announce("missing") - console.warn("Background service is unresponsive; recovery cannot preserve persistent terminals") - await PtyHandoff.clear(options.file ?? fallback()) - await terminate(registration.info, options, timing) - pool.evict(registration.info.pid) - pool.recruitNow() - timeouts = undefined - } - } else timeouts = undefined + const info = registration.info + // A server this call started can block its event loop while it boots, opening a large database on a slow disk. + // The startup deadline bounds it rather than the unresponsive budget, so that boot is never killed and repeated. + const owner = registration.timedOut && info !== undefined && !pool.spawned(info.pid) ? info : undefined + unanswered = owner !== undefined + const diskSleep = owner !== undefined && (await isUninterruptibleDiskSleep(owner.pid)) + const expired = unresponsive.observe(owner, probedAt, Date.now(), diskSleep) + let evicted = false + if (expired !== undefined) { + announce("missing") + console.warn("Background service is unresponsive; recovery cannot preserve persistent terminals") + await PtyHandoff.clear(options.file ?? fallback()) + await terminate(expired, options, timing) + pool.evict(expired.pid) + pool.recruitNow() + unresponsive.forget() + deadline = Date.now() + timing.promiseTimeout + evicted = true + } if (registration.service !== undefined) { pool.serviceAnswered() @@ -87,7 +109,8 @@ export async function ensure(options: EnsureOptions = {}): Promise { } else { const failed = pool.reap() if (failed !== undefined) throw failed - if (pool.shouldRecruit(registration.info !== undefined)) { + // An owner that times out still holds the port; a contender would only add load until it is evicted. + if ((!registration.timedOut || evicted) && pool.shouldRecruit(info !== undefined)) { announce("missing") pool.add(await spawnContender()) } diff --git a/packages/client/src/service-contender.ts b/packages/client/src/service-contender.ts index 79542c30a44b..7994ecc0d9e4 100644 --- a/packages/client/src/service-contender.ts +++ b/packages/client/src/service-contender.ts @@ -84,6 +84,10 @@ export function contenderPool(timing: EnsureTiming) { failure = undefined lastSpawn = 0 }, + /** Whether this call started the process `pid`. */ + spawned(pid: number) { + return [...contenders].some((item) => item.child.pid === pid) + }, /** Let the next attempt start without waiting a spawn delay. */ recruitNow() { lastSpawn = Date.now() - spawnDelay diff --git a/packages/client/src/service-probe.ts b/packages/client/src/service-probe.ts index 7f61c84f8e0c..6c709927cd08 100644 --- a/packages/client/src/service-probe.ts +++ b/packages/client/src/service-probe.ts @@ -1,3 +1,4 @@ +import { readFile } from "node:fs/promises" import { homedir } from "node:os" import { join } from "node:path" import type { DiscoverOptions, Endpoint, Info } from "./service.js" @@ -103,3 +104,77 @@ function decodeInfo(input: unknown) { if (!("pid" in input) || typeof input.pid !== "number" || !Number.isInteger(input.pid) || input.pid < 0) return return { version: input.version, pid: input.pid } } + +/** + * Parse the single-character task state from Linux `/proc//stat`. + * `comm` is enclosed in parentheses and may itself contain spaces or `)`, so scan from the final `)`. + */ +export function parseLinuxProcStatState(stat: string) { + const close = stat.lastIndexOf(")") + if (close === -1) return undefined + const state = stat[close + 2] + return state && state !== " " ? state : undefined +} + +/** + * Tracks how long one registered owner has accepted connections without answering its health probe. + * An owner that answers, or a different owner, starts the clock over. Time the owner spends in disk sleep + * does not count: it is waiting for the disk, not hung, and a check only at the end of the budget would + * evict it the moment the disk catches up. + */ +export function unresponsiveOwner(budget: number) { + let current: + | { + readonly info: Info + readonly since: number + readonly last: number + readonly counted: number + readonly past: boolean + } + | undefined + return { + /** + * Records a probe that began at `probedAt`, timed out for `info` (or answered, when `info` is undefined), and + * whether the owner was then in disk sleep. Returns the owner once its counted time reaches the budget. + */ + observe(info: Info | undefined, probedAt: number, now: number, diskSleep: boolean) { + if (info === undefined) { + current = undefined + return undefined + } + const previous = + current !== undefined && same(current.info, info) + ? current + : { info, since: probedAt, last: probedAt, counted: 0, past: false } + const counted = previous.counted + (diskSleep ? 0 : now - previous.last) + current = { + info, + since: previous.since, + last: now, + counted, + past: counted >= budget || now - previous.since >= budget, + } + return counted >= budget ? info : undefined + }, + /** + * Whether no probe has yet seen the owner past its budget, so the startup deadline waits for the probe that may + * evict it. Wall time also ends the wait, so an owner that stays in disk sleep cannot hold the deadline forever. + */ + waiting() { + return current !== undefined && !current.past + }, + forget() { + current = undefined + }, + } +} + +/** + * Whether `pid` is currently in Linux uninterruptible disk sleep (`state D`). Always false elsewhere. + * It is one sample per probe, so the time it excludes from an unresponsive budget is an estimate. + */ +export async function isUninterruptibleDiskSleep(pid: number) { + if (process.platform !== "linux") return false + const stat = await readFile(`/proc/${pid}/stat`, "utf8").catch(() => undefined) + return stat !== undefined && parseLinuxProcStatState(stat) === "D" +} diff --git a/packages/client/src/service-timing.ts b/packages/client/src/service-timing.ts index 03db9f6cb2ce..af9383bd05fd 100644 --- a/packages/client/src/service-timing.ts +++ b/packages/client/src/service-timing.ts @@ -1,6 +1,7 @@ export type EnsureTiming = { readonly pollInterval: number readonly requestTimeout: number + readonly unresponsiveTimeout: number readonly spawnDelay: number readonly maxSpawnDelay: number readonly promiseTimeout: number @@ -16,6 +17,9 @@ const timings = new WeakMap() export const defaultEnsureTiming: EnsureTiming = { pollInterval: 25, requestTimeout: 2_000, + // Every client shares one service, and replacing it drops all of their sessions and terminals, so every caller + // waits the same: a per-caller budget would let the least patient one decide for all of them. + unresponsiveTimeout: 3 * 60_000, spawnDelay: 5_000, maxSpawnDelay: 30_000, promiseTimeout: 120_000, diff --git a/packages/client/src/solid/connection.ts b/packages/client/src/solid/connection.ts index 01e6b75b28c1..9fd17f2f3e5c 100644 --- a/packages/client/src/solid/connection.ts +++ b/packages/client/src/solid/connection.ts @@ -31,6 +31,7 @@ export type ClientConnectionOptions = { const connectTimeout = 2_000 const reconnectDelay = 1_000 +const maxReconnectDelay = 15_000 const connectionHistoryLimit = 50 export const defaultIdleTimeout = 45_000 // Longer than one server keepalive interval: a stream that is silent this long when the page @@ -178,7 +179,12 @@ export function createClientConnection(initialApi: OpenCodeClient, options: Clie forced = false continue } - await wait(reconnectDelay, controller.signal) + // Back off only when reconnecting can start a server: a plain stream retry is one cheap request, and a remote + // server that comes back should be picked up within a second. + const delay = options.reconnect + ? Math.min(reconnectDelay * 2 ** (attempt - 1), maxReconnectDelay) + : reconnectDelay + await wait(delay, controller.signal) } } diff --git a/packages/client/test/fixture/service-fixture.ts b/packages/client/test/fixture/service-fixture.ts index 618fd61493d8..74cf960bc856 100644 --- a/packages/client/test/fixture/service-fixture.ts +++ b/packages/client/test/fixture/service-fixture.ts @@ -77,6 +77,23 @@ export async function serviceFixture() { .text() .catch(() => undefined) }, + // A hanging service records each health probe it leaves unanswered. + async waitForRequests(count: number) { + const deadline = Date.now() + 3_000 + while (true) { + const seen = ( + await Bun.file(registration + ".requests") + .text() + .catch(() => "") + ) + .trim() + .split("\n") + .filter(Boolean).length + if (seen >= count) return + if (Date.now() >= deadline) throw new Error(`Expected ${count} unanswered health probes, saw ${seen}`) + await Bun.sleep(10) + } + }, async waitForFile(file = registration) { for (let attempt = 0; attempt < 600; attempt++) { if (await Bun.file(file).exists()) return diff --git a/packages/client/test/fixture/service-timing.ts b/packages/client/test/fixture/service-timing.ts index 8c9ff85e15a4..654148e0f1a8 100644 --- a/packages/client/test/fixture/service-timing.ts +++ b/packages/client/test/fixture/service-timing.ts @@ -1,16 +1,17 @@ -import { withEnsureTiming } from "../../src/service-timing" +import { withEnsureTiming, type EnsureTiming } from "../../src/service-timing" const timing = { pollInterval: 20, requestTimeout: 100, + unresponsiveTimeout: 320, spawnDelay: 200, maxSpawnDelay: 1_200, promiseTimeout: 3_000, stopPollInterval: 5, } -export function accelerate(ensure: (options?: A) => B) { - return (options: A) => ensure(withEnsureTiming(options, timing)) +export function accelerate(ensure: (options?: A) => B, overrides: Partial = {}) { + return (options: A) => ensure(withEnsureTiming(options, { ...timing, ...overrides })) } export async function waitForExit(pid: number) { diff --git a/packages/client/test/fixture/service.ts b/packages/client/test/fixture/service.ts index 77cabf01410e..36e2839dc41d 100644 --- a/packages/client/test/fixture/service.ts +++ b/packages/client/test/fixture/service.ts @@ -75,7 +75,7 @@ const server = Bun.serve({ return new Response(null, { status: 404 }) requests += 1 if (mode === "starting") await writeFile(registration + ".status-request", "") - if (mode === "hanging" || controlled === "hang") { + if ((mode === "hanging" || controlled === "hang") && !(await Bun.file(registration + ".release").exists())) { await appendFile(registration + ".requests", process.pid + "\n") return new Promise(() => {}) } diff --git a/packages/client/test/promise-service.test.ts b/packages/client/test/promise-service.test.ts index c0cdb746736c..3b76c3b64eab 100644 --- a/packages/client/test/promise-service.test.ts +++ b/packages/client/test/promise-service.test.ts @@ -172,20 +172,27 @@ test("accepts a surviving contender after a failure without recruiting replaceme expect(await fixture.starts()).toHaveLength(2) }) -test("recovers when an unresponsive contender is evicted after a prior failure", async () => { +test("keeps a registered contender that stops answering after its sibling fails", async () => { await using fixture = await serviceFixture() - const pending = ensure({ file: fixture.registration, version: "test", command: fixture.command("controlled") }) + // A longer deadline leaves room for the steps before the survivor answers again. + const pending = accelerate(Service.ensure, { promiseTimeout: 4_000 })({ + file: fixture.registration, + version: "test", + command: fixture.command("controlled"), + }) const [first, second] = await fixture.waitForStarts(2) await fixture.release(second, "hang") await fixture.waitForFile() await fixture.release(first, "fail") - const [, , third] = await fixture.waitForStarts(3) - await fixture.release(third, "ready") + // Unanswered for longer than the 320 ms unresponsive budget, which would evict a service it did not start. + await fixture.waitForRequests(5) + expect(() => process.kill(second, 0)).not.toThrow() + await Bun.write(fixture.registration + ".release", "") const endpoint = await pending - fixture.track(third) expect((await Service.discover({ file: fixture.registration, version: "test" }))?.url).toBe(endpoint.url) - expect((await Bun.file(fixture.registration).json()).pid).toBe(third) + expect((await Bun.file(fixture.registration).json()).pid).toBe(second) + expect(await fixture.starts()).toHaveLength(2) }) test("reports a bounded contender stderr tail with native promises", async () => { @@ -218,7 +225,7 @@ test("evicts an unresponsive registered service before starting its replacement" const replacement = await Bun.file(registration).json() fixture.track(replacement.pid) - expect((await Bun.file(registration + ".requests").text()).trim().split("\n")).toHaveLength(3) + expect((await Bun.file(registration + ".requests").text()).trim().split("\n").length).toBeGreaterThanOrEqual(2) expect(await existing.exited).toBe(stoppedExitCode) expect(replacement.pid).not.toBe(original.pid) expect(endpoint.url).toBe(replacement.url) diff --git a/packages/client/test/service-decision.test.ts b/packages/client/test/service-decision.test.ts index b0d61dbf04f9..07adbcac2dbd 100644 --- a/packages/client/test/service-decision.test.ts +++ b/packages/client/test/service-decision.test.ts @@ -1,5 +1,5 @@ import { expect, test } from "bun:test" -import { decide, type LocalService } from "../src/service-probe" +import { decide, parseLinuxProcStatState, unresponsiveOwner, type LocalService } from "../src/service-probe" const protocolError = "Background service uses an incompatible health protocol. Update this client or explicitly restart the service." @@ -52,3 +52,35 @@ test("a version mismatch replaces the service, handing off terminals only when i pty: "clear", }) }) + +test("parseLinuxProcStatState reads the task state after any parenthesized command name", () => { + expect(parseLinuxProcStatState("3698946 (opencode) D 1 3698946 3698946 0 -1")).toBe("D") + expect(parseLinuxProcStatState("42 (opencode (worker) 1) D 1 42 42 0 -1")).toBe("D") + expect(parseLinuxProcStatState("42 (opencode (worker) 1) S 1 42 42 0 -1")).toBe("S") + expect(parseLinuxProcStatState("42 (opencode) Z 1 42 42 0 -1")).toBe("Z") + expect(parseLinuxProcStatState("42 (opencode)")).toBeUndefined() + expect(parseLinuxProcStatState("malformed")).toBeUndefined() +}) + +const owner = { url: "http://127.0.0.1:4096", pid: 4242, version: "2.0.0" } + +test("disk sleep pauses the unresponsive budget, so an owner is not evicted the moment the disk catches up", () => { + const unresponsive = unresponsiveOwner(3_000) + // In disk sleep for 15s: none of it counts. + for (let at = 0; at < 15_000; at += 500) expect(unresponsive.observe(owner, at, at + 500, true)).toBeUndefined() + // The probe in flight when the disk caught up times out with the process running again. + expect(unresponsive.observe(owner, 15_000, 15_500, false)).toBeUndefined() + // Running but silent for the whole budget is a hang. + expect(unresponsive.observe(owner, 15_500, 18_500, false)).toBe(owner) +}) + +test("an unresponsive owner holds the startup deadline until a probe sees its budget pass, even in disk sleep", () => { + const unresponsive = unresponsiveOwner(3_000) + expect(unresponsive.waiting()).toBe(false) + unresponsive.observe(owner, 0, 2_000, true) + expect(unresponsive.waiting()).toBe(true) + unresponsive.observe(owner, 2_000, 4_000, true) + expect(unresponsive.waiting()).toBe(false) + unresponsive.observe(undefined, 4_000, 4_100, false) + expect(unresponsive.waiting()).toBe(false) +}) diff --git a/packages/client/test/service-reconnect.test.ts b/packages/client/test/service-reconnect.test.ts index ff2a0d9598b7..52fd6f75e8bc 100644 --- a/packages/client/test/service-reconnect.test.ts +++ b/packages/client/test/service-reconnect.test.ts @@ -4,15 +4,20 @@ import { Effect } from "effect" import { Service } from "../src/effect/service" import { ensure, stop } from "../src/promise/service" import type { EnsureOptions, EnsureReason, StopOptions } from "../src/service" +import type { EnsureTiming } from "../src/service-timing" import { serviceFixture, stopSignal, stoppedExitCode } from "./fixture/service-fixture" import { accelerate } from "./fixture/service-timing" -const effect = accelerate(Service.ensure) const clients = [ - { name: "Promise", ensure: accelerate(ensure), stop }, + { + name: "Promise", + ensure: (options: EnsureOptions, timing?: Partial) => accelerate(ensure, timing)(options), + stop, + }, { name: "Effect", - ensure: (options: EnsureOptions) => Effect.runPromise(effect(options).pipe(Effect.provide(NodeFileSystem.layer))), + ensure: (options: EnsureOptions, timing?: Partial) => + Effect.runPromise(accelerate(Service.ensure, timing)(options).pipe(Effect.provide(NodeFileSystem.layer))), stop: (options: StopOptions) => Effect.runPromise(Service.stop(options).pipe(Effect.provide(NodeFileSystem.layer))), }, ] @@ -91,4 +96,70 @@ for (const client of clients) { expect(await fixture.signal()).toBe(stopSignal) expect(await Bun.file(fixture.registration).exists()).toBe(false) }) + + test(`${client.name} keeps a slow owner within its unresponsive budget without spawning contenders`, async () => { + await using fixture = await serviceFixture() + const owner = fixture.spawn("hanging") + await fixture.waitForFile() + const original = await Bun.file(fixture.registration).json() + const starts: EnsureReason[] = [] + + const pending = client.ensure( + { + file: fixture.registration, + command: fixture.command("record-start"), + onStart: (reason) => starts.push(reason), + }, + { unresponsiveTimeout: 60_000 }, + ) + await fixture.waitForRequests(5) + + expect(owner.exitCode).toBe(null) + expect(await Bun.file(fixture.registration + ".signal").exists()).toBe(false) + expect(await Bun.file(fixture.registration + ".started").exists()).toBe(false) + expect(starts).toEqual([]) + + await Bun.write(fixture.registration + ".release", "") + const endpoint = await pending + expect(endpoint.url).toBe(original.url) + expect(owner.exitCode).toBe(null) + }) + + test(`${client.name} evicts a hung owner once a budget longer than the startup deadline runs out`, async () => { + await using fixture = await serviceFixture() + const owner = fixture.spawn("hanging") + await fixture.waitForFile() + const original = await Bun.file(fixture.registration).json() + const started = Date.now() + const endpoint = await client.ensure( + { file: fixture.registration, command: fixture.command("delayed", "10") }, + { requestTimeout: 80, unresponsiveTimeout: 700, spawnDelay: 100, promiseTimeout: 250 }, + ) + const replacement = await Bun.file(fixture.registration).json() + fixture.track(replacement.pid) + + expect(Date.now() - started).toBeGreaterThanOrEqual(700) + expect(await owner.exited).toBe(stoppedExitCode) + expect(replacement.pid).not.toBe(original.pid) + expect(endpoint.url).toBe(replacement.url) + }) + + test(`${client.name} keeps a server it started that stops answering until the startup deadline`, async () => { + await using fixture = await serviceFixture() + const pending = client + .ensure({ file: fixture.registration, command: fixture.command("controlled") }) + .catch((error: unknown) => error) + const [started] = await fixture.waitForStarts(1) + await fixture.release(started, "hang") + // Unanswered for longer than the 320 ms unresponsive budget, which would evict a service it did not start. + await fixture.waitForRequests(5) + expect(() => process.kill(started, 0)).not.toThrow() + const error = await pending + + expect(error).toBeInstanceOf(Error) + if (!(error instanceof Error)) throw error + expect(error.message).toBe("Timed out waiting for the background service to start") + expect((await Bun.file(fixture.registration).json()).pid).toBe(started) + expect(() => process.kill(started, 0)).not.toThrow() + }) } diff --git a/packages/client/test/service.test.ts b/packages/client/test/service.test.ts index 81101f273ab4..7dd6d94e656a 100644 --- a/packages/client/test/service.test.ts +++ b/packages/client/test/service.test.ts @@ -192,7 +192,7 @@ test("evicts an unresponsive registered service before starting its replacement" const replacement = await Bun.file(registration).json() fixture.track(replacement.pid) - expect((await Bun.file(registration + ".requests").text()).trim().split("\n")).toHaveLength(3) + expect((await Bun.file(registration + ".requests").text()).trim().split("\n").length).toBeGreaterThanOrEqual(2) expect(await existing.exited).toBe(stoppedExitCode) expect(replacement.pid).not.toBe(original.pid) expect(endpoint.url).toBe(replacement.url) @@ -398,20 +398,29 @@ test("accepts a surviving contender after a failure without recruiting replaceme expect(await fixture.starts()).toHaveLength(2) }) -test("recovers when an unresponsive contender is evicted after a prior failure", async () => { +test("keeps a registered contender that stops answering after its sibling fails", async () => { await using fixture = await serviceFixture() - const pending = run(ensure({ file: fixture.registration, version: "test", command: fixture.command("controlled") })) + // A longer deadline leaves room for the steps before the survivor answers again. + const pending = run( + accelerate(Service.ensure, { promiseTimeout: 4_000 })({ + file: fixture.registration, + version: "test", + command: fixture.command("controlled"), + }), + ) const [first, second] = await fixture.waitForStarts(2) await fixture.release(second, "hang") await fixture.waitForFile() await fixture.release(first, "fail") - const [, , third] = await fixture.waitForStarts(3) - await fixture.release(third, "ready") + // Unanswered for longer than the 320 ms unresponsive budget, which would evict a service it did not start. + await fixture.waitForRequests(5) + expect(() => process.kill(second, 0)).not.toThrow() + await Bun.write(fixture.registration + ".release", "") const endpoint = await pending - fixture.track(third) expect((await run(Service.discover({ file: fixture.registration, version: "test" })))?.url).toBe(endpoint.url) - expect((await Bun.file(fixture.registration).json()).pid).toBe(third) + expect((await Bun.file(fixture.registration).json()).pid).toBe(second) + expect(await fixture.starts()).toHaveLength(2) }) test("reports a bounded contender stderr tail", async () => { diff --git a/packages/client/test/solid-connection.test.ts b/packages/client/test/solid-connection.test.ts index 60219ed73ea3..7da09217e1ad 100644 --- a/packages/client/test/solid-connection.test.ts +++ b/packages/client/test/solid-connection.test.ts @@ -1,4 +1,4 @@ -import { expect, test } from "bun:test" +import { expect, jest, test } from "bun:test" import { createRoot } from "solid-js" import { createClientConnection } from "../src/solid" import { OpenCode, type OpenCodeEvent } from "../src/promise" @@ -179,3 +179,68 @@ test("a reconnect does not ask the volatile event stream to replay", async () => ctx.dispose() } }) + +test.each([ + { + name: "a failing server resolution backs off exponentially up to 15s", + reconnect: async (): Promise => { + throw new Error("unavailable") + }, + gaps: [1_000, 2_000, 4_000, 8_000, 15_000, 15_000], + }, + { + name: "a plain stream retry keeps a 1s delay", + reconnect: undefined, + gaps: [1_000, 1_000, 1_000, 1_000, 1_000, 1_000], + }, +])("$name", async (input) => { + jest.useFakeTimers() + const attempts: number[] = [] + let disconnects = 0 + const api = OpenCode.make({ + baseUrl: "http://opencode.local", + fetch: async () => { + attempts.push(Date.now()) + return new Response("", { headers: { "content-type": "text/event-stream" } }) + }, + }) + // The connect timeout is cleared before a disconnect is reported, so the next pending timer is the retry delay. + // Spinning on microtasks starves the test timeout, so give up after a bound instead of hanging the run. + const retryScheduled = async (count: number) => { + for (let spin = 0; spin < 10_000; spin++) { + if (disconnects >= count && jest.getTimerCount() > 0) return + await Promise.resolve() + } + throw new Error(`No retry was scheduled after disconnect ${count}`) + } + + const start = Date.now() + let dispose!: () => void + createRoot((cleanup) => { + dispose = cleanup + createClientConnection(api, { + reconnect: input.reconnect, + flushInterval: 0, + onEvent: () => {}, + log: { + info(message) { + if (message === "event stream disconnected") disconnects += 1 + }, + }, + }) + }) + + try { + for (let step = 1; step <= input.gaps.length; step++) { + await retryScheduled(step) + jest.advanceTimersToNextTimer() + } + await retryScheduled(input.gaps.length + 1) + + expect(attempts[0]).toBe(start) + expect(attempts.slice(1).map((time, index) => time - attempts[index])).toEqual(input.gaps) + } finally { + dispose() + jest.useRealTimers() + } +}) diff --git a/packages/desktop/src/main/service/cli-version.test.ts b/packages/desktop/src/main/service/cli-version.test.ts index eb10c19547e0..217bc015d969 100644 --- a/packages/desktop/src/main/service/cli-version.test.ts +++ b/packages/desktop/src/main/service/cli-version.test.ts @@ -21,6 +21,7 @@ describe("cli-version", () => { test.each([ { version: undefined, message: "Background service failed to start", expected: true }, { version: undefined, message: "Timed out waiting for the background service to start", expected: true }, + { version: undefined, message: "Timed out waiting for the background service to respond", expected: false }, { version: undefined, message: "Background service uses an incompatible health protocol. Update this client or explicitly restart the service.",