diff --git a/apps/desktop/src/settings/DesktopClientSettings.test.ts b/apps/desktop/src/settings/DesktopClientSettings.test.ts index 28cce3cfb507..0d9ddc8fde91 100644 --- a/apps/desktop/src/settings/DesktopClientSettings.test.ts +++ b/apps/desktop/src/settings/DesktopClientSettings.test.ts @@ -26,7 +26,6 @@ const clientSettings: ClientSettings = { confirmThreadArchive: true, confirmThreadDelete: false, confirmThreadUnpin: false, - continueThreadsAfterServerUpdate: true, contextWindowMeterEnabled: false, composerCollapseOnBlur: false, composerCollapseOnScroll: true, diff --git a/apps/server/src/provider/Layers/ProviderService.test.ts b/apps/server/src/provider/Layers/ProviderService.test.ts index 5e59960685a7..95e5a7983346 100644 --- a/apps/server/src/provider/Layers/ProviderService.test.ts +++ b/apps/server/src/provider/Layers/ProviderService.test.ts @@ -452,6 +452,125 @@ function makeProviderServiceLayer( }; } +for (const [enabled, completed] of [ + [false, false], + [true, false], + [true, true], +] as const) { + it.effect( + `persists shutdown recovery before stopping providers when enabled=${enabled}, completed=${completed}`, + () => + Effect.gen(function* () { + const codex = makeFakeCodexAdapter(); + const persistence = yield* Layer.build( + ProviderSessionDirectoryLive.pipe( + Layer.provide( + ProviderSessionRuntime.layer.pipe(Layer.provide(SqlitePersistenceMemory)), + ), + ), + ); + const directory = yield* ProviderSessionDirectory.ProviderSessionDirectory.pipe( + Effect.provide(persistence), + ); + const threadId = asThreadId("shutdown-recovery"); + const turnId = asTurnId("shutdown-recovery-turn"); + const scope = yield* Scope.make(); + const services = yield* Layer.build( + makeProviderServiceLive().pipe( + Layer.provide( + Layer.succeed(ProviderSessionDirectory.ProviderSessionDirectory, directory), + ), + Layer.provide( + Layer.succeed( + ProviderAdapterRegistry.ProviderAdapterRegistry, + makeStaticInstanceRegistry([[codexInstanceId, codex.adapter]]), + ), + ), + Layer.provide(ServerSettings.layerTest({ continueThreadsAfterServerUpdate: enabled })), + Layer.provide(serverConfigTestLayer), + Layer.provide(AnalyticsService.layerTest), + Layer.provide( + Layer.succeed( + ProviderEventLoggers.ProviderEventLoggers, + ProviderEventLoggers.NoOpProviderEventLoggers, + ), + ), + ), + ).pipe(Scope.provide(scope)); + const provider = yield* ProviderService.ProviderService.pipe(Effect.provide(services)); + const session = yield* provider.startSession(threadId, { + provider: CODEX_DRIVER, + providerInstanceId: codexInstanceId, + threadId, + runtimeMode: "full-access", + }); + codex.listSessions.mockReturnValue( + Effect.succeed([ + { + ...session, + status: completed ? "ready" : "running", + activeTurnId: completed ? undefined : turnId, + }, + ]), + ); + const pending = yield* directory.getBinding(threadId); + assert(Option.isSome(pending)); + yield* directory.upsert({ + ...pending.value, + runtimePayload: { activeTurnId: null, continueAfterServerUpdate: turnId }, + }); + const accepted = yield* provider.sendTurn({ threadId, continuation: true }); + const admitted = yield* directory.getBinding(threadId); + assert(Option.isSome(admitted)); + assert.propertyVal(admitted.value.runtimePayload, "activeTurnId", accepted.turnId); + assert.propertyVal(admitted.value.runtimePayload, "continueAfterServerUpdate", null); + if (completed) { + // Updates can mark an already-admitted turn immediately before it finishes. + yield* directory.upsert({ + ...admitted.value, + runtimePayload: { + continueAfterServerUpdate: accepted.turnId, + continueAfterServerUpdatePrepared: null, + }, + }); + } + const markers: unknown[] = []; + codex.stopAll.mockImplementation(() => + Effect.gen(function* () { + const binding = yield* directory.getBinding(threadId); + assert(Option.isSome(binding)); + markers.push(binding.value.runtimePayload); + }).pipe(Effect.orDie), + ); + yield* Scope.close(scope, Exit.void); + const binding = yield* directory.getBinding(threadId); + assert(Option.isSome(binding)); + assert.equal(codex.stopAll.mock.calls.length, 1); + assert.deepStrictEqual(binding.value.resumeCursor, session.resumeCursor); + assert.equal(binding.value.status, "stopped"); + assert.propertyVal(markers[0], "activeTurnId", completed ? null : turnId); + if (enabled && !completed) { + assert.propertyVal(markers[0], "continueAfterServerUpdate", turnId); + assert.propertyVal(binding.value.runtimePayload, "continueAfterServerUpdate", turnId); + } else if (completed) { + assert.propertyVal( + binding.value.runtimePayload, + "continueAfterServerUpdate", + accepted.turnId, + ); + assert.propertyVal( + binding.value.runtimePayload, + "continueAfterServerUpdatePrepared", + null, + ); + } else { + assert.propertyVal(markers[0], "continueAfterServerUpdate", null); + assert.propertyVal(binding.value.runtimePayload, "continueAfterServerUpdate", null); + } + }).pipe(Effect.provide(NodeServices.layer)), + ); +} + it.effect("ProviderServiceLive catches stopAll failures during shutdown", () => Effect.gen(function* () { const codex = makeFakeCodexAdapter(); diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index 2098cacda5eb..cf9d9b395d5b 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -223,6 +223,7 @@ function toRuntimePayloadFromSession( session: ProviderSession, extra?: { readonly modelSelection?: unknown; + readonly continueAfterServerUpdate?: TurnId; readonly lastRuntimeEvent?: string; readonly lastRuntimeEventAt?: string; }, @@ -232,6 +233,9 @@ function toRuntimePayloadFromSession( model: session.model ?? null, activeTurnId: session.activeTurnId ?? null, lastError: session.lastError ?? null, + ...(extra?.continueAfterServerUpdate !== undefined + ? { continueAfterServerUpdate: extra.continueAfterServerUpdate } + : {}), ...(extra?.modelSelection !== undefined ? { modelSelection: extra.modelSelection } : {}), ...(extra?.lastRuntimeEvent !== undefined ? { lastRuntimeEvent: extra.lastRuntimeEvent } : {}), ...(extra?.lastRuntimeEventAt !== undefined @@ -831,6 +835,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( threadId: ThreadId, extra?: { readonly modelSelection?: unknown; + readonly continueAfterServerUpdate?: TurnId; readonly lastRuntimeEvent?: string; readonly lastRuntimeEventAt?: string; }, @@ -1433,6 +1438,9 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( runtimePayload: { ...(input.modelSelection !== undefined ? { modelSelection: input.modelSelection } : {}), activeTurnId: turn.turnId, + // Admission and marker consumption must survive the same restart. + continueAfterServerUpdate: null, + continueAfterServerUpdatePrepared: null, lastRuntimeEvent: "provider.sendTurn", lastRuntimeEventAt: yield* nowIso, }, @@ -1733,6 +1741,8 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( status: "stopped", runtimePayload: { activeTurnId: null, + continueAfterServerUpdate: null, + continueAfterServerUpdatePrepared: null, }, }); yield* analytics.record("provider.session.stopped", { @@ -1942,6 +1952,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ); const runStopAll = Effect.fn("runStopAll")(function* () { + const continueAfterRestart = yield* serverSettings.getSettings.pipe( + Effect.map((settings) => settings.continueThreadsAfterServerUpdate), + Effect.orElseSucceed(() => false), + ); const properties = yield* Ref.modify(turnAnalytics, (state) => { const completed: Array>> = []; for (const [sessionKey, session] of state.sessions) { @@ -1969,6 +1983,9 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( yield* Effect.forEach(activeSessions, (session) => Effect.flatMap(nowIso, (lastRuntimeEventAt) => upsertSessionBinding(session, session.threadId, { + ...(continueAfterRestart && session.status === "running" && session.activeTurnId + ? { continueAfterServerUpdate: session.activeTurnId } + : {}), lastRuntimeEvent: "provider.stopAll", lastRuntimeEventAt, }), diff --git a/apps/server/src/serverRuntimeStartup.reconcile.test.ts b/apps/server/src/serverRuntimeStartup.reconcile.test.ts index 39c284330d46..2c95a6163acd 100644 --- a/apps/server/src/serverRuntimeStartup.reconcile.test.ts +++ b/apps/server/src/serverRuntimeStartup.reconcile.test.ts @@ -1,6 +1,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { type OrchestrationCommand, + type OrchestrationSessionStatus, ProviderDriverKind, ProviderInstanceId, type ProviderSendTurnInput, @@ -10,15 +11,21 @@ import { import { assert, it } from "@effect/vitest"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Stream from "effect/Stream"; import { OrchestrationCommandInvariantError } from "./orchestration/Errors.ts"; import * as OrchestrationEngine from "./orchestration/Services/OrchestrationEngine.ts"; import * as ProjectionSnapshotQuery from "./orchestration/Services/ProjectionSnapshotQuery.ts"; -import { ProviderSessionDirectoryPersistenceError } from "./provider/Errors.ts"; +import { + ProviderSessionDirectoryPersistenceError, + ProviderSessionNotFoundError, +} from "./provider/Errors.ts"; import * as ProviderService from "./provider/Services/ProviderService.ts"; import * as ProviderSessionDirectory from "./provider/Services/ProviderSessionDirectory.ts"; +import { ServerActivation } from "./serverActivation.ts"; +import * as ServerSettings from "./serverSettings.ts"; import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts"; const providerInstanceId = ProviderInstanceId.make("codex"); @@ -26,7 +33,7 @@ const updatedAt = "2026-08-20T12:00:00.000Z"; const makeThread = ( id: string, - status: "starting" | "running" | "ready" | "stopped" | "error", + status: OrchestrationSessionStatus, activeTurnId: TurnId | null = null, archivedAt: string | null = null, deletedAt: string | null = null, @@ -73,6 +80,7 @@ const queryWithThreads = (threads: ReadonlyArray>) const runReconciliation = (input: { readonly threads: ReadonlyArray>; + readonly continueAfterRestart?: boolean; readonly liveThreadIds?: ReadonlyArray; readonly providerService?: ProviderService.ProviderService["Service"]; readonly directory: ProviderSessionDirectory.ProviderSessionDirectory["Service"]; @@ -97,7 +105,14 @@ const runReconciliation = (input: { subscribeDomainEvents: Effect.succeed(Stream.empty), latestSequence: Effect.succeed(0), }), - Effect.provide(NodeServices.layer), + Effect.provide( + Layer.mergeAll( + ServerSettings.layerTest({ + continueThreadsAfterServerUpdate: input.continueAfterRestart ?? false, + }), + NodeServices.layer, + ), + ), ); it.effect("marks active running sessions that have persisted resume state", () => { @@ -138,7 +153,7 @@ it.effect("marks active running sessions that have persisted resume state", () = upsert: (binding) => Effect.sync(() => upserts.push(binding)), getProvider: () => Effect.die("unused"), listThreadIds: () => Effect.die("unused"), - listBindings: () => Effect.die("unused"), + listBindings: () => Effect.succeed([]), }), Effect.tap((marked) => Effect.sync(() => { @@ -147,169 +162,186 @@ it.effect("marks active running sessions that have persisted resume state", () = assert.deepStrictEqual(upserts[0]?.runtimePayload, { activeTurnId: "turn-mark-active", continueAfterServerUpdate: active.session.activeTurnId, + continueAfterServerUpdatePrepared: null, }); }), ), ); }); -it.effect("continues marked sessions after activation with provider-specific input", () => - Effect.gen(function* () { - const codex = makeThread( - "thread-continue-codex", - "running", - TurnId.make("turn-continue-codex"), - ); - const fallback = makeThread("thread-continue-fallback", "starting"); - const fallbackContinuationTurnId = TurnId.make("turn-continue-fallback"); - const fallbackProviderInstanceId = ProviderInstanceId.make("claudeAgent"); - const continuationSent = yield* Deferred.make(); - const continuationCleared = yield* Deferred.make(); - const sends: ProviderSendTurnInput[] = []; - const dispatched: OrchestrationCommand[] = []; - const upserts: ProviderSessionDirectory.ProviderRuntimeBinding[] = []; - const bindings = new Map( - [codex, fallback].map((thread) => [ - thread.id, - { - threadId: thread.id, - provider: - thread.id === codex.id - ? ProviderDriverKind.make("codex") - : ProviderDriverKind.make("claudeAgent"), - providerInstanceId: - thread.id === codex.id ? providerInstanceId : fallbackProviderInstanceId, - status: "running" as const, - runtimePayload: { - continueAfterServerUpdate: - thread.id === codex.id ? codex.session.activeTurnId : fallbackContinuationTurnId, +it.effect.each(["marked update", "opt-in restart"] as const)( + "continues %s sessions after activation with provider-specific input", + (recovery) => + Effect.gen(function* () { + const codex = makeThread( + "thread-continue-codex", + "running", + TurnId.make("turn-continue-codex"), + ); + const fallbackContinuationTurnId = TurnId.make("turn-continue-fallback"); + const fallback = makeThread( + "thread-continue-fallback", + recovery === "marked update" ? "starting" : "running", + recovery === "marked update" ? null : fallbackContinuationTurnId, + ); + const fallbackProviderInstanceId = ProviderInstanceId.make("claudeAgent"); + const continuationSent = yield* Deferred.make(); + const continuationCleared = yield* Deferred.make(); + const sends: ProviderSendTurnInput[] = []; + const dispatched: OrchestrationCommand[] = []; + const upserts: ProviderSessionDirectory.ProviderRuntimeBinding[] = []; + const bindings = new Map( + [codex, fallback].map((thread) => [ + thread.id, + { + threadId: thread.id, + provider: + thread.id === codex.id + ? ProviderDriverKind.make("codex") + : ProviderDriverKind.make("claudeAgent"), + providerInstanceId: + thread.id === codex.id ? providerInstanceId : fallbackProviderInstanceId, + status: "running" as const, + resumeCursor: { threadId: thread.id }, + runtimePayload: + recovery === "marked update" + ? { + continueAfterServerUpdate: + thread.id === codex.id + ? codex.session.activeTurnId + : fallbackContinuationTurnId, + } + : { activeTurnId: thread.session.activeTurnId }, }, - }, - ]), - ); - const providerService: ProviderService.ProviderService["Service"] = { - ...makeProviderService(), - getCapabilities: (instanceId) => - Effect.succeed({ - sessionModelSwitch: "in-session", - ...(instanceId === providerInstanceId ? { promptlessTurnContinuation: true } : {}), - }), - sendTurn: (input) => - Effect.gen(function* () { - sends.push(input); - if (sends.length === 2) { - yield* Deferred.succeed(continuationSent, undefined); - } - return { - threadId: input.threadId, - turnId: TurnId.make(`continued-${String(input.threadId)}`), - }; - }), - }; - - yield* runReconciliation({ - threads: [codex, fallback], - providerService, - directory: { - getBinding: (threadId) => - Effect.sync(() => { - const binding = bindings.get(threadId); - return binding === undefined ? Option.none() : Option.some(binding); + ]), + ); + const providerService: ProviderService.ProviderService["Service"] = { + ...makeProviderService(), + getCapabilities: (instanceId) => + Effect.succeed({ + sessionModelSwitch: "in-session", + ...(instanceId === providerInstanceId ? { promptlessTurnContinuation: true } : {}), }), - upsert: (binding) => - Effect.sync(() => { - bindings.set(binding.threadId, binding); - upserts.push(binding); - const clearedCount = upserts.filter((candidate) => { - const payload = candidate.runtimePayload; - return ( - payload !== null && - typeof payload === "object" && - !Array.isArray(payload) && - "continueAfterServerUpdate" in payload && - payload.continueAfterServerUpdate === null - ); - }).length; - return clearedCount === 1; - }).pipe( - Effect.flatMap((firstMarkerCleared) => - firstMarkerCleared ? Deferred.succeed(continuationCleared, undefined) : Effect.void, + sendTurn: (input) => + Effect.gen(function* () { + sends.push(input); + if (sends.length === 2) { + yield* Deferred.succeed(continuationSent, undefined); + } + return { + threadId: input.threadId, + turnId: TurnId.make(`continued-${String(input.threadId)}`), + }; + }), + }; + + yield* runReconciliation({ + threads: [codex, fallback], + continueAfterRestart: recovery === "opt-in restart", + providerService, + directory: { + getBinding: (threadId) => + Effect.sync(() => { + const binding = bindings.get(threadId); + return binding === undefined ? Option.none() : Option.some(binding); + }), + upsert: (binding) => + Effect.sync(() => { + bindings.set(binding.threadId, binding); + upserts.push(binding); + const clearedCount = upserts.filter((candidate) => { + const payload = candidate.runtimePayload; + return ( + payload !== null && + typeof payload === "object" && + !Array.isArray(payload) && + "continueAfterServerUpdate" in payload && + payload.continueAfterServerUpdate === null + ); + }).length; + return clearedCount === 1; + }).pipe( + Effect.flatMap((firstMarkerCleared) => + firstMarkerCleared ? Deferred.succeed(continuationCleared, undefined) : Effect.void, + ), ), + getProvider: () => Effect.die("unused"), + listThreadIds: () => Effect.die("unused"), + listBindings: () => Effect.succeed([]), + }, + dispatch: (command) => + Effect.sync(() => dispatched.push(command)).pipe( + Effect.as({ sequence: dispatched.length }), ), - getProvider: () => Effect.die("unused"), - listThreadIds: () => Effect.die("unused"), - listBindings: () => Effect.die("unused"), - }, - dispatch: (command) => - Effect.sync(() => dispatched.push(command)).pipe( - Effect.as({ sequence: dispatched.length }), - ), - }); - yield* Deferred.await(continuationSent); - yield* Deferred.await(continuationCleared); + }); + yield* Deferred.await(continuationSent); + yield* Deferred.await(continuationCleared); - assert.deepStrictEqual( - sends.toSorted((left, right) => String(left.threadId).localeCompare(String(right.threadId))), - [ - { threadId: codex.id, continuation: true, interactionMode: "default" }, - { - threadId: fallback.id, - input: "Continue where you left off.", - interactionMode: "default", - }, - ], - ); - assert.deepStrictEqual( - dispatched.map((command) => - command.type === "thread.session.set" - ? { - threadId: command.threadId, - status: command.session.status, - activeTurnId: command.session.activeTurnId, - } - : null, - ), - [ - { - threadId: codex.id, - status: "starting", - activeTurnId: null, - }, - { - threadId: fallback.id, - status: "starting", - activeTurnId: fallback.session.activeTurnId, - }, - ], - ); - for (const [thread, continuationTurnId] of [ - [codex, codex.session.activeTurnId], - [fallback, fallbackContinuationTurnId], - ] as const) { assert.deepStrictEqual( - upserts - .filter((binding) => binding.threadId === thread.id) - .map((binding) => binding.runtimePayload)[0], - { - continueAfterServerUpdate: continuationTurnId, - activeTurnId: null, - }, + sends.toSorted((left, right) => + String(left.threadId).localeCompare(String(right.threadId)), + ), + [ + { threadId: codex.id, continuation: true, interactionMode: "default" }, + { + threadId: fallback.id, + input: "Continue where you left off.", + interactionMode: "default", + }, + ], + ); + assert.deepStrictEqual( + dispatched.map((command) => + command.type === "thread.session.set" + ? { + threadId: command.threadId, + status: command.session.status, + activeTurnId: command.session.activeTurnId, + } + : null, + ), + [ + { + threadId: codex.id, + status: "starting", + activeTurnId: null, + }, + { + threadId: fallback.id, + status: "starting", + activeTurnId: null, + }, + ], ); - } - assert.equal( - upserts.some((binding) => { - const payload = binding.runtimePayload; - return ( - payload !== null && - typeof payload === "object" && - !Array.isArray(payload) && - "continueAfterServerUpdate" in payload && - payload.continueAfterServerUpdate === null + for (const [thread, continuationTurnId] of [ + [codex, codex.session.activeTurnId], + [fallback, fallbackContinuationTurnId], + ] as const) { + assert.deepStrictEqual( + upserts + .filter((binding) => binding.threadId === thread.id) + .map((binding) => binding.runtimePayload)[0], + { + continueAfterServerUpdate: continuationTurnId, + continueAfterServerUpdatePrepared: true, + activeTurnId: null, + }, ); - }), - true, - ); - }), + } + assert.equal( + upserts.some((binding) => { + const payload = binding.runtimePayload; + return ( + payload !== null && + typeof payload === "object" && + !Array.isArray(payload) && + "continueAfterServerUpdate" in payload && + payload.continueAfterServerUpdate === null + ); + }), + true, + ); + }), ); it.effect("does not continue archived or deleted marked sessions", () => { @@ -361,7 +393,7 @@ it.effect("does not continue archived or deleted marked sessions", () => { upsert: () => Effect.void, getProvider: () => Effect.die("unused"), listThreadIds: () => Effect.die("unused"), - listBindings: () => Effect.die("unused"), + listBindings: () => Effect.succeed([]), }, dispatch: (command) => Effect.sync(() => dispatched.push(command)).pipe(Effect.as({ sequence: dispatched.length })), @@ -416,7 +448,7 @@ it.effect("retries continuation preparation before settling a persistent failure upsert: () => Effect.void, getProvider: () => Effect.die("unused"), listThreadIds: () => Effect.die("unused"), - listBindings: () => Effect.die("unused"), + listBindings: () => Effect.succeed([]), }, dispatch: (command) => { if (command.type !== "thread.session.set") { @@ -487,7 +519,7 @@ it.effect("reconciles multiple active and archived orphans but skips live sessio upsert: (binding) => Effect.sync(() => upserts.push(binding)), getProvider: () => Effect.die("unused"), listThreadIds: () => Effect.die("unused"), - listBindings: () => Effect.die("unused"), + listBindings: () => Effect.succeed([]), }, dispatch: (command) => Effect.sync(() => dispatched.push(command)).pipe(Effect.as({ sequence: dispatched.length })), @@ -521,6 +553,7 @@ it.effect("reconciles multiple active and archived orphans but skips live sessio activeTurnId: null, unrelated: binding.threadId, continueAfterServerUpdate: null, + continueAfterServerUpdatePrepared: null, } : { activeTurnId: null, unrelated: binding.threadId }, ); @@ -565,7 +598,7 @@ it.effect( upsert: () => Effect.fail(writeFailure), getProvider: () => Effect.die("unused"), listThreadIds: () => Effect.die("unused"), - listBindings: () => Effect.die("unused"), + listBindings: () => Effect.succeed([]), }, dispatch: (command) => Effect.sync(() => dispatched.push(command)).pipe( @@ -602,7 +635,7 @@ it.effect("retries failed projections and continues after a persistent failure", upsert: () => Effect.void, getProvider: () => Effect.die("unused"), listThreadIds: () => Effect.die("unused"), - listBindings: () => Effect.die("unused"), + listBindings: () => Effect.succeed([]), }, dispatch: (command) => { if (command.type !== "thread.session.set") { @@ -651,7 +684,7 @@ it.effect("does not fail startup when the live provider session inventory cannot upsert: () => Effect.die("unused"), getProvider: () => Effect.die("unused"), listThreadIds: () => Effect.die("unused"), - listBindings: () => Effect.die("unused"), + listBindings: () => Effect.succeed([]), }), Effect.provideService(OrchestrationEngine.OrchestrationEngineService, { readEvents: () => Stream.empty, @@ -662,7 +695,283 @@ it.effect("does not fail startup when the live provider session inventory cannot subscribeDomainEvents: Effect.succeed(Stream.empty), latestSequence: Effect.succeed(0), }), - Effect.provide(NodeServices.layer), + Effect.provide(Layer.mergeAll(NodeServices.layer, ServerSettings.layerTest())), Effect.tap(() => Effect.sync(() => assert.equal(queried, false))), ); }); + +for (const scenario of [ + "disabled", + "stopped projection", + "finished projection", + "stopped binding", + "finished binding", + "missing cursor", + "mismatched turn", + "marked without cursor", + "marked stopped projection", + "marked superseded turn", +] as const) { + it.effect(`does not recover an interrupted session with ${scenario}`, () => { + const turnId = TurnId.make("turn-excluded-recovery"); + const thread = makeThread( + "thread-excluded-recovery", + scenario.includes("stopped projection") + ? "stopped" + : scenario === "finished projection" + ? "ready" + : scenario === "marked superseded turn" + ? "starting" + : "running", + scenario === "marked superseded turn" ? null : turnId, + ); + const dispatched: OrchestrationCommand[] = []; + const upserts: ProviderSessionDirectory.ProviderRuntimeBinding[] = []; + return runReconciliation({ + threads: [thread], + continueAfterRestart: scenario !== "disabled", + directory: { + getBinding: () => + Effect.succeed( + Option.some({ + threadId: thread.id, + provider: ProviderDriverKind.make("codex"), + providerInstanceId, + status: scenario === "stopped binding" ? "stopped" : "running", + ...(scenario.includes("cursor") ? {} : { resumeCursor: { threadId: thread.id } }), + runtimePayload: { + activeTurnId: + scenario === "finished binding" + ? null + : scenario === "mismatched turn" || scenario === "marked superseded turn" + ? "another-turn" + : turnId, + ...(scenario.startsWith("marked") ? { continueAfterServerUpdate: turnId } : {}), + }, + }), + ), + upsert: (binding) => + Effect.sync(() => { + upserts.push(binding); + }), + getProvider: () => Effect.die("unused"), + listThreadIds: () => Effect.die("unused"), + listBindings: () => Effect.succeed([]), + }, + dispatch: (command) => + Effect.sync(() => { + dispatched.push(command); + return { sequence: dispatched.length }; + }), + }).pipe( + Effect.tap(() => + Effect.sync(() => { + assert.deepStrictEqual( + dispatched.map( + (command) => command.type === "thread.session.set" && command.session.status, + ), + ["error"], + ); + assert.deepStrictEqual( + upserts.map((binding) => binding.status), + ["stopped"], + ); + }), + ), + ); + }); +} + +for (const preparedStatus of [ + "starting", + "ready", + "ready with failed scan", + "completed after update marking", +] as const) { + it.effect(`recovers again if startup exits with a prepared ${preparedStatus} session`, () => + Effect.gen(function* () { + const turnId = TurnId.make("turn-interrupted-startup"); + const thread = makeThread("thread-interrupted-startup", "running", turnId); + const activation = yield* Deferred.make(); + const cleared = yield* Deferred.make(); + const sends: ProviderSendTurnInput[] = []; + let binding: ProviderSessionDirectory.ProviderRuntimeBinding = { + threadId: thread.id, + provider: ProviderDriverKind.make("codex"), + providerInstanceId, + status: "running", + resumeCursor: { threadId: thread.id }, + runtimePayload: { activeTurnId: turnId }, + }; + const input = { + threads: [thread], + continueAfterRestart: true, + providerService: { + ...makeProviderService(), + getCapabilities: () => + Effect.succeed({ + sessionModelSwitch: "in-session" as const, + promptlessTurnContinuation: true, + }), + sendTurn: (input: ProviderSendTurnInput) => + Effect.sync(() => { + sends.push(input); + return { threadId: input.threadId, turnId: TurnId.make("turn-recovered") }; + }), + }, + directory: { + getBinding: () => Effect.sync(() => Option.some(binding)), + upsert: (next: ProviderSessionDirectory.ProviderRuntimeBinding) => + Effect.gen(function* () { + binding = next; + if (binding.status !== "starting" || sends.length === 0) return; + yield* Deferred.succeed(cleared, undefined); + }), + getProvider: () => Effect.die("unused"), + listThreadIds: () => Effect.die("unused"), + listBindings: () => + preparedStatus === "ready with failed scan" + ? Effect.fail( + new ProviderSessionDirectoryPersistenceError({ + operation: "listBindings", + detail: "unreadable unrelated binding", + }), + ) + : Effect.sync(() => [{ ...binding, lastSeenAt: "2026-01-01T00:00:00.000Z" }]), + }, + dispatch: (command: OrchestrationCommand) => + Effect.sync(() => { + if (command.type === "thread.session.set") { + thread.session.status = command.session.status; + thread.session.activeTurnId = command.session.activeTurnId; + } + return { sequence: 1 }; + }), + }; + + yield* runReconciliation(input).pipe( + Effect.provideService(ServerActivation, Deferred.await(activation)), + Effect.scoped, + ); + assert.deepStrictEqual(sends, []); + assert.equal(thread.session.status, "starting"); + assert.equal(thread.session.activeTurnId, null); + assert.deepStrictEqual(binding.runtimePayload, { + activeTurnId: null, + continueAfterServerUpdate: turnId, + continueAfterServerUpdatePrepared: true, + }); + + if (preparedStatus === "completed after update marking") { + thread.session.status = "ready"; + binding = { + ...binding, + status: "stopped", + runtimePayload: { + activeTurnId: null, + continueAfterServerUpdate: turnId, + continueAfterServerUpdatePrepared: null, + }, + }; + yield* runReconciliation(input); + assert.deepStrictEqual(sends, []); + assert.equal(thread.session.status, "ready"); + return; + } + thread.session.status = + preparedStatus === "ready with failed scan" ? "ready" : preparedStatus; + yield* runReconciliation(input); + yield* Deferred.await(cleared); + assert.deepStrictEqual(sends, [ + { threadId: thread.id, continuation: true, interactionMode: "default" }, + ]); + assert.deepStrictEqual(binding.runtimePayload, { + activeTurnId: null, + continueAfterServerUpdate: null, + continueAfterServerUpdatePrepared: null, + }); + }), + ); +} + +it.effect("settles failed opt-in recovery without retrying the provider turn", () => + Effect.gen(function* () { + const turnId = TurnId.make("turn-failed-recovery"); + const thread = makeThread("thread-failed-recovery", "running", turnId); + const settled = yield* Deferred.make(); + const sends: ProviderSendTurnInput[] = []; + const dispatched: OrchestrationCommand[] = []; + const preparedPayloads: unknown[] = []; + let binding: ProviderSessionDirectory.ProviderRuntimeBinding = { + threadId: thread.id, + provider: ProviderDriverKind.make("codex"), + providerInstanceId, + status: "running", + resumeCursor: { threadId: thread.id }, + runtimePayload: { activeTurnId: turnId }, + }; + yield* runReconciliation({ + threads: [thread], + continueAfterRestart: true, + providerService: { + ...makeProviderService(), + getCapabilities: () => + Effect.succeed({ sessionModelSwitch: "in-session", promptlessTurnContinuation: true }), + sendTurn: (input) => + Effect.gen(function* () { + sends.push(input); + preparedPayloads.push(binding.runtimePayload); + return yield* Effect.fail( + new ProviderSessionNotFoundError({ threadId: input.threadId }), + ); + }), + }, + directory: { + getBinding: () => Effect.sync(() => Option.some(binding)), + upsert: (next) => + Effect.sync(() => { + binding = next; + }), + getProvider: () => Effect.die("unused"), + listThreadIds: () => Effect.die("unused"), + listBindings: () => Effect.succeed([]), + }, + dispatch: (command) => + Effect.gen(function* () { + dispatched.push(command); + if (command.type === "thread.session.set" && command.session.status === "error") { + yield* Deferred.succeed(settled, undefined); + } + return { sequence: dispatched.length }; + }), + }); + yield* Deferred.await(settled); + assert.equal(sends.length, 1); + assert.deepStrictEqual(preparedPayloads, [ + { + activeTurnId: null, + continueAfterServerUpdate: turnId, + continueAfterServerUpdatePrepared: true, + }, + ]); + assert.deepStrictEqual( + dispatched.map( + (command) => + command.type === "thread.session.set" && { + status: command.session.status, + activeTurnId: command.session.activeTurnId, + }, + ), + [ + { status: "starting", activeTurnId: null }, + { status: "error", activeTurnId: null }, + ], + ); + assert.equal(binding.status, "stopped"); + assert.deepStrictEqual(binding.runtimePayload, { + activeTurnId: null, + continueAfterServerUpdate: null, + continueAfterServerUpdatePrepared: null, + }); + }), +); diff --git a/apps/server/src/serverRuntimeStartup.ts b/apps/server/src/serverRuntimeStartup.ts index 064796b2810b..a34d1bdbde91 100644 --- a/apps/server/src/serverRuntimeStartup.ts +++ b/apps/server/src/serverRuntimeStartup.ts @@ -394,6 +394,7 @@ export const markRunningProviderSessionsForContinuation = Effect.gen(function* ( runtimePayload: { ...readRuntimePayload(binding.value.runtimePayload), [SERVER_UPDATE_CONTINUATION_KEY]: activeTurnId, + continueAfterServerUpdatePrepared: null, }, }); marked.push(thread.id); @@ -423,6 +424,7 @@ const clearContinuationMarkers = ( runtimePayload: { ...readRuntimePayload(binding.runtimePayload), [SERVER_UPDATE_CONTINUATION_KEY]: null, + continueAfterServerUpdatePrepared: null, }, }), }), @@ -443,17 +445,56 @@ export const reconcileProviderSessions = Effect.gen(function* () { const orchestrationEngine = yield* OrchestrationEngine.OrchestrationEngineService; const providerService = yield* ProviderService.ProviderService; const query = yield* ProjectionSnapshotQuery.ProjectionSnapshotQuery; + const settings = yield* ServerSettings.ServerSettingsService; + const continueAfterRestart = yield* settings.getSettings.pipe( + Effect.map((value) => value.continueThreadsAfterServerUpdate), + Effect.catch((cause) => + Effect.logWarning("could not read restart continuation preference", { cause }).pipe( + Effect.as(false), + ), + ), + ); const liveThreadIds = new Set( (yield* providerService.listSessions()).map((session) => session.threadId), ); const { threads } = yield* query.getCommandReadModel(); + // Provider startup can report ready before the continuation is submitted. + // Find those markers in one read rather than querying every idle thread. + const preparedThreadIds = new Set( + (yield* directory.listBindings().pipe( + Effect.catch((cause) => + Effect.logWarning("failed to read prepared provider continuations", { cause }).pipe( + Effect.andThen( + Effect.forEach( + threads.filter( + (thread) => thread.session?.status === "ready" && !liveThreadIds.has(thread.id), + ), + (thread) => + directory.getBinding(thread.id).pipe(Effect.orElseSucceed(() => Option.none())), + ), + ), + Effect.map((bindings) => + bindings.flatMap((binding) => (Option.isSome(binding) ? [binding.value] : [])), + ), + ), + ), + )) + .filter( + (binding) => + readServerUpdateContinuationTurnId(binding.runtimePayload) !== null && + readRuntimePayload(binding.runtimePayload).activeTurnId === null && + readRuntimePayload(binding.runtimePayload).continueAfterServerUpdatePrepared === true, + ) + .map((binding) => binding.threadId), + ); const orphanedThreads = threads.filter( (thread) => thread.session !== null && (thread.session.status === "starting" || thread.session.status === "running" || - thread.session.activeTurnId !== null) && + thread.session.activeTurnId !== null || + (thread.session.status === "ready" && preparedThreadIds.has(thread.id))) && !liveThreadIds.has(thread.id), ); @@ -479,7 +520,27 @@ export const reconcileProviderSessions = Effect.gen(function* () { : null; const continuationMarked = continuationTurnId !== null && - (session.activeTurnId === null || continuationTurnId === session.activeTurnId); + (session.activeTurnId === null || continuationTurnId === session.activeTurnId) && + Option.isSome(binding) && + (readRuntimePayload(binding.value.runtimePayload).activeTurnId == null || + readRuntimePayload(binding.value.runtimePayload).activeTurnId === continuationTurnId); + const preparedWhileReady = + session.status === "ready" && + session.activeTurnId === null && + continuationMarked && + Option.isSome(binding) && + readRuntimePayload(binding.value.runtimePayload).activeTurnId === null && + readRuntimePayload(binding.value.runtimePayload).continueAfterServerUpdatePrepared === true; + // Abrupt shutdowns cannot write an update marker. Require both durable + // records to agree on an unfinished turn before recovering one implicitly. + const interruptedByRestart = + continueAfterRestart && + session.status === "running" && + session.activeTurnId !== null && + Option.isSome(binding) && + binding.value.status === "running" && + binding.value.resumeCursor != null && + readRuntimePayload(binding.value.runtimePayload).activeTurnId === session.activeTurnId; const settleAsError = (lastError: string) => Effect.gen(function* () { yield* Effect.gen(function* () { @@ -490,7 +551,12 @@ export const reconcileProviderSessions = Effect.gen(function* () { runtimePayload: { ...readRuntimePayload(binding.value.runtimePayload), activeTurnId: null, - ...(continuationMarkerPresent ? { [SERVER_UPDATE_CONTINUATION_KEY]: null } : {}), + ...(continuationMarkerPresent || interruptedByRestart + ? { + [SERVER_UPDATE_CONTINUATION_KEY]: null, + continueAfterServerUpdatePrepared: null, + } + : {}), }, }); } @@ -535,7 +601,9 @@ export const reconcileProviderSessions = Effect.gen(function* () { if ( Option.isSome(binding) && - continuationMarked && + (continuationMarked || interruptedByRestart) && + (session.status === "running" || session.status === "starting" || preparedWhileReady) && + binding.value.resumeCursor != null && thread.archivedAt === null && thread.deletedAt === null ) { @@ -545,6 +613,9 @@ export const reconcileProviderSessions = Effect.gen(function* () { status: "starting", runtimePayload: { ...readRuntimePayload(binding.value.runtimePayload), + // Keep recovery durable if this process also exits before sending. + [SERVER_UPDATE_CONTINUATION_KEY]: session.activeTurnId ?? continuationTurnId, + continueAfterServerUpdatePrepared: true, activeTurnId: null, }, }); @@ -608,12 +679,12 @@ export const reconcileProviderSessions = Effect.gen(function* () { } return; } - yield* Effect.logWarning("failed to continue provider session after server update", { + yield* Effect.logWarning("failed to continue provider session after server restart", { threadId: thread.id, cause: continuationExit.cause, }); yield* settleAsError( - "Could not continue this thread after the server update. Send a new message to continue.", + "Could not continue this thread after the server restart. Send a new message to continue.", ).pipe(Effect.ignoreCause); }), ); diff --git a/apps/web/src/components/ServerUpdateAction.test.tsx b/apps/web/src/components/ServerUpdateAction.test.tsx index 584078c12bf4..67236361f677 100644 --- a/apps/web/src/components/ServerUpdateAction.test.tsx +++ b/apps/web/src/components/ServerUpdateAction.test.tsx @@ -15,7 +15,8 @@ vi.mock("~/hooks/useCopyToClipboard", () => ({ useCopyToClipboard: () => ({ copyToClipboard: vi.fn() }), })); vi.mock("~/hooks/useSettings", () => ({ - useClientSettings: ( + useEnvironmentSettings: ( + _environmentId: EnvironmentId, selector: (settings: { continueThreadsAfterServerUpdate: boolean }) => unknown, ) => selector({ continueThreadsAfterServerUpdate: testState.continueThreadsAfterServerUpdate }), })); diff --git a/apps/web/src/components/ServerUpdateAction.tsx b/apps/web/src/components/ServerUpdateAction.tsx index 71b974416dd3..1845ada716b6 100644 --- a/apps/web/src/components/ServerUpdateAction.tsx +++ b/apps/web/src/components/ServerUpdateAction.tsx @@ -8,7 +8,7 @@ import type { ComponentProps } from "react"; import { requestConfirmDialog } from "~/confirmDialog"; import { useCopyToClipboard } from "~/hooks/useCopyToClipboard"; -import { useClientSettings } from "~/hooks/useSettings"; +import { useEnvironmentSettings } from "~/hooks/useSettings"; import { serverEnvironment } from "~/state/server"; import { useAtomCommand } from "~/state/use-atom-command"; import { manualServerUpdateCommand } from "~/versionSkew"; @@ -99,7 +99,8 @@ export function ServerUpdateAction({ readonly size?: ComponentProps["size"]; }) { const isDesktopAppUpdate = selfUpdate === "desktop-managed"; - const continueThreadsAfterServerUpdate = useClientSettings( + const continueThreadsAfterServerUpdate = useEnvironmentSettings( + environmentId, (settings) => settings.continueThreadsAfterServerUpdate, ); const updateServer = useAtomCommand(serverEnvironment.updateServer, { diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx index b0344b640393..66f7691a4a07 100644 --- a/apps/web/src/components/settings/SettingsPanels.tsx +++ b/apps/web/src/components/settings/SettingsPanels.tsx @@ -564,7 +564,7 @@ export function useSettingsRestore(onRestored?: () => void) { : []), ...(settings.continueThreadsAfterServerUpdate !== DEFAULT_UNIFIED_SETTINGS.continueThreadsAfterServerUpdate - ? ["Continue threads after server updates"] + ? ["Continue threads after restarts"] : []), ...(isBackgroundActivityDirty ? ["Background activity"] : []), ...(settings.defaultThreadEnvMode !== DEFAULT_UNIFIED_SETTINGS.defaultThreadEnvMode @@ -2438,12 +2438,13 @@ export function GeneralSettingsPanel() { updateSettings({ continueThreadsAfterServerUpdate: @@ -2459,7 +2460,7 @@ export function GeneralSettingsPanel() { onCheckedChange={(checked) => updateSettings({ continueThreadsAfterServerUpdate: Boolean(checked) }) } - aria-label="Continue threads after server updates" + aria-label="Continue threads after restarts" /> } /> diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index 22e2204d8a27..7358ed8f9a17 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -218,9 +218,11 @@ export const SETTINGS_SEARCH_ITEMS = [ }, { id: "continue-threads-after-server-update", - title: "Continue threads after server updates", + title: "Continue threads after restarts", to: "/settings/general", - searchTerms: ["resume running active work restart desktop update automatically"], + searchTerms: [ + "resume running active interrupted work restart reboot machine crash desktop update automatically", + ], }, { id: "background-activity", diff --git a/docs/user/updating.md b/docs/user/updating.md index 8ccefb4e5157..14500cbe4620 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -10,9 +10,13 @@ notice. Server updates restart the connection and can interrupt active agents and terminal commands. Saved threads, settings, and project files remain. -**Settings → General → Continue threads after server updates** is off by default. -Enable it to resume supported active threads once the replacement server is -ready. Terminal commands may still be interrupted. +**Settings → General → Continue threads after restarts** is off by default. +Enable it for each environment to resume supported active threads after an +update, crash, or machine restart. T3 Code must start again on that machine; +the setting does not enable automatic startup. Terminal commands may still be +interrupted, and threads without saved provider resume state need a new message. +If you previously enabled continuation for updates, enable this environment +setting once to allow recovery without a connected client. ## Update a connected server diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index cd4b12713edf..51fbc812225c 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -255,9 +255,6 @@ export const ClientSettingsSchema = Schema.Struct({ confirmThreadArchive: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), confirmThreadDelete: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), confirmThreadUnpin: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), - continueThreadsAfterServerUpdate: Schema.Boolean.pipe( - Schema.withDecodingDefault(Effect.succeed(false)), - ), dismissedProviderUpdateNotificationKeys: Schema.Array(TrimmedNonEmptyString).pipe( Schema.withDecodingDefault(Effect.succeed([])), ), @@ -835,6 +832,10 @@ export const ServerSettings = Schema.Struct({ Schema.withDecodingDefault(Effect.succeed(false)), ), enableProviderUpdateChecks: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), + // Retain the update-era key; recovery now needs an environment-owned opt-in. + continueThreadsAfterServerUpdate: Schema.Boolean.pipe( + Schema.withDecodingDefault(Effect.succeed(false)), + ), /** * Whether agents may drive the in-app preview browser. Turning this off * withholds the MCP credential, so the `t3-code` server (and with it every @@ -1105,6 +1106,7 @@ export const ServerSettingsPatch = Schema.Struct({ // Server settings enableLegacyTokenStreaming: Schema.optionalKey(Schema.Boolean), enableProviderUpdateChecks: Schema.optionalKey(Schema.Boolean), + continueThreadsAfterServerUpdate: Schema.optionalKey(Schema.Boolean), enableAgentBrowserAccess: Schema.optionalKey(Schema.Boolean), sidebarAutoSettleAfterDays: Schema.optionalKey(Schema.NullOr(SidebarAutoSettleAfterDays)), sidebarAutoSettleOnMerge: Schema.optionalKey(Schema.Boolean), @@ -1181,7 +1183,6 @@ export const ClientSettingsPatch = Schema.Struct({ confirmThreadArchive: Schema.optionalKey(Schema.Boolean), confirmThreadDelete: Schema.optionalKey(Schema.Boolean), confirmThreadUnpin: Schema.optionalKey(Schema.Boolean), - continueThreadsAfterServerUpdate: Schema.optionalKey(Schema.Boolean), diffIgnoreWhitespace: Schema.optionalKey(Schema.Boolean), diffLayout: Schema.optionalKey(DiffLayout), environmentIdentificationMode: Schema.optionalKey(EnvironmentIdentificationMode),