diff --git a/architecture.md b/architecture.md index 363bb8e1f..d9cd86fe8 100644 --- a/architecture.md +++ b/architecture.md @@ -4640,6 +4640,24 @@ callbacks, documentation capture and public execution policy. Setup receives no installation array, request, entry identifier or receipt and creates no second execution owner. +A host may also **stop** the execution it owns. `HostExecution.stop()` is the +same seam's opt-in: it answers for the stop and the join, never for the +document. It reaches a document that is still running, unwinds it, and joins +every teardown the execution owns before it answers — a stop that returned +while an owned cleanup was still running would be reporting an ending that had +not happened. What the journal keeps is the terminal representation the +protocol already has: one cancelled `Close` for the root, carrying a status and +nothing else. No answer, outcome or successful payload is invented for work the +document never finished, and asking again or reading the completion again +writes no second ending. + +It creates no second execution owner. Initialization and invocation +installations stay execution-owned, whole-document failure ranking is +unchanged, and a persistence failure raised while that ending is being written +stays the run's own failure by identity — the stop answers with it, the owned +teardown is still attempted and joined, and nothing compensating is appended in +its place. + Which middleware a command installs on that seam is the host's own selection, made once by the runtime entrypoint. The bundled Plugin a run-profile command begins with is read from one CLI-private Context whose default is the diff --git a/packages/cli/src/sidekick-acp/posix.ts b/packages/cli/src/sidekick-acp/posix.ts index ae945abd8..9b89e2ca5 100644 --- a/packages/cli/src/sidekick-acp/posix.ts +++ b/packages/cli/src/sidekick-acp/posix.ts @@ -2,7 +2,7 @@ import { action, createSignal, ensure, resource, scoped, until, useScope } from import type { Operation } from "effection"; import { constants } from "node:fs"; import { chmod, link, lstat, mkdtemp, open, realpath, rmdir, unlink } from "node:fs/promises"; -import type { Stats } from "node:fs"; +import type { BigIntStats, Stats } from "node:fs"; import type { FileHandle } from "node:fs/promises"; import { createConnection, createServer } from "node:net"; import type { Server, Socket } from "node:net"; @@ -57,9 +57,71 @@ function closeSocket(socket: Socket): Operation { return () => socket.off("close", closed); }); } +/** + * Whether these two readings are of the same file, and not merely of the same + * name and number. + * + * An inode number is reused. A file this owner published, unlinked by somebody + * else, and a new file created in its place can be handed the number the first + * one had — on Linux commonly, and ext4 in CI did exactly that: the owner's + * exit removed a replacement it had never published, and the name it was asked + * to leave alone was gone. macOS does not reuse as eagerly, which is why one + * host saw it and the other did not. + * + * So the moment the file came into existence is part of its identity. An + * unchanged file keeps it — measured, two readings of one untouched file agree + * — and a file created after another was unlinked cannot share it. Stricter + * rather than looser: it can only ever leave a name alone, never remove one + * this owner did not publish. Where a filesystem reports no creation time both + * readings carry the same substitute and this is the pair it always was. + */ function same(a: Stats, b: Stats): boolean { - return a.dev === b.dev && a.ino === b.ino; + return a.dev === b.dev && a.ino === b.ino && a.birthtimeMs === b.birthtimeMs; } +/** + * Refuse a second reading that is not of the same file as the first. + * + * Each identity below is read again once this owner has finished changing the + * object it names — the times it was created with are not the times it has. + * What may not change between the two readings is which file it is, and that + * is what this says: a device and inode that moved means somebody replaced the + * name while it was being published, and publishing it would be publishing + * theirs. + */ +function sameNode(before: Stats | undefined, after: Stats, what: string): void { + if (before !== undefined && (before.dev !== after.dev || before.ino !== after.ino)) { + throw new Error(`${what} was replaced while it was being published.`); + } +} + +/** Which identity a refresh is about, for the test seam that holds one. */ +type IdentityWindow = "directory" | "stage" | "published"; + +/** + * The regular file this publication created, still open, and which node it is. + * + * A device and inode read from a name is a reading of history: the file it + * described may be gone, and an inode number is handed back to the next file + * created once its last link goes. An open handle is not history. While this + * one is held the original cannot be freed, so its number cannot be given to + * anybody else — which is what makes it safe to say, of a name, that it still + * holds *this* file. + * + * So the handle is the authority and the two numbers below come from it, read + * as bigints because an inode number is one and rounding it to a double would + * put a second ambiguity where this is removing the first. A name's own + * reading answers one question only: is it this node. It never becomes the + * authority itself. + */ +interface Pinned { + handle: FileHandle; + dev: bigint; + ino: bigint; +} +function isPinned(pinned: Pinned, reading: BigIntStats): boolean { + return reading.dev === pinned.dev && reading.ino === pinned.ino; +} + function* removeOwned(path: string, identity: Stats | undefined): Operation { if (identity === undefined) { return; @@ -224,8 +286,49 @@ export function* usePosixSidekickTransport( * nothing. Omitted everywhere but that test. */ beforeCloseAcceptedSocket?: () => Operation, + /** + * Seams for the two things a host can do to this publication that this host + * cannot be made to do. Omitted everywhere but those tests. + * + * `holdIdentityRefresh` keeps a refresh's read in flight, so a test can + * replace the object under it — or cancel this publication — while the read + * this owner's cleanup authority comes from has not resolved yet. + * + * `readDirectoryIdentity` reads the attachment directory the way a host that + * cannot read a creation time reads it: `birthtime` standing in for `ctime`, + * which is libuv's documented fallback and not something APFS will do here. + */ + probes?: { + holdIdentityRefresh?: (window: IdentityWindow) => Promise; + readDirectoryIdentity?: (path: string) => Promise; + /** + * The publication's own open file, as a test can see it: `acquired` is + * handed the handle the moment this publication owns it, and `readName` + * stands in for every reading of a regular name this publication owns, + * including the ones its cleanup takes. + * + * A host cannot be asked to hand one file the number of another, so a test + * that needs that outcome has to describe it. Describing it is only + * honest while the original is gone — which is why `acquired` exists: the + * fixture decides from the real handle, not from a flag. + */ + pin?: { + acquired?: (handle: FileHandle) => void; + readName?: (name: string) => Promise; + /** + * Held between a completed open and this publication taking the handle + * it produced, so a test can cancel in a window no host can be asked to + * provide: the file exists and is open, and nothing owns it yet. + */ + holdAcquisition?: () => Promise; + }; + }, ): Operation { const uid = process.getuid?.(); + const readDirectoryIdentity = probes?.readDirectoryIdentity ?? lstat; + const readName = + probes?.pin?.readName ?? + ((name: string): Promise => lstat(name, { bigint: true })); yield* SidekickTransport.set({ supported: () => platform !== "win32" && uid !== undefined, identify: randomUUID, @@ -255,13 +358,90 @@ export function* usePosixSidekickTransport( acquisitions.push(owned); return until(owned); } + /** + * Read what this owner owns again, and adopt the reading only if it is + * still the same object. + * + * `owned` is the identity this owner already has authority over. It is + * handed in and never written over here, because the check has to + * happen inside the retained callback: that callback is where cleanup's + * authority is installed, it still runs when the caller is cancelled + * while the read is in flight, and the statement after this one may + * never be reached. So a reading of some other device and inode is + * refused there, before it can authorize anything, and what this owner + * already held stays in place — the replacement is somebody else's file + * and keeps being one. + */ + function refresh( + read: () => Promise, + owned: Stats | undefined, + window: IdentityWindow, + what: string, + install: (value: Stats) => void, + ): Operation { + const held = probes?.holdIdentityRefresh?.(window); + return retain(held === undefined ? read() : held.then(read), (value) => { + sameNode(owned, value, what); + install(value); + }); + } + /** + * What this publication owns, and the handle that keeps owning it. + * + * `pinnedFile` is the handle the moment it exists, so a cancellation that + * lands between opening and anything else still has something to + * close. `pin` only exists once the node behind it is known, because + * an identity this publication could not read is not one it may act + * on: without it, no name is removed at all. + */ + let pinnedFile: FileHandle | undefined; + let pin: Pinned | undefined; + /** + * Is this name the file this publication opened — asked of the live + * handle, and asked again every time it matters. + */ + function pinnedRefresh( + name: string, + window: IdentityWindow, + what: string, + ): Operation { + const waiting = probes?.holdIdentityRefresh?.(window); + const read = () => readName(name); + return retain(waiting === undefined ? read() : waiting.then(read), (value) => { + const owned = pin; + if (owned === undefined || !isPinned(owned, value)) { + throw new Error(`${what} was replaced while it was being published.`); + } + }); + } + /** + * Remove a name only while it still holds the pinned file. Somebody + * else's file under that name is left where it is, and a name that is + * already gone is nothing to do. + */ + function* removePinnedName(name: string): Operation { + const owned = pin; + if (owned === undefined) { + return; + } + try { + if (isPinned(owned, yield* until(readName(name)))) { + yield* until(unlink(name)); + } + } catch (error) { + if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) { + throw error; + } + } + } let directory: string | undefined; let directoryAcquisition: Promise | undefined; let directoryIdentity: Stats | undefined; + // The reading the directory was created with. Never written over by a + // refresh: it is what a refreshed reading has to still be. + let directoryOwned: Stats | undefined; let socketIdentity: Stats | undefined; - let descriptorIdentity: Stats | undefined; let stage: string | undefined; - let stageIdentity: Stats | undefined; let server: Server | undefined; const sockets = new Set(); const connections = yield* useScope(); @@ -315,41 +495,132 @@ export function* usePosixSidekickTransport( if (directoryAcquisition !== undefined) { yield* until(directoryAcquisition); } + // Whether the directory is still the one this owner made, asked here + // and nowhere later. Closing the listener below unlinks its socket — + // Node's own documented behaviour for a Unix socket it bound — and + // taking a child out of a directory changes the directory. Asked + // after that, the question is about a directory this teardown has + // already modified; asked here, it is still a fair one. Every join + // below still happens before the directory is removed. + let ours = false; + // Kept, not raised here. Failing to read the directory says nothing + // about the listener, the peers or the two names this owner knows it + // published, and every one of those is still this cleanup's to + // finish. So the failure waits below, where it is what this teardown + // ends with — and `ours` stays false, because a directory this owner + // could not identify is not one it may remove. + let unidentified: unknown; + if (directory !== undefined && directoryIdentity !== undefined) { + const owned = directoryIdentity; + try { + ours = same(owned, yield* until(readDirectoryIdentity(directory))); + } catch (error) { + if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) { + unidentified = error; + } + } + } + const failures: unknown[] = []; + if (unidentified !== undefined) { + failures.push(unidentified); + } + // Each of these is this owner's, and none of them is any of the + // others: a peer that will not close says nothing about a name that + // can still be unlinked, and a descriptor that will not go says + // nothing about the staged name beside it. So a stage that fails is + // kept and the next one is still attempted, in the order they + // happened — and what this teardown ends with is all of it rather + // than the first of it. + function* attempt(stage: () => Operation): Operation { + try { + yield* stage(); + } catch (error) { + failures.push(error); + } + } + // Cancelled between opening the file and reading what it is, this + // publication owns a handle and no identity — and a name it may not + // touch without one. The handle itself can still say which node it + // is, which is not the same as asking the name: it is the authority + // being read directly. If even that fails, the failure is kept and + // the names are left alone. + if (pin === undefined && pinnedFile !== undefined) { + const owned = pinnedFile; + yield* attempt(function* () { + const value = yield* until(owned.stat({ bigint: true })); + pin = { handle: owned, dev: value.dev, ino: value.ino }; + }); + } for (const socket of sockets.keys()) { - yield* closeSocket(socket); + yield* attempt(function* () { + yield* closeSocket(socket); + }); } if (server !== undefined) { server.off("connection", connected); const held = server; - yield* action((resolve) => { - held.close(() => resolve()); - return () => {}; + yield* attempt(function* () { + yield* action((resolve) => { + held.close(() => resolve()); + return () => {}; + }); }); } - yield* removeOwned(path, descriptorIdentity); + yield* attempt(function* () { + yield* removePinnedName(path); + }); if (stage !== undefined) { - yield* removeOwned(stage, stageIdentity); + const staged = stage; + yield* attempt(function* () { + yield* removePinnedName(staged); + }); } if (directory !== undefined) { - yield* removeOwned(join(directory, "s"), socketIdentity); - try { - if ( - directoryIdentity !== undefined && - same(directoryIdentity, yield* until(lstat(directory))) - ) { - yield* until(rmdir(directory)); - } - } catch (error) { - if ( - !( - error instanceof Error && - "code" in error && - (error.code === "ENOENT" || error.code === "ENOTEMPTY") - ) - ) { - throw error; + const made = directory; + yield* attempt(function* () { + yield* removeOwned(join(made, "s"), socketIdentity); + }); + yield* attempt(function* () { + try { + if (ours) { + yield* until(rmdir(made)); + } + } catch (error) { + if ( + !( + error instanceof Error && + "code" in error && + (error.code === "ENOENT" || error.code === "ENOTEMPTY") + ) + ) { + throw error; + } } - } + }); + } + // Last, and after every name this publication might still have had + // to remove: the handle is what made those decisions answerable, so + // letting go of it is the final thing released rather than the first. + // A close that fails joins the others in its turn; nothing is + // removed on an unpinned reading afterwards, because there is no + // afterwards. + if (pinnedFile !== undefined) { + const owned = pinnedFile; + yield* attempt(function* () { + yield* until(owned.close()); + }); + } + // One failure leaves as itself, because the caller's own fence may + // be looking for that exact object. More than one travel together, + // in the order they were met; neither stands in for the other. + if (failures.length === 1) { + throw failures[0]; + } + if (failures.length > 1) { + throw new AggregateError( + failures, + "The attachment could not finish removing what it published.", + ); } }); try { @@ -357,12 +628,14 @@ export function* usePosixSidekickTransport( // Cancellation must join creation and identity before removing the owned directory. directoryAcquisition = createDirectory(join(temp, "xsk-")).then((created) => { directory = created; - return lstat(created).then((identity) => { + return readDirectoryIdentity(created).then((identity) => { directoryIdentity = identity; + directoryOwned = identity; return created; }); }); directory = yield* until(directoryAcquisition); + const directoryPath = directory; yield* until(chmod(directory, 0o700)); const socketPath = join(directory, "s"); if (Buffer.byteLength(socketPath) > 100) { @@ -389,20 +662,50 @@ export function* usePosixSidekickTransport( socketIdentity = yield* retain(lstat(socketPath), (value) => { socketIdentity = value; }); + // The directory, read again now that this owner has finished changing + // it. Creating the socket inside it changed its times, and so did the + // `chmod` before that — so a reading taken when it was created is a + // reading of a directory this owner then modified twice. The first + // reading stays until this one replaces it, because a cancellation + // between the two still has to know which directory is ours. + directoryIdentity = yield* refresh( + () => readDirectoryIdentity(directoryPath), + directoryOwned, + "directory", + "the attachment directory", + (value) => { + directoryIdentity = value; + }, + ); stage = join(dirname(path), ".sidekick-" + randomUUID()); const stagingPath = stage; - yield* scoped(function* () { - const file = yield* useFile(stagingPath, "wx", 0o600, (value) => { - stageIdentity = value; - }); - stageIdentity = yield* until(file.stat()); - yield* until(file.writeFile(JSON.stringify(descriptor))); - }); - yield* retain(link(stage, path), () => { - descriptorIdentity = stageIdentity; + // Opened here and held until this publication is over. Every + // question below about which file a name holds is answered against + // this handle, and it stays open through all of them — including + // the ones cleanup asks, where a closed original is exactly what + // would let somebody else's file wear its number. + const holdAcquisition = probes?.pin?.holdAcquisition; + const opening = open(stagingPath, "wx", 0o600); + const opened = yield* retain( + holdAcquisition === undefined + ? opening + : opening.then((value) => holdAcquisition().then(() => value)), + (value) => { + pinnedFile = value; + probes?.pin?.acquired?.(value); + }, + ); + yield* retain(opened.stat({ bigint: true }), (value) => { + pin = { handle: opened, dev: value.dev, ino: value.ino }; }); - yield* removeOwned(stage, stageIdentity); - stageIdentity = undefined; + yield* retain(opened.writeFile(JSON.stringify(descriptor)), () => {}); + yield* retain(link(stagingPath, path), () => {}); + // One node under two names now, and then under one. Each reading is + // checked against the open file rather than against the reading + // before it, so nothing it reports can promote a different file. + yield* pinnedRefresh(stagingPath, "stage", "the staged descriptor"); + yield* removePinnedName(stagingPath); + yield* pinnedRefresh(path, "published", "the published descriptor"); yield* provide({ descriptor, *accept() { diff --git a/packages/cli/tests/repl-sidekick-acp.test.ts b/packages/cli/tests/repl-sidekick-acp.test.ts index 462712afb..fea88b2be 100644 --- a/packages/cli/tests/repl-sidekick-acp.test.ts +++ b/packages/cli/tests/repl-sidekick-acp.test.ts @@ -7,7 +7,17 @@ import { Agent, Execution } from "@executablemd/core"; import { API } from "@executablemd/runtime"; import { createHash } from "node:crypto"; import { createRequire } from "node:module"; -import { chmod, lstat, mkdtemp, readdir, realpath, rmdir, symlink, unlink } from "node:fs/promises"; +import { + chmod, + lstat, + mkdtemp, + readdir, + realpath, + rename, + rmdir, + symlink, + unlink, +} from "node:fs/promises"; import { join, dirname } from "node:path"; import { fileURLToPath } from "node:url"; import process from "node:process"; @@ -15,6 +25,8 @@ import { spawn as spawnChild } from "node:child_process"; import type { ChildProcess } from "node:child_process"; import { createConnection } from "node:net"; import type { Socket } from "node:net"; +import type { BigIntStats, Stats } from "node:fs"; +import type { FileHandle } from "node:fs/promises"; import { once } from "@effectionx/node/events"; import { Writable } from "node:stream"; import { @@ -217,6 +229,132 @@ function* deadline(what: string): Operation { yield* sleep(5000); throw new Error(what); } +/** + * Put a file somebody else owns where this one is, and be sure it is somebody + * else's. + * + * Unlinking the original and writing in its place is not a replacement on + * every filesystem. Where the published name is the inode's last link, Linux + * frees that inode at the unlink and hands its number straight back to the + * next file created — so the "replacement" arrives wearing the identity of the + * file it replaced, and nothing that compares device and inode can tell them + * apart. CI showed exactly that: the publication under test accepted the + * stranger as its own. + * + * So the stranger is written under its own name first, while the original + * still holds the one under test — which is what makes its inode necessarily a + * different one — checked to be different, and then moved onto the name in a + * single step that never leaves the name empty. + */ +function* replaceFile(path: string, contents: string): Operation { + const ours = yield* until(lstat(path)); + const theirs = path + ".theirs"; + yield* writeTextFile(theirs, contents); + const written = yield* until(lstat(theirs)); + expect(written.dev === ours.dev && written.ino === ours.ino).toBe(false); + yield* until(rename(theirs, path)); + return yield* until(lstat(path)); +} +/** + * The one allocator outcome this publication used to be exposed to, described + * rather than waited for. + * + * No host can be asked to hand a new file the number of an old one, so the + * only honest way to put that case in front of the publisher is to say it + * happened. Saying it is only honest where it could have happened: after the + * original's last name is gone *and* its handle is closed. While the handle is + * open the original still exists, its number is not free, and a stranger keeps + * its own -- so this reader reports the truth in that branch and records which + * branch it took, and the fixture asserts the branch rather than trusting it. + */ +function recycledIdentity(file: string) { + let handle: FileHandle | undefined; + let original: Promise | undefined; + let replaced = false; + const observed: { branch: string; links: string }[] = []; + const order: string[] = []; + async function stillOpen(): Promise { + if (handle === undefined) { + return undefined; + } + try { + return await handle.stat({ bigint: true }); + } catch { + return undefined; + } + } + return { + observed, + order, + held: () => handle, + replace() { + replaced = true; + }, + probe: { + acquired(value: FileHandle) { + handle = value; + original = value.stat({ bigint: true }); + }, + async readName(name: string): Promise { + const reading = await lstat(name, { bigint: true }); + order.push("read " + (name === file ? "published" : "staged")); + if (!replaced || name !== file || original === undefined) { + return reading; + } + const ours = await original; + const live = await stillOpen(); + if (live !== undefined) { + observed.push({ branch: "live", links: String(live.nlink) }); + return reading; + } + observed.push({ branch: "recycled", links: "freed" }); + return Object.assign(Object.create(Object.getPrototypeOf(reading)), reading, { + dev: ours.dev, + ino: ours.ino, + atimeMs: ours.atimeMs, + mtimeMs: ours.mtimeMs, + ctimeMs: ours.ctimeMs, + birthtimeMs: ours.birthtimeMs, + atimeNs: ours.atimeNs, + mtimeNs: ours.mtimeNs, + ctimeNs: ours.ctimeNs, + birthtimeNs: ours.birthtimeNs, + }); + }, + }, + }; +} +/** + * Watch -- and when asked, hold or fail -- the publication's own close, from + * the real handle it was handed. + */ +function watchClose( + handle: FileHandle, + order: string[], + options: { hold?: Promise; started?: () => void; fail?: Error } = {}, +) { + const real = handle.close.bind(handle); + let closes = 0; + handle.close = async () => { + closes += 1; + order.push("close"); + options.started?.(); + if (options.hold !== undefined) { + await options.hold; + } + await real(); + if (options.fail !== undefined) { + throw options.fail; + } + }; + return { count: () => closes }; +} +function errorOf(value: unknown): Error { + if (!(value instanceof Error)) { + throw new Error("Expected an Error, and got " + String(value) + "."); + } + return value; +} function* refused(operation: Operation): Operation { let error: unknown; try { @@ -1527,3 +1665,978 @@ describe("ACP startup bootstrap", () => { }); } }); + +function* publishingHome(prefix: string): Operation { + const home = yield* until(realpath(yield* useTemporary(prefix))); + yield* until(chmod(home, 0o700)); + return home; +} +function* stageNameIn(home: string): Operation { + const staged = (yield* until(readdir(home))).filter((name) => name.startsWith(".sidekick-")); + expect(staged).toHaveLength(1); + return join(home, staged[0]); +} +function binding(owner: string) { + return { + owner, + execution: owner + "-execution", + conversation: owner + "-conversation", + }; +} + +describe("ACP81: a replacement inside a refreshed identity window", () => { + it("ACP81: a stage name replaced under its refresh is refused and left where it is", function* () { + const home = yield* publishingHome("xmd-stage-replaced-"); + const file = join(home, "attachment.json"); + const reached = withResolvers(); + // A raw promise, because the hold below is a plain callback the + // publication calls outside this operation: it cannot await an operation, + // and this test has to let go of it from its own `finally`. + let release!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + let directory: string | undefined; + yield* usePosixSidekickTransport( + process.platform, + (prefix) => + mkdtemp(prefix).then((created) => { + directory = created; + return created; + }), + undefined, + { + holdIdentityRefresh: (window) => { + if (window !== "stage") { + return Promise.resolve(); + } + reached.resolve(); + return held; + }, + }, + ); + const host = yield* SidekickTransport.expect(); + let failure: unknown; + let published = false; + let foreign: string | undefined; + let foreignIdentity: Awaited> | undefined; + try { + const owner = yield* spawn(function* () { + try { + yield* host.publish(file, { ...binding("stage-replaced"), cwd: home }); + published = true; + yield* suspend(); + } catch (error) { + failure = error; + } + }); + yield* reached.operation; + foreign = yield* stageNameIn(home); + yield* until(unlink(foreign)); + yield* writeTextFile(foreign, "someone else's file"); + foreignIdentity = yield* until(lstat(foreign)); + release(); + yield* owner; + } finally { + release(); + } + console.log( + JSON.stringify({ + kind: "stage-window-replacement", + refused: failure instanceof Error ? failure.message : null, + published, + }), + ); + expect(published).toBe(false); + expect(failure).toBeInstanceOf(Error); + expect(errorOf(failure).message).toBe( + "the staged descriptor was replaced while it was being published.", + ); + expect(yield* readTextFile(foreign!)).toBe("someone else's file"); + expect((yield* until(lstat(foreign!))).ino).toBe(foreignIdentity!.ino); + expect(yield* until(readdir(home))).toEqual([foreign!.slice(home.length + 1)]); + yield* refused(until(lstat(file))); + yield* refused(until(lstat(directory!))); + }); + + it("ACP81: a published name replaced under its refresh is refused and never unlinked", function* () { + const home = yield* publishingHome("xmd-published-replaced-"); + const file = join(home, "attachment.json"); + const reached = withResolvers(); + let release!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + let directory: string | undefined; + yield* usePosixSidekickTransport( + process.platform, + (prefix) => + mkdtemp(prefix).then((created) => { + directory = created; + return created; + }), + undefined, + { + holdIdentityRefresh: (window) => { + if (window !== "published") { + return Promise.resolve(); + } + reached.resolve(); + return held; + }, + }, + ); + const host = yield* SidekickTransport.expect(); + let failure: unknown; + let published = false; + let foreignIdentity: Awaited> | undefined; + try { + const owner = yield* spawn(function* () { + try { + yield* host.publish(file, { ...binding("published-replaced"), cwd: home }); + published = true; + } catch (error) { + failure = error; + } + }); + yield* reached.operation; + foreignIdentity = yield* replaceFile(file, "someone else's attachment"); + release(); + yield* owner; + } finally { + release(); + } + console.log( + JSON.stringify({ + kind: "published-window-replacement", + refused: failure instanceof Error ? failure.message : null, + published, + }), + ); + expect(published).toBe(false); + expect(failure).toBeInstanceOf(Error); + expect(errorOf(failure).message).toBe( + "the published descriptor was replaced while it was being published.", + ); + expect(yield* readTextFile(file)).toBe("someone else's attachment"); + expect((yield* until(lstat(file))).ino).toBe(foreignIdentity!.ino); + expect(yield* until(readdir(home))).toEqual(["attachment.json"]); + yield* refused(until(lstat(directory!))); + }); + + it("ACP81: cancellation while a refresh is in flight adopts nothing and still joins", function* () { + const home = yield* publishingHome("xmd-refresh-cancelled-"); + const file = join(home, "attachment.json"); + const reached = withResolvers(); + const cancelling = withResolvers(); + let release!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + let directory: string | undefined; + yield* usePosixSidekickTransport( + process.platform, + (prefix) => + mkdtemp(prefix).then((created) => { + directory = created; + return created; + }), + undefined, + { + holdIdentityRefresh: (window) => { + if (window !== "published") { + return Promise.resolve(); + } + reached.resolve(); + return held; + }, + }, + ); + const host = yield* SidekickTransport.expect(); + let published = false; + let joined = false; + let foreignIdentity: Awaited> | undefined; + try { + const owner = yield* spawn(function* () { + yield* host.publish(file, { ...binding("refresh-cancelled"), cwd: home }); + published = true; + yield* suspend(); + }); + yield* reached.operation; + foreignIdentity = yield* replaceFile(file, "someone else's attachment"); + const stopping = yield* spawn(function* () { + cancelling.resolve(); + yield* owner.halt(); + joined = true; + }); + yield* cancelling.operation; + expect(joined).toBe(false); + release(); + yield* stopping; + } finally { + release(); + } + console.log(JSON.stringify({ kind: "refresh-cancellation", joined, published, directory })); + expect(joined).toBe(true); + expect(published).toBe(false); + expect(yield* readTextFile(file)).toBe("someone else's attachment"); + expect((yield* until(lstat(file))).ino).toBe(foreignIdentity!.ino); + expect(yield* until(readdir(home))).toEqual(["attachment.json"]); + yield* refused(until(lstat(directory!))); + }); + + it("ACP81: a directory read that fails at teardown still finishes the cleanup it can", function* () { + const home = yield* publishingHome("xmd-directory-unreadable-"); + const file = join(home, "attachment.json"); + let directory: string | undefined; + let reads = 0; + // The read this teardown takes before anything else, failing the way a + // filesystem fails: not absent, unreadable. + const unreadable = Object.assign(new Error("simulated directory read failure"), { + code: "EIO", + }); + let accepted = 0; + yield* usePosixSidekickTransport( + process.platform, + (prefix) => + mkdtemp(prefix).then((created) => { + directory = created; + return created; + }), + // Counted, not held: this runs in the accepted connection's own cleanup, + // so seeing it once is seeing that cleanup joined. + function* () { + accepted += 1; + }, + { + readDirectoryIdentity: (path) => { + reads += 1; + return reads === 3 ? Promise.reject(unreadable) : lstat(path); + }, + }, + ); + const host = yield* SidekickTransport.expect(); + let socketPath: string | undefined; + let peer: Socket | undefined; + let failure: unknown; + try { + yield* scoped(function* () { + const attachment = yield* host.publish(file, { + ...binding("directory-unreadable"), + cwd: home, + }); + socketPath = attachment.descriptor.socket; + peer = createConnection(socketPath); + yield* once(peer, "connect"); + }); + } catch (error) { + failure = error; + } finally { + peer?.destroy(); + } + console.log( + JSON.stringify({ + kind: "directory-read-failure", + reads, + same: failure === unreadable, + message: failure === undefined ? null : errorOf(failure).message, + accepted, + }), + ); + // The failure this teardown ended with is the one it was handed. + expect(failure).toBe(unreadable); + expect(errorOf(failure).message).toBe("simulated directory read failure"); + expect(reads).toBe(3); + // And everything that did not depend on that question is finished: the + // accepted connection's own close was reached, and the listener closed — + // which is what took its socket out of the directory. + expect(accepted).toBe(1); + yield* refused(until(lstat(file))); + yield* refused(until(lstat(socketPath!))); + expect(yield* until(readdir(home))).toEqual([]); + // The directory it could not identify is still there, which is the + // fail-safe direction: it is never removed on an unconfirmed reading. + expect((yield* until(lstat(directory!))).isDirectory()).toBe(true); + expect(yield* until(readdir(directory!))).toEqual([]); + yield* until(rmdir(directory!)); + }); + + it("ACP81: a second cleanup failure is reported with the first, not instead of it", function* () { + const home = yield* publishingHome("xmd-directory-and-name-"); + const file = join(home, "attachment.json"); + let directory: string | undefined; + let reads = 0; + const unreadable = Object.assign(new Error("simulated directory read failure"), { + code: "EIO", + }); + yield* usePosixSidekickTransport( + process.platform, + (prefix) => + mkdtemp(prefix).then((created) => { + directory = created; + return created; + }), + undefined, + { + readDirectoryIdentity: (path) => { + reads += 1; + return reads === 3 ? Promise.reject(unreadable) : lstat(path); + }, + }, + ); + const host = yield* SidekickTransport.expect(); + let socketPath: string | undefined; + let failure: unknown; + try { + yield* scoped(function* () { + const attachment = yield* host.publish(file, { + ...binding("directory-and-name"), + cwd: home, + }); + socketPath = attachment.descriptor.socket; + // The published name cannot be unlinked from a directory this owner + // may no longer write to, so the stage after the failed reading fails + // as well. + yield* until(chmod(home, 0o500)); + }); + } catch (error) { + failure = error; + } + yield* until(chmod(home, 0o700)); + const carried = failure instanceof AggregateError ? failure.errors : []; + console.log( + JSON.stringify({ + kind: "directory-read-and-removal-failure", + aggregate: failure instanceof AggregateError, + message: failure === undefined ? null : errorOf(failure).message, + carried: carried.map((one: unknown) => errorOf(one).message), + }), + ); + expect(failure).toBeInstanceOf(AggregateError); + expect(errorOf(failure).message).toBe( + "The attachment could not finish removing what it published.", + ); + // The first failure is carried by identity, not described. + expect(carried[0]).toBe(unreadable); + expect(carried).toHaveLength(2); + expect(errorOf(carried[1]).message).toContain("attachment.json"); + // The listener still closed its own socket, and the name it could not + // remove is still there to be seen rather than silently forgotten. + yield* refused(until(lstat(socketPath!))); + expect((yield* until(lstat(file))).isFile()).toBe(true); + expect((yield* until(lstat(directory!))).isDirectory()).toBe(true); + yield* until(unlink(file)); + yield* until(rmdir(directory!)); + }); + + it("ACP81: a publication cancelled part-way still tries both of the names it owns", function* () { + const home = yield* publishingHome("xmd-partial-publication-"); + const file = join(home, "attachment.json"); + const reached = withResolvers(); + const cancelling = withResolvers(); + let release!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + let directory: string | undefined; + let reads = 0; + const unreadable = Object.assign(new Error("simulated directory read failure"), { + code: "EIO", + }); + yield* usePosixSidekickTransport( + process.platform, + (prefix) => + mkdtemp(prefix).then((created) => { + directory = created; + return created; + }), + undefined, + { + holdIdentityRefresh: (window) => { + if (window !== "stage") { + return Promise.resolve(); + } + reached.resolve(); + return held; + }, + readDirectoryIdentity: (path) => { + reads += 1; + return reads === 3 ? Promise.reject(unreadable) : lstat(path); + }, + }, + ); + const host = yield* SidekickTransport.expect(); + let published = false; + let joined = false; + let failure: unknown; + let stagePath: string | undefined; + // Established before the mode below is narrowed, so an assertion that + // throws still leaves this directory writable for its own removal. + yield* ensure(function* () { + release(); + yield* until(chmod(home, 0o700)); + }); + { + const owner = yield* spawn(function* () { + try { + yield* host.publish(file, { ...binding("partial-publication"), cwd: home }); + published = true; + yield* suspend(); + } catch (error) { + failure = error; + } + }); + yield* reached.operation; + // Held here, the publication owns two names for one inode: the staged + // file and the link it has just published. + stagePath = yield* stageNameIn(home); + expect((yield* until(lstat(file))).ino).toBe((yield* until(lstat(stagePath))).ino); + expect(reads).toBe(2); + // Neither of them can be unlinked from a directory this owner may no + // longer write to. + yield* until(chmod(home, 0o500)); + const stopping = yield* spawn(function* () { + cancelling.resolve(); + try { + yield* owner.halt(); + } catch (error) { + failure = error; + } + joined = true; + }); + yield* cancelling.operation; + expect(joined).toBe(false); + release(); + yield* stopping; + } + yield* until(chmod(home, 0o700)); + const carried = failure instanceof AggregateError ? failure.errors : []; + console.log( + JSON.stringify({ + kind: "partial-publication-cleanup", + reads, + joined, + published, + aggregate: failure instanceof AggregateError, + carried: carried.map((one: unknown) => errorOf(one).message), + }), + ); + expect(joined).toBe(true); + expect(published).toBe(false); + expect(reads).toBe(3); + expect(failure).toBeInstanceOf(AggregateError); + expect(errorOf(failure).message).toBe( + "The attachment could not finish removing what it published.", + ); + // Both removals were attempted, and all three failures came back: the + // reading, and then each name in the order it was tried. + expect(carried).toHaveLength(3); + expect(carried[0]).toBe(unreadable); + expect(errorOf(carried[1]).message).toContain("attachment.json"); + expect(errorOf(carried[2]).message).toContain(".sidekick-"); + expect((yield* until(lstat(file))).isFile()).toBe(true); + expect((yield* until(lstat(stagePath!))).isFile()).toBe(true); + yield* until(unlink(file)); + yield* until(unlink(stagePath!)); + expect(yield* until(readdir(home))).toEqual([]); + // Still fail-safe about the directory it could not identify. + expect((yield* until(lstat(directory!))).isDirectory()).toBe(true); + yield* until(rmdir(directory!)); + }); + + it("ACP81: the directory is judged while its socket is still in it, and still removed", function* () { + const home = yield* publishingHome("xmd-ctime-directory-"); + const file = join(home, "attachment.json"); + let directory: string | undefined; + // Whether the socket was still a child of the directory at each reading of + // the directory's own identity. A reading taken after the listener close + // has unlinked it is a reading of a directory this teardown has already + // changed, which is what the identity substitute below cannot survive. + const socketPresent: boolean[] = []; + yield* usePosixSidekickTransport( + process.platform, + (prefix) => + mkdtemp(prefix).then((created) => { + directory = created; + return created; + }), + undefined, + { + // The documented libuv fallback, which this filesystem will not do: + // no creation time to read, so `birthtime` is the `ctime`. Deno's own + // `ctime` is truncated to the second, so the leak this would cause is + // not observable here — the ordering above it is, and that is what the + // sequence below pins. + readDirectoryIdentity: (path) => { + const asked = directory; + const observed = + asked === undefined + ? Promise.resolve() + : lstat(join(asked, "s")).then( + () => { + socketPresent.push(true); + }, + () => { + socketPresent.push(false); + }, + ); + return observed + .then(() => lstat(path)) + .then((stat) => + Object.assign(Object.create(Object.getPrototypeOf(stat)), stat, { + birthtimeMs: stat.ctimeMs, + birthtime: new Date(stat.ctimeMs), + }), + ); + }, + }, + ); + const host = yield* SidekickTransport.expect(); + let socketPath: string | undefined; + yield* scoped(function* () { + const attachment = yield* host.publish(file, { + ...binding("ctime-directory"), + cwd: home, + }); + socketPath = attachment.descriptor.socket; + expect(dirname(socketPath)).toBe(directory); + expect((yield* until(lstat(socketPath))).isSocket()).toBe(true); + expect(socketPresent).toEqual([false, true]); + }); + console.log(JSON.stringify({ kind: "ctime-substitute-directory", directory, socketPresent })); + // Created (no socket yet), refreshed once the socket existed, and judged at + // teardown while it still did. + expect(socketPresent).toEqual([false, true, true]); + yield* refused(until(lstat(file))); + yield* refused(until(lstat(socketPath!))); + yield* refused(until(lstat(directory!))); + expect(yield* until(readdir(home))).toEqual([]); + }); +}); + +describe("ACP82: the file this publication keeps open", () => { + it("ACP82: a stranger wearing the original's number is refused while the original is open", function* () { + const home = yield* publishingHome("xmd-recycled-live-"); + const file = join(home, "attachment.json"); + const collision = recycledIdentity(file); + const reached = withResolvers(); + let release!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + let directory: string | undefined; + yield* usePosixSidekickTransport( + process.platform, + (prefix) => + mkdtemp(prefix).then((created) => { + directory = created; + return created; + }), + undefined, + { + holdIdentityRefresh: (window) => { + if (window !== "published") { + return Promise.resolve(); + } + reached.resolve(); + return held; + }, + pin: collision.probe, + }, + ); + const host = yield* SidekickTransport.expect(); + let failure: unknown; + let published = false; + let stranger: BigIntStats | undefined; + try { + const owner = yield* spawn(function* () { + try { + yield* host.publish(file, { ...binding("recycled-live"), cwd: home }); + published = true; + } catch (error) { + failure = error; + } + }); + yield* reached.operation; + // The original's last name goes -- the staged one was already removed -- + // and a real file of somebody else's, with somebody else's bytes, takes + // it. From here the reader is asked to describe the number as recycled. + yield* until(unlink(file)); + yield* writeTextFile(file, "someone else's attachment"); + stranger = yield* until(lstat(file, { bigint: true })); + collision.replace(); + release(); + yield* owner; + } finally { + release(); + } + const handle = collision.held(); + if (handle === undefined) { + throw new Error("The publication never opened a file of its own."); + } + console.log( + JSON.stringify({ + kind: "recycled-identity-live", + observed: collision.observed, + published, + refused: failure instanceof Error ? failure.message : null, + }), + ); + // The premise the model rests on: the original was still open and had no + // names left, so its number was not free and the stranger kept its own. + expect(collision.observed.length).toBeGreaterThan(0); + expect(collision.observed.every((one) => one.branch === "live")).toBe(true); + expect(collision.observed.every((one) => one.links === "0")).toBe(true); + expect(published).toBe(false); + expect(errorOf(failure).message).toBe( + "the published descriptor was replaced while it was being published.", + ); + expect(yield* readTextFile(file)).toBe("someone else's attachment"); + expect((yield* until(lstat(file, { bigint: true }))).ino).toBe(stranger!.ino); + expect(yield* until(readdir(home))).toEqual(["attachment.json"]); + yield* refused(until(lstat(directory!))); + // And the handle this all rested on is let go of at the end, not before. + expect(collision.order[collision.order.length - 1]).toBe("read published"); + yield* refused(until(handle.stat())); + }); + + it("ACP82: cancelled in that window, the close waits for every name decision", function* () { + const home = yield* publishingHome("xmd-recycled-cancelled-"); + const file = join(home, "attachment.json"); + const collision = recycledIdentity(file); + const reached = withResolvers(); + const cancelling = withResolvers(); + const closeStarted = withResolvers(); + let release!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + let releaseClose!: () => void; + const closeHeld = new Promise((resolve) => { + releaseClose = resolve; + }); + let directory: string | undefined; + yield* usePosixSidekickTransport( + process.platform, + (prefix) => + mkdtemp(prefix).then((created) => { + directory = created; + return created; + }), + undefined, + { + holdIdentityRefresh: (window) => { + if (window !== "published") { + return Promise.resolve(); + } + reached.resolve(); + return held; + }, + pin: collision.probe, + }, + ); + const host = yield* SidekickTransport.expect(); + let published = false; + let joined = false; + let stranger: BigIntStats | undefined; + yield* ensure(function* () { + release(); + releaseClose(); + }); + const owner = yield* spawn(function* () { + yield* host.publish(file, { ...binding("recycled-cancelled"), cwd: home }); + published = true; + yield* suspend(); + }); + yield* reached.operation; + const handle = collision.held(); + if (handle === undefined) { + throw new Error("The publication never opened a file of its own."); + } + const closes = watchClose(handle, collision.order, { + hold: closeHeld, + started: () => closeStarted.resolve(), + }); + yield* until(unlink(file)); + yield* writeTextFile(file, "someone else's attachment"); + stranger = yield* until(lstat(file, { bigint: true })); + collision.replace(); + const stopping = yield* spawn(function* () { + cancelling.resolve(); + yield* owner.halt(); + joined = true; + }); + yield* cancelling.operation; + expect(joined).toBe(false); + release(); + // Deterministic: cleanup has reached its last stage, and is waiting there. + yield* closeStarted.operation; + expect(joined).toBe(false); + expect(yield* readTextFile(file)).toBe("someone else's attachment"); + expect(collision.order.filter((one) => one === "close")).toHaveLength(1); + expect(collision.order[collision.order.length - 1]).toBe("close"); + expect(collision.order.filter((one) => one.startsWith("read")).length).toBeGreaterThan(0); + releaseClose(); + yield* stopping; + console.log( + JSON.stringify({ + kind: "recycled-identity-cancelled", + observed: collision.observed, + order: collision.order, + joined, + published, + closes: closes.count(), + }), + ); + expect(joined).toBe(true); + expect(published).toBe(false); + expect(closes.count()).toBe(1); + expect(collision.observed.length).toBeGreaterThan(0); + expect(collision.observed.every((one) => one.branch === "live")).toBe(true); + expect(yield* readTextFile(file)).toBe("someone else's attachment"); + expect((yield* until(lstat(file, { bigint: true }))).ino).toBe(stranger!.ino); + expect(yield* until(readdir(home))).toEqual(["attachment.json"]); + yield* refused(until(lstat(directory!))); + yield* refused(until(handle.stat())); + }); + + it("ACP82: timestamps that all read the same refuse nothing on this publication's own file", function* () { + const home = yield* publishingHome("xmd-coarsened-self-"); + const file = join(home, "attachment.json"); + let reads = 0; + let directory: string | undefined; + yield* usePosixSidekickTransport( + process.platform, + (prefix) => + mkdtemp(prefix).then((created) => { + directory = created; + return created; + }), + undefined, + { + // Every time this publication reads one of its own names, every time + // on it is the same time: the write, the link and the unlink leave no + // trace a reading can see. The node is what it has, and the node has + // not moved. + pin: { + readName: (name) => + lstat(name, { bigint: true }).then((reading) => { + reads += 1; + return Object.assign(Object.create(Object.getPrototypeOf(reading)), reading, { + atimeMs: 0, + mtimeMs: 0, + ctimeMs: 0, + birthtimeMs: 0, + atimeNs: 0n, + mtimeNs: 0n, + ctimeNs: 0n, + birthtimeNs: 0n, + }); + }), + }, + }, + ); + const host = yield* SidekickTransport.expect(); + let descriptor: z.infer | undefined; + yield* scoped(function* () { + const attachment = yield* host.publish(file, { + ...binding("coarsened-self"), + cwd: home, + }); + descriptor = attachment.descriptor; + expect((yield* until(lstat(file))).isFile()).toBe(true); + expect(JSON.parse(yield* readTextFile(file)).socket).toBe(descriptor.socket); + }); + console.log(JSON.stringify({ kind: "coarsened-self-refresh", reads })); + expect(reads).toBeGreaterThan(0); + yield* refused(until(lstat(file))); + yield* refused(until(lstat(directory!))); + expect(yield* until(readdir(home))).toEqual([]); + }); + + it("ACP82: an open delivered after cancellation is still owned, cleaned up and closed", function* () { + const home = yield* publishingHome("xmd-held-open-"); + const file = join(home, "attachment.json"); + const opening = withResolvers(); + const cancelling = withResolvers(); + let releaseOpen!: () => void; + const openHeld = new Promise((resolve) => { + releaseOpen = resolve; + }); + let handle: FileHandle | undefined; + let directory: string | undefined; + yield* ensure(function* () { + releaseOpen(); + }); + yield* usePosixSidekickTransport( + process.platform, + (prefix) => + mkdtemp(prefix).then((created) => { + directory = created; + return created; + }), + undefined, + { + pin: { + acquired: (value) => { + handle = value; + }, + holdAcquisition: () => { + opening.resolve(); + return openHeld; + }, + }, + }, + ); + const host = yield* SidekickTransport.expect(); + let published = false; + let joined = false; + const owner = yield* spawn(function* () { + yield* host.publish(file, { ...binding("held-open"), cwd: home }); + published = true; + yield* suspend(); + }); + // The file exists and is open, and nothing owns it yet. + yield* opening.operation; + expect(handle).toBeUndefined(); + const stage = yield* stageNameIn(home); + const stagedNode = yield* until(lstat(stage, { bigint: true })); + const stopping = yield* spawn(function* () { + cancelling.resolve(); + yield* owner.halt(); + joined = true; + }); + yield* cancelling.operation; + expect(joined).toBe(false); + releaseOpen(); + yield* stopping; + console.log( + JSON.stringify({ + kind: "held-open-cancellation", + joined, + published, + owned: handle !== undefined, + }), + ); + expect(joined).toBe(true); + expect(published).toBe(false); + if (handle === undefined) { + throw new Error("A completed open was never adopted."); + } + // Adopted after the cancellation, identified from the handle rather than + // from the name, removed, and closed. + expect(stagedNode.ino).toBeGreaterThan(0n); + expect(yield* until(readdir(home))).toEqual([]); + yield* refused(until(lstat(directory!))); + yield* refused(until(handle.stat())); + }); + + it("ACP82: a close that fails joins the other failures in its turn", function* () { + const home = yield* publishingHome("xmd-close-failed-"); + const file = join(home, "attachment.json"); + const reached = withResolvers(); + const cancelling = withResolvers(); + let release!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + let directory: string | undefined; + let reads = 0; + let handle: FileHandle | undefined; + const order: string[] = []; + const unreadable = Object.assign(new Error("simulated directory read failure"), { + code: "EIO", + }); + const unclosable = new Error("simulated close failure"); + yield* ensure(function* () { + release(); + yield* until(chmod(home, 0o700)); + }); + yield* usePosixSidekickTransport( + process.platform, + (prefix) => + mkdtemp(prefix).then((created) => { + directory = created; + return created; + }), + undefined, + { + holdIdentityRefresh: (window) => { + if (window !== "stage") { + return Promise.resolve(); + } + reached.resolve(); + return held; + }, + readDirectoryIdentity: (path) => { + reads += 1; + return reads === 3 ? Promise.reject(unreadable) : lstat(path); + }, + pin: { + acquired: (value) => { + handle = value; + }, + }, + }, + ); + const host = yield* SidekickTransport.expect(); + let published = false; + let joined = false; + let failure: unknown; + const owner = yield* spawn(function* () { + try { + yield* host.publish(file, { ...binding("close-failed"), cwd: home }); + published = true; + yield* suspend(); + } catch (error) { + failure = error; + } + }); + yield* reached.operation; + if (handle === undefined) { + throw new Error("The publication never opened a file of its own."); + } + watchClose(handle, order, { fail: unclosable }); + const stagePath = yield* stageNameIn(home); + yield* until(chmod(home, 0o500)); + const stopping = yield* spawn(function* () { + cancelling.resolve(); + try { + yield* owner.halt(); + } catch (error) { + failure = error; + } + joined = true; + }); + yield* cancelling.operation; + expect(joined).toBe(false); + release(); + yield* stopping; + yield* until(chmod(home, 0o700)); + const carried = failure instanceof AggregateError ? failure.errors : []; + console.log( + JSON.stringify({ + kind: "close-failure-order", + joined, + published, + order, + carried: carried.map((one: unknown) => errorOf(one).message), + }), + ); + expect(joined).toBe(true); + expect(published).toBe(false); + expect(failure).toBeInstanceOf(AggregateError); + expect(carried).toHaveLength(4); + expect(carried[0]).toBe(unreadable); + expect(errorOf(carried[1]).message).toContain("attachment.json"); + expect(errorOf(carried[2]).message).toContain(".sidekick-"); + // Last, and kept rather than dropped because earlier stages already failed. + expect(carried[3]).toBe(unclosable); + expect(order).toEqual(["close"]); + expect((yield* until(lstat(file))).isFile()).toBe(true); + expect((yield* until(lstat(stagePath))).isFile()).toBe(true); + yield* until(unlink(file)); + yield* until(unlink(stagePath)); + yield* until(rmdir(directory!)); + }); +}); diff --git a/packages/cli/tests/session-ownership-cli.test.ts b/packages/cli/tests/session-ownership-cli.test.ts index ca9372cb8..4ca72aec4 100644 --- a/packages/cli/tests/session-ownership-cli.test.ts +++ b/packages/cli/tests/session-ownership-cli.test.ts @@ -325,7 +325,11 @@ describe("Session ownership command", () => { ).join(); expect(plugin.code).toBe(1); expect((yield* until(readdir(work))).sort()).toEqual(["poison.cjs", "poison.ts", "session"]); - expect(yield* until(readdir(join(home, ".acpx")))).toEqual([ + // Sorted, like the directory above it: what this asserts is exactly which + // names are there, and a directory does not promise the order it hands + // them back in. Measured, one runner enumerated these two the other way + // round and the membership was the membership either way. + expect((yield* until(readdir(join(home, ".acpx")))).sort()).toEqual([ "config.json", "xmd-native-sessions", ]); diff --git a/packages/core/src/execute.ts b/packages/core/src/execute.ts index b1a301532..9b935b005 100644 --- a/packages/core/src/execute.ts +++ b/packages/core/src/execute.ts @@ -33,7 +33,7 @@ import { cwd as processCwd } from "@effectionx/fs"; import { useComponentModules } from "./typescript-module.ts"; import type { LoadComponentModule } from "./typescript-module.ts"; import { pathToFileURL } from "node:url"; -import type { Workflow, Json } from "@executablemd/durable-streams"; +import type { DurableRootControl, Workflow, Json } from "@executablemd/durable-streams"; import { createReplayStream } from "./replay-stream.ts"; import { ExecutionProtocolError, issueExecution } from "./execution-request.ts"; import { DocumentProtocolError, issueDocument } from "./document-request.ts"; @@ -1015,6 +1015,16 @@ const NO_ROOT_VOCABULARY = "this execution minted no canonical Syntax implementation, so a generated root has no " + "vocabulary to describe"; +/** + * What a document that was stopped completes as. + * + * Not a fault the document raised, and not a success: a run somebody ended is a + * run that produced no outcome of its own, and this is what a caller reads + * where there is nothing else to read. A run that had already acknowledged a + * terminal keeps that terminal and never reaches this. + */ +const DOCUMENT_STOPPED = "this document execution was stopped"; + /** * What a recorded event turned out to be. * @@ -2576,6 +2586,29 @@ export interface HostExecution extends DocumentExecution { * the data nor the matched text. */ bindings(): Operation>; + /** + * End this execution explicitly, and wait for all of it. + * + * For the one caller that owns the run: a host that decided this document + * should stop. It asks this execution's own root to end as cancelled while + * the root is still open, halts the document runner so the root's own + * finalizer writes that ending through the journal it still owns, joins it, + * closes the output with whatever had actually been rendered, and then waits + * for this invocation's installations to unwind. + * + * `Ok` means the stop and the join completed. It never means the document + * succeeded: a stopped document is not `Ok` to `yield* execution`, and a + * document that had already acknowledged a terminal of its own keeps it. A + * failed append or a cleanup that could not finish is `Err` here and is this + * execution's completion too, so a caller cannot read a successful stop + * around one. A document that merely failed on its own is still a successful + * stop — the failure is its completion's to carry. + * + * It writes no record itself and is handed no stream, no append and no event + * to write one with. What it holds is the authority to ask the root that owns + * them. + */ + stop(): Operation>; } /** @@ -2922,6 +2955,39 @@ function* executeDocument( const channel = createReplayStream(); const { operation, resolve } = withResolvers>(); + // What this invocation owns about its own run, held out here rather than + // inside the runner task, because the stop below is a *second* frame that has + // to read and finish the same things: the text that actually cleared the live + // scanner, the one settlement this completion ever takes, and the one close + // its consumers ever see. + let emittedText = ""; + let settled = false; + /** Settle this completion once. A terminal already acknowledged wins. */ + const settle = (result: Result): void => { + if (!settled) { + settled = true; + resolve(result); + } + }; + let closed = false; + /** Close the output once, with whatever had actually been rendered. */ + function* closeOutput(text: string): Operation { + if (closed) { + return; + } + closed = true; + yield* channel.close(text); + } + /** + * This root's own authority to end as cancelled, while it is still open. + * + * Minted by `durableRun` over its own ctx, replay index and terminal phase, + * and handed here by value: it carries no stream, no append and no event, so + * what this invocation holds is the authority to *ask* and to read back what + * actually settled. + */ + let rootControl: DurableRootControl | undefined; + // Created before the run it belongs to, because the handle is published // before the document workflow has installed anything: a trusted host holds // the operation immediately, and what that operation can do begins when the @@ -2935,9 +3001,12 @@ function* executeDocument( // require having activated its runner first. yield* roots.attend(); - yield* spawn(function* () { + // Retained, because stopping this document is halting *this* task and + // joining it: the root's own cancellation finalizer runs inside it, while the + // canonical journal and its policy are still owned. Halting anything above it + // would take that journal down before the close could be written. + const runner = yield* spawn(function* () { let emitted = false; - let emittedText = ""; let restoring = false; // The ENTIRE setup and workflow sit inside one error boundary: once the @@ -3337,6 +3406,9 @@ function* executeDocument( }, { stream: guardedJournal(journal, root, ROOT_COROUTINE, admissions), + control: (own) => { + rootControl = own; + }, }, ); // Taken rather than read, so the handoff belongs to the run that made it. @@ -3358,25 +3430,93 @@ function* executeDocument( restoring = false; } - yield* channel.close(result.output); + yield* closeOutput(result.output); if (result.status === "err") { const failure = failureError(result.error, live); - resolve(Err(failure instanceof Error ? failure : new Error(String(failure)))); + settle(Err(failure instanceof Error ? failure : new Error(String(failure)))); return; } // The execution's own evaluation policy first, then whatever the host // registered. An authoritative failure one of this execution's root // evaluations raised is this run's outcome: a host that caught the // failure on the call cannot read a successful main result around it. - resolve(settleCompletion(Ok(result.value), [roots.completion, ...completions])); + settle(settleCompletion(Ok(result.value), [roots.completion, ...completions])); } catch (error) { // Close with everything already emitted — printed errors produced before // an abort stay visible to consumers of the close value. - yield* channel.close(emittedText); - resolve(Err(error instanceof Error ? error : new Error(String(error)))); + yield* closeOutput(emittedText); + settle(Err(error instanceof Error ? error : new Error(String(error)))); } }); + /** + * End this document explicitly, and join everything it owns. + * + * Four things in one order, and the order is the whole of it. + * + * The root is **asked** first, while it is still open: a root whose own + * terminal append has already begun keeps the ending it was writing, and + * this says so by being refused. Then the runner task is **halted** — that + * task and not the invocation owner above it, because the root's cancellation + * finalizer appends inside it and the canonical journal, its secret policy + * and its guard are still owned there. Halting returns once that unwinding + * has finished, so what comes back is a joined runner rather than a request + * that one stop. + * + * Then the consumers are finished: the output is closed with the text that + * actually cleared the live scanner — partial, and still only what this run + * rendered — and completion settles. A normal terminal that had already been + * acknowledged wins both, because both are taken once. + * + * `Ok` says the stop and the join completed; it never says the document + * succeeded. A failure the halt raised — a refused append, a cleanup that + * could not finish — is this operation's answer *and* this document's + * completion, so a caller cannot read a successful stop around it. + */ + function* stopDocument(): Operation> { + // The root has not been created yet: this run is still starting, or it + // failed before reaching its workflow. There is no terminal to lose, so + // halting the runner is the whole of ending it. + if (rootControl === undefined) { + return yield* haltRunner(); + } + if (!rootControl.requestInterruption()) { + // Refused, which means this root's own terminal is being written or has + // already been acknowledged. That terminal is the one it ends with. So + // nothing is halted, nothing is closed here and nothing is settled here: + // the run finishes writing, publishes its own result and output, and this + // waits for it. Halting now would cancel an append over a backend that + // may already hold it, and a cancelled ending written around it would be + // this run recording two. + yield* operation; + return Ok(undefined); + } + return yield* haltRunner(); + } + + /** + * End the runner and finish the consumers, for the two endings that halt it. + * + * The accepted interruption, where the root's own cancellation finalizer + * appends inside this task; and the startup that has no root yet. Both leave + * nothing acknowledged behind them, so the output is closed with what had + * actually been rendered and completion settles as stopped — unless the run + * settled something of its own first, in which case both are already taken. + */ + function* haltRunner(): Operation> { + try { + yield* runner.halt(); + } catch (error) { + const failure = error instanceof Error ? error : new Error(String(error)); + yield* closeOutput(emittedText); + settle(Err(failure)); + return Err(failure); + } + yield* closeOutput(emittedText); + settle(Err(new HostExecutionError(DOCUMENT_STOPPED))); + return Ok(undefined); + } + return { execution: { *[Symbol.iterator]() { @@ -3387,6 +3527,7 @@ function* executeDocument( evaluateGeneratedXmdRoot: roots.evaluate, evaluation: roots.completion, computed, + stop: stopDocument, }; } @@ -3415,6 +3556,15 @@ interface CanonicalInvocation { * document always ends — has nothing of its own for it to be additive to. */ readonly evaluation: CompletionFailure; + /** + * End this document explicitly, and join it. + * + * Private to canonical execution and published only to a trusted host. It is + * bound over this invocation's own runner task and root writer state, so + * there is nothing to look up and nothing to name: a caller that does not + * already hold this invocation cannot reach it. + */ + readonly stop: () => Operation>; } /** @@ -3663,6 +3813,16 @@ function* runInvocation( // window in which no evaluation can have been asked for. let evaluation: CompletionFailure = () => undefined; + /** + * What this invocation's own teardown raised, from the frame that raised it. + * + * Kept beside the completion rather than read back out of it: a stop is a + * success only when the cleanup after it finished, and "did cleanup fail" is + * not a question the reconciled completion's error text can be asked. A + * document that failed on its own is a different thing and leaves this unset. + */ + let teardownFailure: Error | undefined; + const owner = yield* spawn(function* () { // Whatever ends this task — completion, failure, or a cancelled handle — // leaves no observer waiting on a run that is over. A document that already @@ -3697,6 +3857,7 @@ function* runInvocation( }); } catch (error) { const teardown = error instanceof Error ? error : new Error(String(error)); + teardownFailure = teardown; if (!published) { // No handle exists, so this throws — but a durability or Files failure // raised during cleanup is still the failure that gets reported. @@ -3710,9 +3871,60 @@ function* runInvocation( }); const invoked = yield* ready.operation; + /** + * End this invocation explicitly, and wait for all of it. + * + * The canonical stop first: it asks the root, halts the document runner and + * joins it, so the cancelled close is written while the journal is still + * owned. Then this invocation's own completion, which is what the owner + * resolves *after* its installations have unwound — so a caller whose stop + * has returned knows the whole of it is over, not merely the document part. + * + * What the two say is ranked rather than merged. A document that failed on + * its own is not this operation's failure: the stop still happened, and the + * completion carries the document's outcome for whoever reads it. A failed + * append or a cleanup that could not finish is different — it comes from the + * frame that raised it rather than from the completion's text, and it is this + * operation's answer, because a stop whose cleanup failed stopped nothing + * safely. + */ + function* stopInvocation(): Operation> { + const stopped = yield* invoked.stop(); + // The whole of it, joined: this resolves once the installations have + // unwound and the completion has been reconciled, so what is ranked below + // is what this invocation actually ended as rather than what the document + // part of it reported. + const completion = yield* settled.operation; + const stopFailure = stopped.ok ? undefined : stopped.error; + const documentFailure = completion.ok ? undefined : completion.error; + + // Kind first, then occurrence within that kind, by identity — the same + // ranking the completion was reconciled by, extended by this stop's own + // report, which happened last. A durability or Files failure is the run's + // whichever layer raised it, so it can neither be reported as a successful + // stop nor lose to an ordinary cleanup error that happened after it. + const durable = fromEach(durabilityFailure, documentFailure, teardownFailure, stopFailure); + if (durable !== undefined) { + return Err(durable); + } + const files = fromEach(filesFatalFailure, documentFailure, teardownFailure, stopFailure); + if (files !== undefined) { + return Err(files); + } + // Below the fatal kinds, what this operation answers for is the stop and + // the join, not the document. A document that failed on its own stopped + // cleanly and that failure is its completion's to carry; a cleanup that + // could not finish did not stop cleanly, and neither did a refused append. + if (stopFailure !== undefined) { + return Err(stopFailure); + } + return teardownFailure === undefined ? Ok(undefined) : Err(teardownFailure); + } + return { evaluateGeneratedXmdRoot: invoked.evaluateGeneratedXmdRoot, computed: invoked.computed, + stop: stopInvocation, // Carried through unchanged. The completion this handle publishes has // already been reconciled with it, so nothing above reads it again; it // stays on the handle because it is this invocation's policy and a handle @@ -3762,15 +3974,11 @@ function reconcileExpansion( const canonical = settlement.status === "raised" ? settlement.raised : undefined; const raised = policy?.raised; - const durable = - (canonical === undefined ? undefined : durabilityFailure(canonical)) ?? - (raised === undefined ? undefined : durabilityFailure(raised)); + const durable = fromEach(durabilityFailure, canonical, raised); if (durable !== undefined) { throw durable; } - const files = - (canonical === undefined ? undefined : filesFatalFailure(canonical)) ?? - (raised === undefined ? undefined : filesFatalFailure(raised)); + const files = fromEach(filesFatalFailure, canonical, raised); if (files !== undefined) { throw files; } @@ -3798,6 +4006,29 @@ function fatalOf(error: unknown): Error | undefined { return durabilityFailure(error) ?? filesFatalFailure(error); } +/** + * The first failure of one kind among these, in the order they happened. + * + * The order of the arguments is the order of occurrence, which is what decides + * within a kind. What comes back is the object the classifier found, never a + * rebuilt one, because the engine's fences match on identity. + */ +function fromEach( + classify: (error: unknown) => T | undefined, + ...failures: readonly unknown[] +): T | undefined { + for (const failure of failures) { + if (failure === undefined) { + continue; + } + const found = classify(failure); + if (found !== undefined) { + return found; + } + } + return undefined; +} + /** * The one result of a document whose invocation also failed to tear down, or * whose own root evaluations did. @@ -3827,17 +4058,11 @@ function reconcile( // Kind first, then occurrence *within* that kind — by identity, because the // engine's fences match the exact object rather than a rebuilt one. - const durable = - (evaluation === undefined ? undefined : durabilityFailure(evaluation)) ?? - (failed === undefined ? undefined : durabilityFailure(failed)) ?? - (teardown === undefined ? undefined : durabilityFailure(teardown)); + const durable = fromEach(durabilityFailure, evaluation, failed, teardown); if (durable !== undefined) { return Err(durable); } - const files = - (evaluation === undefined ? undefined : filesFatalFailure(evaluation)) ?? - (failed === undefined ? undefined : filesFatalFailure(failed)) ?? - (teardown === undefined ? undefined : filesFatalFailure(teardown)); + const files = fromEach(filesFatalFailure, evaluation, failed, teardown); if (files !== undefined) { return Err(files); } @@ -4435,6 +4660,7 @@ export function* executeInstalled( }, evaluateGeneratedXmdRoot: invoked.evaluateGeneratedXmdRoot, bindings: invoked.computed.read, + stop: invoked.stop, }; } diff --git a/packages/core/tests/execution-protocol.test.ts b/packages/core/tests/execution-protocol.test.ts index e0a6d3afa..9857d2c34 100644 --- a/packages/core/tests/execution-protocol.test.ts +++ b/packages/core/tests/execution-protocol.test.ts @@ -20,6 +20,7 @@ import { ensure, resource, scoped, + sleep, spawn, suspend, until, @@ -1074,6 +1075,396 @@ describe("Tier EP — the execution protocol", () => { } }); + /** + * EP29: what an explicit host Stop answers for, and what it leaves alone. + * + * `HostExecution.stop()` answers for the stop and the join — never for the + * document. These four are the boundaries that distinction is decided at, and + * each is reached by a real gate rather than by timing: a normal terminal + * whose append began first, a durability failure, that same failure beside an + * ordinary cleanup error, and an ordinary document failure on its own. + */ + it("EP29: Stop joins a terminal already being written and keeps fatal identity", function* () { + // A normal root Close whose append is held open. Stop arrives while it is + // in flight, which is the one moment a stop could cancel an ending the + // backend may already hold. + const inner = new InMemoryStream(); + const appending = withResolvers(); + const release = withResolvers(); + let appendCancelled = false; + const held = { + readAll: () => inner.readAll(), + *append(event: DurableEvent): Operation { + if (event.type !== "close") { + yield* inner.append(event); + return; + } + let acknowledged = false; + try { + appending.resolve(); + yield* release.operation; + yield* inner.append(event); + acknowledged = true; + } finally { + appendCancelled = !acknowledged; + } + }, + }; + + yield* scoped(function* () { + const execution = yield* executeInstalled( + { ...inlineSource("# a normal ending\n"), stream: held }, + [], + ); + yield* appending.operation; + const stopped = withResolvers>(); + const asking = withResolvers(); + yield* spawn(function* () { + asking.resolve(); + stopped.resolve(yield* execution.stop()); + }); + yield* asking.operation; + release.resolve(); + + // The ending it was writing is the ending it has. The append was not + // cancelled, the retained close says `ok`, the completion is that same + // success, and the stop is `Ok` because the stop and the join both + // completed — not because the document was stopped. + const answer = yield* stopped.operation; + const completion = yield* execution; + expect(answer.ok).toBe(true); + expect(completion.ok).toBe(true); + expect(appendCancelled).toBe(false); + const closes = inner.snapshot().filter((event) => event.type === "close"); + expect(closes.length).toBe(1); + expect(closes[0]?.type === "close" ? closes[0].result.status : undefined).toBe("ok"); + }); + + // A durability failure, alone and then beside an ordinary cleanup error. + // Both times the run's own failure is the stop's answer, by identity: a + // fatal failure belongs to the run whichever layer raised it, and it cannot + // be reported as a successful stop or lose to a later ordinary error. + for (const alsoFailCleanup of [false, true]) { + const refusing = new InMemoryStream(); + const appendCause = new Error("this journal refused the root close"); + const cleanupCause = new Error("ORDINARY-CLEANUP"); + const stream = { + readAll: () => refusing.readAll(), + *append(event: DurableEvent): Operation { + if (event.type === "close") { + throw appendCause; + } + yield* refusing.append(event); + }, + }; + yield* scoped(function* () { + const execution = yield* executeInstalled( + { ...inlineSource("# a refused ending\n"), stream }, + alsoFailCleanup + ? [ + { + *install(): Operation { + yield* ensure(() => { + throw cleanupCause; + }); + }, + }, + ] + : [], + ); + const completion = yield* execution; + const answer = yield* execution.stop(); + expect(completion.ok).toBe(false); + expect(answer.ok).toBe(false); + const failed = completion.ok ? undefined : completion.error; + const refused = answer.ok ? undefined : answer.error; + expect(failed).toBeInstanceOf(DurablePersistenceError); + expect(refused).toBeInstanceOf(DurablePersistenceError); + // The same object, not a second description of it. + expect(refused).toBe(failed); + expect(String(refused?.message)).not.toContain("ORDINARY-CLEANUP"); + }); + } + + // An ordinary document failure with a clean teardown is still a successful + // stop: the failure is the completion's to carry, and the stop did stop. + yield* scoped(function* () { + const execution = yield* executeInstalled( + { ...inlineSource("\n"), stream: new InMemoryStream() }, + [ + { + *install(): Operation { + yield* registerComponents([ + { + name: "OrdinaryFailure", + origin: "execution-protocol", + props: { type: "object", properties: {}, additionalProperties: false }, + // deno-lint-ignore require-yield + *fn(): Operation { + throw new Error("AUTHORED-FAILURE"); + }, + }, + ]); + }, + }, + ], + ); + const completion = yield* execution; + const answer = yield* execution.stop(); + expect(completion.ok).toBe(false); + expect(String(completion.ok ? "" : completion.error.message)).toContain("AUTHORED-FAILURE"); + expect(answer.ok).toBe(true); + }); + + // And a cleanup that could not finish is not a successful stop, even though + // the document itself ended perfectly well. + yield* scoped(function* () { + const cleanupCause = new Error("CLEANUP-REFUSED"); + const execution = yield* executeInstalled( + { ...inlineSource("# a clean document\n"), stream: new InMemoryStream() }, + [ + { + *install(): Operation { + yield* ensure(() => { + throw cleanupCause; + }); + }, + }, + ], + ); + const answer = yield* execution.stop(); + expect(answer.ok).toBe(false); + expect(answer.ok ? undefined : answer.error).toBe(cleanupCause); + }); + }); + + /** + * EP30: a host Stop that reaches a document still running, through the one + * entrypoint a trusted host has. + * + * No session, no entry and no CLI: `executeInstalled` is the whole of the + * seam, and what is proved is the stop itself. The document really has + * printed something — a prefix this row reads off `execution.output` before + * it asks for anything — and is then waiting on a real operation; its own + * installation holds a cleanup gate, so the Stop cannot answer until that + * teardown has been joined. + * + * What a stop owes that partial text is the discriminating part: the output + * channel closes with exactly what was emitted before the document stopped. + * A stop that threw the prefix away, or that closed the channel empty, + * passes every other assertion here and fails that one. + * + * What the journal keeps is the protocol's existing cancelled Close — one of + * them, carrying a status and nothing else — and the completion is not a + * success. Nothing is invented to stand in for the answer the document never + * gave, and asking again adds neither output nor terminal. + */ + it("EP30: a host Stop reaches a running document, joins its cleanup and retains one cancelled close", function* () { + const stream = new InMemoryStream(); + const reached = withResolvers(); + const waiting = withResolvers(); + const releaseCleanup = withResolvers(); + const order: string[] = []; + /** Body text the document renders before it reaches anything that blocks. */ + const PREFIX = "PARTIAL-OUTPUT-BEFORE-STOP"; + + yield* scoped(function* () { + const execution = yield* executeInstalled( + { ...inlineSource(`${PREFIX}\n\n\n`), stream }, + [ + { + *install(): Operation { + // Registered first, so it unwinds last: the Stop below cannot + // answer until this has been released and joined. + yield* ensure(function* (): Operation { + order.push("cleanup:begin"); + yield* releaseCleanup.operation; + order.push("cleanup:end"); + }); + yield* registerComponents([ + { + name: "Waiting", + origin: "execution-protocol", + props: { type: "object", properties: {}, additionalProperties: false }, + *fn(): Operation { + reached.resolve(); + yield* waiting.operation; + return "unreachable"; + }, + }, + ]); + }, + }, + ], + ); + + // Drained from the moment the execution exists, so what is read is what + // the document emitted rather than whatever was left in a buffer. The + // close value of this channel is the whole of it — full for a document + // that finished, partial for one that did not. + const chunks: string[] = []; + const closed = withResolvers(); + yield* spawn(function* () { + const subscription = yield* execution.output; + let next = yield* subscription.next(); + while (!next.done) { + chunks.push(next.value); + next = yield* subscription.next(); + } + closed.resolve(next.value); + }); + + // The document has printed and is now blocked. Both are waited for, so + // the stop below happens at a known point rather than a timed one. + yield* reached.operation; + for (let round = 0; round < 200 && !chunks.join("").includes(PREFIX); round += 1) { + yield* sleep(1); + } + const observed = chunks.join(""); + expect(observed).toContain(PREFIX); + + const stopped = withResolvers>(); + const asking = withResolvers(); + yield* spawn(function* () { + asking.resolve(); + order.push("stop:asked"); + const answer = yield* execution.stop(); + order.push("stop:answered"); + stopped.resolve(answer); + }); + yield* asking.operation; + + // Held: the cleanup this execution owns has begun and has not finished, + // so the stop has nothing to answer with yet. + for (let round = 0; round < 200 && !order.includes("cleanup:begin"); round += 1) { + yield* sleep(1); + } + if (!order.includes("cleanup:begin")) { + throw new Error("the owned cleanup never began"); + } + expect(order).not.toContain("stop:answered"); + + releaseCleanup.resolve(); + const answer = yield* stopped.operation; + expect(answer.ok).toBe(true); + // Joined before answered, which is the whole of the gate. + expect(order.indexOf("cleanup:end")).toBeLessThan(order.indexOf("stop:answered")); + + // The document did not succeed, and nothing was fabricated for it. + const completion = yield* execution; + expect(completion.ok).toBe(false); + + // And the text it had already printed is exactly what its output channel + // closed with. This is the assertion a stop that dropped the prefix, or + // closed the channel empty, does not survive. + const closure = yield* closed.operation; + expect(closure).toContain(PREFIX); + expect(closure).toBe(observed); + + // One cancelled root Close, carrying a status and nothing else. + const closes = stream.snapshot().filter((event) => event.type === "close"); + const roots = closes.filter( + (event) => event.type === "close" && event.coroutineId === "root", + ); + expect(roots.length).toBe(1); + const only = roots[0]; + if (only === undefined || only.type !== "close") { + throw new Error("this execution recorded no root close"); + } + expect(only.result.status).toBe("cancelled"); + expect(Object.keys(only.result)).toEqual(["status"]); + + // Asked again, and read again: neither writes a second ending, and + // neither adds a byte to what was printed. + const again = yield* execution.stop(); + expect(again.ok).toBe(true); + expect(yield* execution).toEqual(completion); + expect( + stream.snapshot().filter((event) => event.type === "close" && event.coroutineId === "root") + .length, + ).toBe(1); + expect(chunks.join("")).toBe(observed); + expect(yield* closed.operation).toBe(closure); + }); + }); + + /** + * EP31: the journal refuses the cancelled ending that Stop was writing. + * + * The refusal is the run's own failure, by identity, and it stays that + * failure: the stop answers `Err` with it, the cleanup this execution owns is + * still attempted and joined, and nothing compensating is appended in its + * place. A persistence failure is not an ending. + */ + it("EP31: a refused cancelled close keeps its own failure and appends no compensating terminal", function* () { + const inner = new InMemoryStream(); + const appendCause = new Error("this journal refused the cancelled close"); + const reached = withResolvers(); + const waiting = withResolvers(); + const order: string[] = []; + const refusing = { + readAll: () => inner.readAll(), + *append(event: DurableEvent): Operation { + if (event.type === "close" && event.coroutineId === "root") { + order.push("append:refused"); + throw appendCause; + } + yield* inner.append(event); + }, + }; + + yield* scoped(function* () { + const execution = yield* executeInstalled( + { ...inlineSource("\n"), stream: refusing }, + [ + { + *install(): Operation { + yield* ensure(() => { + order.push("cleanup"); + }); + yield* registerComponents([ + { + name: "Waiting", + origin: "execution-protocol", + props: { type: "object", properties: {}, additionalProperties: false }, + *fn(): Operation { + reached.resolve(); + yield* waiting.operation; + return "unreachable"; + }, + }, + ]); + }, + }, + ], + ); + yield* reached.operation; + + const answer = yield* execution.stop(); + const completion = yield* execution; + + // The refusal, not a description of it, and the same one the run carries. + expect(answer.ok).toBe(false); + expect(completion.ok).toBe(false); + const refused = answer.ok ? undefined : answer.error; + const failed = completion.ok ? undefined : completion.error; + expect(refused).toBe(failed); + // The persistence failure this layer raises, carrying the journal's own + // refusal as its cause: the identity that matters is the run's, and the + // reason it gives is the reason the backend gave. + expect(refused).toBeInstanceOf(DurablePersistenceError); + expect(refused?.cause).toBe(appendCause); + + // The cleanup was attempted and joined even though the ending was not + // written, and nothing stood in for the terminal that was refused. + expect(order).toContain("append:refused"); + expect(order).toContain("cleanup"); + expect( + inner.snapshot().filter((event) => event.type === "close" && event.coroutineId === "root"), + ).toEqual([]); + }); + }); + // EP28: an observer belongs to the invocation it watches. Two executions run // concurrently, each with its own callback and its own finalizer; cancelling // one consumer must reach only that invocation. A single shared slot would diff --git a/packages/durable-streams/mod.ts b/packages/durable-streams/mod.ts index 03c313c0f..39231b57c 100644 --- a/packages/durable-streams/mod.ts +++ b/packages/durable-streams/mod.ts @@ -111,4 +111,4 @@ export { ephemeral } from "./ephemeral.ts"; // Entry point export { durableRun } from "./run.ts"; -export type { DurableRunOptions } from "./run.ts"; +export type { DurableRootControl, DurableRunOptions } from "./run.ts"; diff --git a/packages/durable-streams/run.ts b/packages/durable-streams/run.ts index be8f2bb21..1df1a48e6 100644 --- a/packages/durable-streams/run.ts +++ b/packages/durable-streams/run.ts @@ -13,10 +13,14 @@ * See integration doc §10, protocol spec §4. */ -import { scoped, useScope } from "effection"; +import { ensure, scoped, useScope } from "effection"; import type { Operation, Scope } from "effection"; import { DurableContext } from "./context.ts"; -import { activeDurabilityFailure, appendDurableEvent } from "./durability.ts"; +import { + activeDurabilityFailure, + appendDurableEvent, + rememberDurabilityFailure, +} from "./durability.ts"; import { EarlyReturnDivergenceError, TerminalDivergenceError } from "./errors.ts"; import { ReplayGuard } from "./replay-guard.ts"; import { consumable, observeEvent } from "./retained.ts"; @@ -65,6 +69,68 @@ export interface DurableRunOptions { stream: DurableStream; /** Coroutine ID for the root workflow. Defaults to "root". */ coroutineId?: string; + /** + * Hand this run's own root control to its caller, before anything runs. + * + * Absent — which is every ordinary caller — this run's cancellation + * behaviour is exactly what it has always been: a halted root unwinds + * without a terminal, because a generator's return cannot be observed by + * the `catch` below. The control is how one trusted caller asks for the + * other ending, and it is minted over *this* call's own context, replay + * index and terminal state rather than looked up by name. + */ + control?: (control: DurableRootControl) => void; +} + +/** + * This root ending because a Stop owned it, rather than because of a fault. + * + * It is thrown to the caller on the return path by design, and the catch lets + * it past without writing a terminal around it; the cancellation finalizer + * records what actually happened. It exists so an interrupted root stops + * *being* a successful completion without a fault being invented to carry it + * out. + * + * The bypass is keyed to the exact object one run minted, never to this class: + * which root was stopped is not something the type of an error can answer, and + * a root that throws another root's interruption is an ordinary failure. + */ +class RootInterrupted extends Error { + override name = "RootInterrupted"; + + constructor(coroutineId: string) { + super(`Workflow ${coroutineId} was stopped`); + } +} + +/** Where this root stands, and what it will still answer for. */ +type RootPhase = "open" | "claimed" | "appending" | "settled" | "ended"; + +/** + * One run's own authority to end its root as cancelled. + * + * It carries no stream, no append and no event: what a holder may do is ask, + * and read back what actually settled. Whether the ask is accepted is this + * run's decision and nobody else's — a root whose terminal append has already + * begun keeps the terminal it was writing. + */ +export interface DurableRootControl { + /** + * Ask this root to end as cancelled, and say whether it will. + * + * Synchronous and decided here, because the question is whether this root is + * still open: a decision made anywhere else would be made about a terminal + * phase that may have moved while the answer travelled. Repeated asks are + * the same ask. + */ + requestInterruption(): boolean; + /** + * Which terminal this root actually retained, once one is acknowledged. + * + * `undefined` until an append has been acknowledged, so a caller cannot + * mistake an attempt for a record. + */ + settled(): "ok" | "err" | "cancelled" | undefined; } /** @@ -91,7 +157,28 @@ export function* durableRun( workflow: () => Workflow | Operation, options: DurableRunOptions, ): Operation { - const { stream, coroutineId = "root" } = options; + const { stream, coroutineId = "root", control } = options; + // This root's own terminal phase, read and written only here. + // + // `open` is the only phase a cancellation may claim, and claiming is + // exclusive: once a Stop holds the open root, the ordinary paths may no + // longer acknowledge an ok or err terminal as though no claim existed, and + // once an append has begun the record being written is the one this root + // ends with. `ended` is where a root stops answering at all — a request + // after its owner has gone is refused rather than accepted into nothing. + let phase: RootPhase = "open"; + let retained: "ok" | "err" | "cancelled" | undefined; + // This run's own leaving, minted only if this run is actually stopped. + // + // The bypass below is keyed to this exact object rather than to the class: + // an interruption is a fact about *one* invocation, and a root that merely + // happens to throw somebody else's — a stopped root's error carried into a + // later workflow, say — was not itself stopped. Classifying by type would + // let that root leave with no terminal at all, having asked for nothing. + let interruption: RootInterrupted | undefined; + // Read through a call rather than directly: the control below moves this + // while the workflow is suspended, which narrowing by assignment cannot see. + const phaseNow = (): RootPhase => phase; const events = yield* stream.readAll(); const replayIndex = new ReplayIndex(events); @@ -150,6 +237,96 @@ export function* durableRun( } replayIndex.claim(coroutineId); + // Retirement, registered first so it unwinds last: whatever ends this root — + // its own terminal, an ordinary halt, a failure — it stops answering + // requests afterwards. What it retained stays readable; what it will no + // longer do is accept a new claim on an owner that has gone. + yield* ensure(() => { + if (phaseNow() !== "settled") { + phase = "ended"; + } + }); + + // The other ending, and only where it was asked for. Registered after the + // retirement above so it runs before it, and before the workflow so it + // unwinds after everything the workflow owns. Inert unless this root + // accepted a claim while it was still open: an ordinary halt reaches here + // unclaimed and appends nothing, which is what every caller without a + // control has always seen. + /** + * End this root as stopped: the refusals its ordinary terminals make, then + * the one close the protocol already has. + * + * Called from the finalizer when the root is unwound, and directly from the + * return path when the workflow finished after the request was accepted — + * that frame leaves by throwing, and an append started while an error is + * already unwinding does not reliably finish. + */ + function* settleStopped(): Operation { + // A sticky durability failure is already this root's outcome, and + // retained history the definition did not align with is divergence rather + // than a stopped run: neither is papered over with a cancelled close, and + // neither is swallowed — whoever asked this root to stop has to learn + // which failure ended it. + const durabilityFailure = activeDurabilityFailure(ctx); + if (durabilityFailure !== undefined) { + phase = "ended"; + throw durabilityFailure; + } + const unconsumed = unalignedReplay(replayIndex, coroutineId); + if (unconsumed !== undefined) { + phase = "ended"; + const failure = new TerminalDivergenceError( + unconsumed.coroutineId, + unconsumed.cursor, + unconsumed.totalYields, + { + message: `Divergence: workflow ${coroutineId} was stopped before retained history was exhausted`, + unconsumed: unconsumed.entry, + }, + ); + // Remembered in the shared state before it is thrown, as a cancelled + // child's divergence is: this is the failure this run ended on, so every + // later append through the same context has to be refused by it rather + // than writing into a history that has already diverged. + rememberDurabilityFailure(ctx, failure); + throw failure; + } + phase = "appending"; + const closeEvent: Close = { + type: "close", + coroutineId, + result: { status: "cancelled" }, + }; + yield* scoped(() => appendDurableEvent(ctx, closeEvent)); + phase = "settled"; + retained = "cancelled"; + } + + yield* ensure(function* () { + if (phaseNow() !== "claimed") { + return; + } + yield* settleStopped(); + }); + + if (control !== undefined) { + control({ + requestInterruption(): boolean { + if (phaseNow() === "claimed") { + // The same ask, answered the same way. + return true; + } + if (phaseNow() !== "open") { + return false; + } + phase = "claimed"; + return true; + }, + settled: () => retained, + }); + } + try { const result: T = yield* workflow(); @@ -179,10 +356,37 @@ export function* durableRun( // compensating Close(err) from the same scope, so a turn still held here // would make that append wait forever. A boundary around the attempt // returns the turn as part of the outcome. + // + // A Stop accepted while this root was open owns its ending, and the + // workflow returning afterwards does not undo that. This root was + // interrupted, so it does not complete successfully for anybody: no ok + // close is acknowledged, no value is published, and the cancellation + // finalizer is what settles it. Leaving as `RootInterrupted` is how the + // caller learns that, and the catch below lets it past untouched so no + // compensating error close is written around it. + if (phaseNow() === "claimed") { + yield* settleStopped(); + interruption = new RootInterrupted(coroutineId); + throw interruption; + } + // The phase moves first, so a cancellation asked for from here on joins + // this ending rather than cancelling the write and compensating over a + // backend that may already hold it. + phase = "appending"; yield* scoped(() => appendDurableEvent(ctx, closeEvent)); + phase = "settled"; + retained = "ok"; return result; } catch (error) { + // A root a Stop already owns leaves as itself. The finalizer is what + // settles it, and a terminal written around that would be this run + // recording two endings. By identity, and only this run's own: an + // interruption some other root minted is an ordinary failure here, and it + // gets the ordinary error close every other failure gets. + if (interruption !== undefined && Object.is(error, interruption)) { + throw error; + } const primary = error instanceof Error ? error : new Error(String(error)); const durabilityFailure = activeDurabilityFailure(ctx, primary); if (durabilityFailure) { @@ -207,8 +411,16 @@ export function* durableRun( }, }; + // The same claim rule as the ordinary return: a Stop that holds this open + // root is its ending, and this failure does not acknowledge one around it. + if (phaseNow() === "claimed") { + throw primary; + } try { + phase = "appending"; yield* scoped(() => appendDurableEvent(ctx, closeEvent)); + phase = "settled"; + retained = "err"; } catch (closeError) { const closeDurabilityFailure = activeDurabilityFailure(ctx, closeError); if (closeDurabilityFailure) { diff --git a/packages/durable-streams/specs/effection-integration.md b/packages/durable-streams/specs/effection-integration.md index 63186393d..199fed0b1 100644 --- a/packages/durable-streams/specs/effection-integration.md +++ b/packages/durable-streams/specs/effection-integration.md @@ -926,6 +926,17 @@ closure, and emits `Close(err)` only for an ordinary workflow failure after the whole retained coroutine tree is aligned. Durability and persistence failures never become terminal events. +A root may additionally be handed a **control** by its caller +(`control: (own) => …`). Asking it to interrupt is answerable only while the +root is open, repeated asks are the same ask, and asking retains nothing by +itself — what the journal records is written when the root actually unwinds, +and it is the existing cancelled `Close`: a status and nothing else, one of +them, for the root. The control reports what was retained rather than what was +asked. This adds no event family, no field and no inference that a terminal +means success; terminal replay short-circuiting and subtree alignment are +unchanged, and a backing persistence failure raised while that ending is being +written is still never turned into a workflow terminal. + The entry point creates a scope, builds the replay index, sets up the durable context, and runs the workflow: diff --git a/packages/durable-streams/tests/durable-run.test.ts b/packages/durable-streams/tests/durable-run.test.ts index c9d8dd35d..dac2bcbfc 100644 --- a/packages/durable-streams/tests/durable-run.test.ts +++ b/packages/durable-streams/tests/durable-run.test.ts @@ -8,18 +8,22 @@ import { describe, it } from "@executablemd/test-support/bdd"; import { expect } from "@executablemd/test-support/expect"; -import { spawn, withResolvers, type Operation } from "effection"; +import { scoped, spawn, suspend, withResolvers, type Operation } from "effection"; import { type DurableEvent, DurablePersistenceError, + TerminalDivergenceError, type DurableStream, InMemoryStream, type Json, + type DurableRootControl, type Workflow, durableAction, durableCall, durableRun, + DurableContext, } from "../mod.ts"; +import { activeDurabilityFailure, appendDurableEvent } from "../durability.ts"; class PausedYieldStream implements DurableStream { readonly appendStarted = withResolvers(); @@ -476,3 +480,370 @@ describe("durableRun", () => { } }); }); + +/** + * The other ending a root may have, and only where it was asked for. + * + * A generator's return cannot be observed by the `catch` that writes this + * root's terminals, so an ordinary halt has always unwound without recording + * one — which is what leaves an interrupted run resumable. One trusted caller + * needs the opposite ending, and asks for it through this run's own control. + * + * What is under test is the authority rather than the plumbing: who may ask, + * when the ask is still answerable, and what is actually retained. + */ +describe("durableRun: the cancelled ending, asked for through its own control", () => { + it("absent the hook, a halted root still records nothing", function* () { + const stream = new InMemoryStream(); + const started = withResolvers(); + const task = yield* spawn(function* () { + return yield* durableRun( + function* () { + started.resolve(); + yield* withResolvers().operation; + return "unreachable"; + }, + { stream }, + ); + }); + yield* started.operation; + yield* task.halt(); + + // Exactly the behaviour every caller has always seen. + expect(stream.snapshot().some((event) => event.type === "close")).toBe(false); + }); + + it("an accepted request records exactly one cancelled close", function* () { + const stream = new InMemoryStream(); + const started = withResolvers(); + let control: DurableRootControl | undefined; + const task = yield* spawn(function* () { + return yield* durableRun( + function* () { + started.resolve(); + yield* withResolvers().operation; + return "unreachable"; + }, + { stream, control: (own) => (control = own) }, + ); + }); + yield* started.operation; + if (control === undefined) { + throw new Error("this run handed out no control"); + } + + // Asked while the root is open, so it is answerable — and nothing is + // retained by the asking itself. + expect(control.requestInterruption()).toBe(true); + expect(control.settled()).toBe(undefined); + // Repeated asks are the same ask. + expect(control.requestInterruption()).toBe(true); + + yield* task.halt(); + + const closes = stream.snapshot().filter((event) => event.type === "close"); + expect(closes.length).toBe(1); + const only = closes[0]; + if (only === undefined || only.type !== "close") { + throw new Error("this run recorded no close"); + } + expect(only.coroutineId).toBe("root"); + expect(only.result.status).toBe("cancelled"); + // The protocol's cancelled result carries a status and nothing else. + expect(Object.keys(only.result)).toEqual(["status"]); + // And the control says what was actually retained, not what was asked. + expect(control.settled()).toBe("cancelled"); + }); + + it("a terminal already being written keeps the ending it was writing", function* () { + const order: string[] = []; + const stream = new PausedCloseStream(order); + let control: DurableRootControl | undefined; + const task = yield* spawn(function* () { + return yield* durableRun( + function* () { + return "done"; + }, + { stream, control: (own) => (control = own) }, + ); + }); + yield* stream.appendStarted.operation; + if (control === undefined) { + throw new Error("this run handed out no control"); + } + + // The root is no longer open: its own terminal is already on its way to + // the stream, so the ask is refused rather than cancelling that write and + // compensating over a backend that may already hold it. + expect(control.requestInterruption()).toBe(false); + stream.allowAppend.resolve(); + expect(yield* task).toBe("done"); + + const closes = stream.inner.snapshot().filter((event) => event.type === "close"); + expect(closes.length).toBe(1); + const only = closes[0]; + if (only === undefined || only.type !== "close") { + throw new Error("this run recorded no close"); + } + expect(only.result.status).toBe("ok"); + expect(control.settled()).toBe("ok"); + }); +}); + +/** + * What an accepted request owns, and what it must not swallow. + * + * A Stop that reaches an open root takes its ending: the ordinary paths may + * not acknowledge an ok or err terminal around it afterwards. But holding a + * root is not a licence to end quietly — one whose durability already failed, + * or whose retained history this definition did not align with, has an outcome + * of its own, and whoever asked it to stop has to learn which one. + */ +describe("durableRun: what an accepted request owns, and what it refuses", () => { + it("a request accepted while open ends the run, however the workflow returns", function* () { + const stream = new InMemoryStream(); + const started = withResolvers(); + const release = withResolvers(); + let control: DurableRootControl | undefined; + // Asked from beside this run rather than from a task that holds it: a + // spawned run that fails takes its parent down with it, so a caller + // cannot both own the run and watch it fail. + yield* spawn(function* () { + yield* started.operation; + control?.requestInterruption(); + release.resolve(); + }); + + let completed = false; + let caught: unknown; + try { + yield* durableRun( + function* () { + started.resolve(); + yield* release.operation; + return "returned after the request"; + }, + { stream, control: (own) => (control = own) }, + ); + completed = true; + } catch (error) { + caught = error; + } + + // The workflow ran to its own end afterwards. That does not undo what was + // accepted: this root was stopped, so it completes successfully for + // nobody and publishes no value to its caller. + expect(completed).toBe(false); + expect(caught).toBeInstanceOf(Error); + expect(control?.settled()).toBe("cancelled"); + + // And exactly one terminal, saying stopped rather than succeeded. + const closes = stream.snapshot().filter((event) => event.type === "close"); + expect(closes.length).toBe(1); + const only = closes[0]; + if (only === undefined || only.type !== "close") { + throw new Error("this run recorded no close"); + } + expect(only.result.status).toBe("cancelled"); + expect(Object.keys(only.result)).toEqual(["status"]); + + // And that leaving belongs to this run alone. Carried into a second root + // that nobody asked to stop, it is an ordinary failure: that root was not + // interrupted, so it writes the error close every other failure writes + // rather than leaving its segment with no terminal at all. + if (!(caught instanceof Error)) { + throw new Error("the stopped root left without an error to carry"); + } + const elsewhere = new InMemoryStream(); + let carried: unknown; + yield* scoped(function* () { + try { + yield* durableRun( + function* () { + throw caught; + }, + { stream: elsewhere }, + ); + } catch (error) { + carried = error; + } + }); + expect(Object.is(carried, caught)).toBe(true); + const theirs = elsewhere.snapshot().filter((event) => event.type === "close"); + expect(theirs.length).toBe(1); + const ending = theirs[0]; + if (ending === undefined || ending.type !== "close") { + throw new Error("the unrequested root recorded no close"); + } + expect(ending.result.status).toBe("err"); + }); + + it("a request over an unaligned prefix raises divergence and records nothing", function* () { + const stream = new InMemoryStream([ + { + type: "yield", + coroutineId: "root", + description: { type: "action", name: "retained-not-consumed" }, + result: { status: "ok", value: "retained" }, + }, + ]); + const started = withResolvers(); + let control: DurableRootControl | undefined; + let captured: DurableContext | undefined; + const task = yield* spawn(function* () { + yield* durableRun( + function* () { + captured = yield* DurableContext.expect(); + started.resolve(); + yield* suspend(); + }, + { stream, control: (own) => (control = own) }, + ); + }); + yield* started.operation; + if (control === undefined) { + throw new Error("this run handed out no control"); + } + expect(control.requestInterruption()).toBe(true); + + let failure: unknown; + try { + yield* task.halt(); + } catch (error) { + failure = error; + } + // Retained history this definition did not reach is divergence, not a + // stopped run, and it is not papered over with a cancelled close. + expect(failure).toBeInstanceOf(TerminalDivergenceError); + expect(stream.snapshot().some((event) => event.type === "close")).toBe(false); + expect(control.settled()).toBe(undefined); + // And the root has ended, so it accepts nothing further. + expect(control.requestInterruption()).toBe(false); + + // That failure is this run's, not only this frame's: it is remembered in + // the shared state, so every later append through the same context is + // refused by the same failure rather than writing into a history that has + // already diverged. + if (captured === undefined) { + throw new Error("this run published no context"); + } + const ctx = captured; + expect(Object.is(activeDurabilityFailure(ctx), failure)).toBe(true); + let late: unknown; + try { + yield* scoped(() => + appendDurableEvent(ctx, { + type: "yield", + coroutineId: "root", + description: { type: "action", name: "after-divergence" }, + result: { status: "ok", value: "must not append" }, + }), + ); + } catch (error) { + late = error; + } + expect(Object.is(late, failure)).toBe(true); + expect(stream.snapshot().length).toBe(1); + }); + + it("a request after a sticky append failure escapes by identity", function* () { + const cause = new Error("the stream refused this append"); + const stream: DurableStream = { + *readAll(): Operation { + return []; + }, + *append(): Operation { + throw cause; + }, + }; + const started = withResolvers(); + let original: unknown; + let control: DurableRootControl | undefined; + const task = yield* spawn(function* () { + yield* durableRun( + function* () { + try { + yield* durableAction("rejected", (resolve) => { + resolve("value"); + return () => {}; + }); + } catch (error) { + original = error; + } + started.resolve(); + yield* suspend(); + }, + { stream, control: (own) => (control = own) }, + ); + }); + yield* started.operation; + if (control === undefined) { + throw new Error("this run handed out no control"); + } + expect(original).toBeInstanceOf(DurablePersistenceError); + expect(control.requestInterruption()).toBe(true); + + let failure: unknown; + try { + yield* task.halt(); + } catch (error) { + failure = error; + } + // The same failure, not a copy and not a silence: a caller that asked + // this root to stop learns what actually ended it. + expect(Object.is(failure, original)).toBe(true); + expect(control.settled()).toBe(undefined); + }); + + it("an owner that ended unrequested takes no later request", function* () { + const stream = new InMemoryStream(); + const started = withResolvers(); + let control: DurableRootControl | undefined; + const task = yield* spawn(function* () { + yield* durableRun( + function* () { + started.resolve(); + yield* suspend(); + }, + { stream, control: (own) => (control = own) }, + ); + }); + yield* started.operation; + if (control === undefined) { + throw new Error("this run handed out no control"); + } + yield* task.halt(); + + // Nothing was asked for, so nothing was recorded — and the control is + // retired rather than left accepting a claim on an owner that has gone. + expect(stream.snapshot().some((event) => event.type === "close")).toBe(false); + expect(control.requestInterruption()).toBe(false); + expect(control.settled()).toBe(undefined); + }); +}); + +/** A stream that holds the root's own Close until its caller lets it through. */ +class PausedCloseStream implements DurableStream { + readonly appendStarted = withResolvers(); + readonly allowAppend = withResolvers(); + readonly inner = new InMemoryStream(); + + constructor(readonly order: string[]) {} + + *readAll(): Operation { + return yield* this.inner.readAll(); + } + + *append(event: DurableEvent): Operation { + if (event.type === "close") { + this.order.push("close:started"); + this.appendStarted.resolve(); + yield* this.allowAppend.operation; + } + yield* this.inner.append(event); + } + + snapshot(): DurableEvent[] { + return this.inner.snapshot(); + } +}