diff --git a/packages/codemode/interpreter-support.md b/packages/codemode/interpreter-support.md index 993f5276ce9a..27ec14d339c3 100644 --- a/packages/codemode/interpreter-support.md +++ b/packages/codemode/interpreter-support.md @@ -651,6 +651,9 @@ Nothing is exposed unless a host provides it; extension calls are not tool calls - [x] A synchronous global that returns an Effect blocks the program until the Effect completes and returns its value or throws its error, as `fs.readFileSync` does. An `async` global (or one marked `Extension.async`) hands the program a promise instead. +- [x] A host function marked `Extension.writesBytes` gets its writes to a `Uint8Array` argument's copy written back + into the program's array when the call settles (returned, thrown, or its promise settled), so it can fill a buffer + it was given, as `fs.readSync(fd, buffer)` does. Unmarked functions never change what they were passed. - [ ] Program functions as arguments to extension code (callbacks such as `forEach`). - [ ] Host classes. Stateful host objects are expressed as closures; a declared method table would be the next step if `new X()` in a program is ever needed. diff --git a/packages/codemode/src/extension.ts b/packages/codemode/src/extension.ts index c6864c1a1487..e9a628f815cf 100644 --- a/packages/codemode/src/extension.ts +++ b/packages/codemode/src/extension.ts @@ -80,3 +80,17 @@ export const async = (global: F): F => { } export const isAsync = (global: Function) => global instanceof AsyncFunction || asyncGlobals.has(global) + +const writingGlobals = new WeakSet() + +/** + * Marks a function that writes into byte arrays it is given, as `fs.readSync(fd, buffer)` does. Arguments still cross + * as copies; when the call settles, the host's writes to each `Uint8Array` copy are written back into the program's + * array. Other functions never change what they were passed. + */ +export const writesBytes = (global: F): F => { + writingGlobals.add(global) + return global +} + +export const isWritingBytes = (global: Function) => writingGlobals.has(global) diff --git a/packages/codemode/src/host.ts b/packages/codemode/src/host.ts index 7ddad97120a1..b05ede8b70f6 100644 --- a/packages/codemode/src/host.ts +++ b/packages/codemode/src/host.ts @@ -274,6 +274,7 @@ export const callExtension = ( label, invocation: slots[3] as ExtensionInvocation | undefined, extension: slots[4] as string | undefined, + writeBack: slots[5] as ExtensionRequest["writeBack"], } if (call.host instanceof Lost) throw lostError(`The function ${label}`) const run = (): HostResult => { diff --git a/packages/codemode/src/runtime/extensions.ts b/packages/codemode/src/runtime/extensions.ts index f0acef90f28d..b047754aed96 100644 --- a/packages/codemode/src/runtime/extensions.ts +++ b/packages/codemode/src/runtime/extensions.ts @@ -1,5 +1,5 @@ import { Effect } from "effect" -import { type Extension, isAsync, StreamingResponse } from "../extension.js" +import { type Extension, isAsync, isWritingBytes, StreamingResponse } from "../extension.js" import { BufferObj } from "../stdlib/buffer.js" import { hooked } from "../tool-runtime.js" import { errorMessage, errorType } from "./errors.js" @@ -254,11 +254,20 @@ export const extensionGlobals = ( const args = values.map((item, index) => toHost(item, `Argument ${index + 1} to ${label}`)) const invocation = within !== undefined && root === undefined ? { extension: extension!, name: label, args } : undefined + const writeBack = isWritingBytes(value) + ? values.flatMap((item, index) => + item instanceof Bytes ? [[item, args[index], new Uint8Array(item.bytes)] as const] : [], + ) + : [] // A denied call fails the way the global would: an async global rejects its promise, a sync one throws. if (isAsync(value)) { - return pending(ctx, { stage: "run", label, extension, invocation, host: value, args, threw: false }, site) + return pending( + ctx, + { stage: "run", label, extension, invocation, host: value, args, threw: false, writeBack }, + site, + ) } - return new Push("extension call", [value, args, label, invocation, extension]) + return new Push("extension call", [value, args, label, invocation, extension, writeBack]) }, }) if (root !== undefined && within!.path !== undefined) { @@ -338,10 +347,27 @@ export const extensionGlobals = ( ), }), ) - if (request.stage === "settle") return settle(outcome(request.host, request.threw)) - return Effect.andThen( - before ?? Effect.void, - settle(Effect.suspend(() => outcome(...callHost(request.host as Function, request.args)))), + // However the call ends, the bytes the host changed in its copies reach the program's arrays. + const written = (effect: Effect.Effect) => + request.writeBack === undefined || request.writeBack.length === 0 + ? effect + : Effect.ensuring( + effect, + Effect.sync(() => { + for (const [program, host, before] of request.writeBack!) { + if (!(host instanceof Uint8Array)) continue + const length = Math.min(program.bytes.length, host.length) + for (let index = 0; index < length; index++) + if (host[index] !== before[index]) program.bytes[index] = host[index]! + } + }), + ) + if (request.stage === "settle") return written(settle(outcome(request.host, request.threw))) + return written( + Effect.andThen( + before ?? Effect.void, + settle(Effect.suspend(() => outcome(...callHost(request.host as Function, request.args)))), + ), ) } @@ -369,7 +395,7 @@ export const startExtensionCall = ( ctx: Interpreter, host: Function, args: ReadonlyArray, - call: Pick, + call: Pick, site: AstNode | undefined, ): PromiseObj | ExtensionRequest => { const [result, threw] = callHost(host, args) diff --git a/packages/codemode/src/scheduler.ts b/packages/codemode/src/scheduler.ts index 3d372013db8a..a4e23019ce39 100644 --- a/packages/codemode/src/scheduler.ts +++ b/packages/codemode/src/scheduler.ts @@ -2,7 +2,7 @@ import type { Json } from "./data.js" import type { ExtensionInvocation } from "./tool-runtime.js" import { locate } from "./runtime/errors.js" import type { AstNode } from "./runtime/model.js" -import type { Callable, Cursor, GeneratorObj, Obj, PromiseObj, Value } from "./runtime/objects.js" +import type { Bytes, Callable, Cursor, GeneratorObj, Obj, PromiseObj, Value } from "./runtime/objects.js" import type { Completion } from "./runtime/signals.js" import type { Frame } from "./vm.js" @@ -89,6 +89,12 @@ export type ExtensionRequest = { host: unknown args: ReadonlyArray readonly threw: boolean + /** + * The program's byte arrays passed as arguments, each with the host copy it crossed as and the bytes it held then. + * When the call settles, only the bytes the host changed are written back, so a host function can fill a buffer it + * was given, as `fs.readSync` does, without undoing what the program or a concurrent call wrote meanwhile. + */ + readonly writeBack?: ReadonlyArray } /** diff --git a/packages/codemode/test/extensions.test.ts b/packages/codemode/test/extensions.test.ts index 236a149fcacb..288ecf766f17 100644 --- a/packages/codemode/test/extensions.test.ts +++ b/packages/codemode/test/extensions.test.ts @@ -411,6 +411,79 @@ describe("host values with overridden methods still cross as copies", () => { }) }) +describe("byte arrays passed to a writesBytes function", () => { + const fill = (bytes: Uint8Array) => { + bytes.fill(7, 0, 2) + return 2 + } + const io = Extension.make({ + name: "io", + globals: { + io: { + fillSync: Extension.writesBytes(fill), + fillEffect: Extension.writesBytes((bytes: Uint8Array) => Effect.sync(() => fill(bytes))), + fillAsync: Extension.writesBytes(Extension.async(async (bytes: Uint8Array) => fill(bytes))), + failAfterFill: Extension.writesBytes((bytes: Uint8Array) => { + bytes[0] = 9 + throw new Error("failed") + }), + unmarked: (bytes: Uint8Array) => fill(bytes), + }, + }, + }) + + test("get the host's writes back when the call settles, however it settles", async () => { + const result = await Effect.runPromise( + CodeMode.make({ extensions: [io] }).execute( + `const a = new Uint8Array(3), b = new Uint8Array(3), c = new Uint8Array(3), d = new Uint8Array(2) + const read = io.fillSync(a) + io.fillEffect(b) + await io.fillAsync(c) + try { io.failAfterFill(d) } catch {} + const e = new Uint8Array(2) + io.unmarked(e) + return [read, [...a], [...b], [...c], [...d], [...e]]`, + ), + ) + // Only functions marked writesBytes write back; an unmarked one still can't change what it was passed. + expect(result).toMatchObject({ ok: true, value: [2, [7, 7, 0], [7, 7, 0], [7, 7, 0], [9, 0], [0, 0]] }) + }) +}) + +describe("concurrent writesBytes calls into one buffer", () => { + test("keep each other's bytes and what the program wrote meanwhile", async () => { + const io = Extension.make({ + name: "io", + globals: { + readAt: Extension.writesBytes( + Extension.async(async (bytes: Uint8Array, offset: number, ms: number, value: number) => { + await new Promise((resolve) => setTimeout(resolve, ms)) + bytes.fill(value, offset, offset + 2) + }), + ), + }, + }) + const result = await Effect.runPromise( + CodeMode.make({ extensions: [io] }).execute( + `const a = new Uint8Array(4) + await Promise.all([readAt(a, 0, 5, 1), readAt(a, 2, 20, 3)]) + const b = new Uint8Array(4) + const pending = readAt(b, 0, 5, 1) + b[3] = 9 + await pending + return [[...a], [...b]]`, + ), + ) + expect(result).toMatchObject({ + ok: true, + value: [ + [1, 1, 3, 3], + [1, 1, 0, 9], + ], + }) + }) +}) + describe("Effect globals", () => { class Session extends Context.Service()("test/Session") {} const sent: Array = []