diff --git a/packages/codemode/interpreter-support.md b/packages/codemode/interpreter-support.md index a2298fd5109e..691086d3e744 100644 --- a/packages/codemode/interpreter-support.md +++ b/packages/codemode/interpreter-support.md @@ -584,6 +584,22 @@ with a hint to encode as text first (`TextDecoder`, `toBase64`, `toHex`). - [x] `sort` in place, numeric ascending by default (`[10, 9, 1]` sorts to `[1, 9, 10]`) or by comparator. - [ ] `ArrayBuffer`, `DataView`, and other typed arrays. +## Buffer + +Node's `Buffer`, where the host has one (Node and Bun; not browsers). It is a `Uint8Array`, so every `Uint8Array` +method works on it, and its text conversions default to UTF-8. + +- [x] `Buffer.from(string, encoding)`, `Buffer.from(bytes | array | array-like | { type: "Buffer", data })`, + `alloc`, `allocUnsafe`, `concat`, `byteLength`, `compare`, `isBuffer`, and `isEncoding`, with the encodings + utf8, hex, base64, base64url, latin1/binary, ascii, and utf16le/ucs2. An unknown encoding is a `TypeError` with + code `ERR_UNKNOWN_ENCODING`. +- [x] `toString(encoding, start, end)` (so string coercion and templates give UTF-8 text), `toJSON` as + `{ type: "Buffer", data }`, `slice` and `subarray` as Buffer views on the same bytes, `equals`, `compare`, + `indexOf`, `lastIndexOf`, `includes`, `write`, `fill`, `copy`, and the `read`/`write` integer and float methods. + `console.log` prints ``. +- [x] A host Buffer crosses into the program as a Buffer, and a program Buffer crosses out as one; snapshots keep it. +- [ ] The 64-bit `readBigInt64*`/`writeBigInt64*` methods, since programs have no BigInt. + ## Web platform helpers - [x] `atob` and `btoa` with forgiving-base64 decoding and WebIDL string conversion; invalid input throws a diff --git a/packages/codemode/src/runtime/extensions.ts b/packages/codemode/src/runtime/extensions.ts index 1a4e8500dc4a..6af8ea6ba17e 100644 --- a/packages/codemode/src/runtime/extensions.ts +++ b/packages/codemode/src/runtime/extensions.ts @@ -1,5 +1,6 @@ import { Effect } from "effect" import { type Extension, isAsync, StreamingResponse } from "../extension.js" +import { BufferObj } from "../stdlib/buffer.js" import { hooked } from "../tool-runtime.js" import { errorMessage, errorType } from "./errors.js" import type { Interpreter } from "./interpreter.js" @@ -164,6 +165,8 @@ export const extensionGlobals = ( return result } if (value instanceof RegExp) return new RegExpObj(builtins.RegExp, value.source, value.flags) + if (typeof Buffer !== "undefined" && Buffer.isBuffer(value)) + return new BufferObj(builtins.Buffer, new Uint8Array(value)) if (value instanceof Uint8Array) return new Bytes(builtins.Uint8Array, new Uint8Array(value)) if (value instanceof ArrayBuffer) { const view = new Uint8Array(value) diff --git a/packages/codemode/src/runtime/globals.ts b/packages/codemode/src/runtime/globals.ts index 6909da9d0a11..fd92250b0e11 100644 --- a/packages/codemode/src/runtime/globals.ts +++ b/packages/codemode/src/runtime/globals.ts @@ -1,6 +1,7 @@ import { Arr, BoundFn, Callable, coerceToInteger, get, Obj, type Value } from "./objects.js" import { arrayGlobal } from "../stdlib/array.js" import { textDecoderGlobal, textEncoderGlobal, uint8ArrayGlobal } from "../stdlib/bytes.js" +import { bufferGlobal } from "../stdlib/buffer.js" import { mapGlobal, setGlobal, weakMapGlobal, weakSetGlobal } from "../stdlib/collections.js" import { consoleGlobal } from "../stdlib/console.js" import { dateGlobal } from "../stdlib/date.js" @@ -120,6 +121,8 @@ const table: Record = { URLSearchParams: (ctx) => urlSearchParamsGlobal(ctx), Headers: (ctx) => headersGlobal(ctx), Uint8Array: (ctx) => uint8ArrayGlobal(ctx), + // Built on the host's Buffer, so only where there is one (Node and Bun, not browsers). + ...(typeof Buffer === "undefined" ? {} : { Buffer: (ctx: Interpreter) => bufferGlobal(ctx) }), TextEncoder: (ctx) => textEncoderGlobal(ctx), TextDecoder: (ctx) => textDecoderGlobal(ctx), encodeURI: (ctx) => uriGlobal(ctx, "encodeURI"), diff --git a/packages/codemode/src/runtime/intrinsics.ts b/packages/codemode/src/runtime/intrinsics.ts index 248452546812..89b80d75735e 100644 --- a/packages/codemode/src/runtime/intrinsics.ts +++ b/packages/codemode/src/runtime/intrinsics.ts @@ -32,6 +32,7 @@ const builtins = [ "URLSearchParams", "Headers", "Uint8Array", + "Buffer", "TextEncoder", "TextDecoder", "Promise", @@ -80,6 +81,7 @@ export const createBuiltins = (): Builtins => { } const iterator = plain() const asyncIterator = plain() + const uint8Array = plain() return { Object: object, Function: fn, @@ -96,7 +98,9 @@ export const createBuiltins = (): Builtins => { URL: plain(), URLSearchParams: plain(), Headers: plain(), - Uint8Array: plain(), + Uint8Array: uint8Array, + /** Node's Buffer: a Uint8Array whose text conversions default to UTF-8. */ + Buffer: new Obj(uint8Array), TextEncoder: plain(), TextDecoder: plain(), Promise: plain(), diff --git a/packages/codemode/src/snapshot/codec.ts b/packages/codemode/src/snapshot/codec.ts index 2f998e5da20d..1af62f67aa91 100644 --- a/packages/codemode/src/snapshot/codec.ts +++ b/packages/codemode/src/snapshot/codec.ts @@ -38,6 +38,7 @@ import { Convert, Interruption, Invoke } from "../runtime/signals.js" import { deleted, OrderedTable } from "../runtime/table.js" import { AsyncFromSyncIterator } from "../generators.js" import { HostCall, Resolving } from "../scheduler.js" +import { BufferObj } from "../stdlib/buffer.js" import { TextDecoderObj } from "../stdlib/bytes.js" import { ToolError, toolError } from "../tool-error.js" import { ToolReference, ToolRuntimeError } from "../tool-runtime.js" @@ -225,6 +226,7 @@ const builtinSpecs = (): ReadonlyArray => [ { name: "HeadersObj", ctor: HeadersObj, fields: ["headers"], obj: true, constants: { tag: "Headers" } }, { name: "URLObj", ctor: URLObj, fields: ["searchParams", "url"], obj: true, constants: { tag: "URL" } }, { name: "Bytes", ctor: Bytes, fields: ["bytes"], obj: true, constants: { tag: "Uint8Array" } }, + { name: "BufferObj", ctor: BufferObj, fields: ["bytes"], obj: true, constants: { tag: "Uint8Array" } }, { name: "TextDecoderObj", ctor: TextDecoderObj, diff --git a/packages/codemode/src/stdlib/buffer.ts b/packages/codemode/src/stdlib/buffer.ts new file mode 100644 index 000000000000..1cfa520073d9 --- /dev/null +++ b/packages/codemode/src/stdlib/buffer.ts @@ -0,0 +1,302 @@ +import { createErrorValue } from "../runtime/intrinsics.js" +import type { Interpreter } from "../runtime/interpreter.js" +import { Throw, typeError } from "../runtime/model.js" +import { constructor, type Method, methods, prototypeFrom, receiver } from "../runtime/native.js" +import { + Arr, + Bytes, + define, + get, + hidden, + Obj, + coerceToNumber, + coerceToString, + record, + type Value, +} from "../runtime/objects.js" +import { describeValue } from "../runtime/references.js" + +/** A Node Buffer: a Uint8Array whose text conversions default to UTF-8. */ +export class BufferObj extends Bytes { + override toString() { + return host(this.bytes).toString("utf8") + } + override toHost() { + return Buffer.from(this.bytes) + } + override inspect() { + const shown = Array.from(this.bytes.subarray(0, 50), (byte) => byte.toString(16).padStart(2, "0")).join(" ") + const more = this.bytes.length > 50 ? ` ... ${this.bytes.length - 50} more bytes` : "" + return `` + } +} + +// The host Buffer over the same memory, so reads and writes through it are the program's bytes. +const host = (bytes: Uint8Array) => Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength) + +const encodings = new Set([ + "utf8", + "utf-8", + "hex", + "base64", + "base64url", + "latin1", + "binary", + "ascii", + "ucs2", + "ucs-2", + "utf16le", + "utf-16le", +]) + +const isEncoding = (value: Value) => typeof value === "string" && encodings.has(value.toLowerCase()) + +// Read and write methods that take only numbers, delegated to the host Buffer as is. +const numeric = [ + "readUInt8", + "readUint8", + "readInt8", + "readUInt16LE", + "readUint16LE", + "readUInt16BE", + "readUint16BE", + "readInt16LE", + "readInt16BE", + "readUInt32LE", + "readUint32LE", + "readUInt32BE", + "readUint32BE", + "readInt32LE", + "readInt32BE", + "readFloatLE", + "readFloatBE", + "readDoubleLE", + "readDoubleBE", + "readUIntLE", + "readUIntBE", + "readIntLE", + "readIntBE", + "writeUInt8", + "writeUint8", + "writeInt8", + "writeUInt16LE", + "writeUint16LE", + "writeUInt16BE", + "writeUint16BE", + "writeInt16LE", + "writeInt16BE", + "writeUInt32LE", + "writeUint32LE", + "writeUInt32BE", + "writeUint32BE", + "writeInt32LE", + "writeInt32BE", + "writeFloatLE", + "writeFloatBE", + "writeDoubleLE", + "writeDoubleBE", + "writeUIntLE", + "writeUIntBE", + "writeIntLE", + "writeIntBE", + "swap16", + "swap32", + "swap64", +] as const + +export const bufferGlobal = (ctx: Interpreter) => { + const builtins = ctx.builtins + const proto = builtins.Buffer + const wrap = (bytes: Uint8Array) => new BufferObj(proto, bytes) + const self = (thisValue: Value, name: string) => receiver(Bytes, thisValue, `Buffer.prototype.${name}`) + + const fail = (type: "Error" | "TypeError" | "RangeError", message: string, code?: string) => { + const value = createErrorValue(builtins[type], message) + if (code !== undefined) define(value, "code", code) + return new Throw(value) + } + // Host Buffer errors carry Node's `code`; they cross as the program error of the same type. + const attempt = (run: () => A): A => { + try { + return run() + } catch (error) { + if (!(error instanceof Error)) throw error + const code = (error as { code?: unknown }).code + throw fail( + error.name === "RangeError" || error.name === "TypeError" ? error.name : "Error", + error.message, + typeof code === "string" ? code : undefined, + ) + } + } + const encoding = (value: Value): BufferEncoding | undefined => { + if (value === undefined) return undefined + if (isEncoding(value)) return coerceToString(value).toLowerCase() as BufferEncoding + throw fail("TypeError", `Unknown encoding: ${coerceToString(value)}`, "ERR_UNKNOWN_ENCODING") + } + const argument = (value: Value): unknown => { + if (value instanceof Bytes) return host(value.bytes) + if (value === null || value === undefined || typeof value !== "object") return value + return coerceToNumber(value) + } + const bytesFrom = (value: Value, second: Value, third: Value): Uint8Array => { + if (typeof value === "string") return new Uint8Array(Buffer.from(value, encoding(second) ?? "utf8")) + if (value instanceof Bytes) { + const offset = second === undefined ? 0 : coerceToNumber(second) + const length = third === undefined ? undefined : coerceToNumber(third) + return value.bytes.slice(offset, length === undefined ? undefined : offset + length) + } + if (value instanceof Arr) return Uint8Array.from(value.items, coerceToNumber) + if (value instanceof Obj) { + const data = get(value, "data") + if (get(value, "type") === "Buffer" && data instanceof Arr) return Uint8Array.from(data.items, coerceToNumber) + const length = get(value, "length") + if (typeof length === "number") + return Uint8Array.from({ length }, (_, index) => coerceToNumber(get(value, String(index)))) + } + throw fail( + "TypeError", + `The first argument must be of type string or an instance of Buffer, Uint8Array, or Array or an Array-like Object. Received ${describeValue(value)}`, + "ERR_INVALID_ARG_TYPE", + ) + } + const size = (value: Value) => { + const length = coerceToNumber(value) + if (!Number.isInteger(length) || length < 0) { + throw fail("RangeError", `The argument 'size' is invalid. Received ${coerceToString(value)}`, "ERR_OUT_OF_RANGE") + } + return length + } + + const buffer = constructor(builtins, proto, { + name: "Buffer", + length: 2, + call: (_, args) => wrap(bytesFrom(args[0], args[1], args[2])), + construct: (args, newTarget) => + new BufferObj(prototypeFrom(newTarget, proto), bytesFrom(args[0], args[1], args[2])), + }) + methods(builtins, buffer, [ + ["from", 3, (_, args) => wrap(bytesFrom(args[0], args[1], args[2]))], + [ + "alloc", + 3, + (_, args) => { + const bytes = new Uint8Array(size(args[0])) + const fill = args[1] === undefined ? undefined : argument(args[1]) + // Node reads an encoding only after a string fill; after a number it would be an offset. + if (typeof fill === "string") attempt(() => host(bytes).fill(fill, encoding(args[2]) ?? "utf8")) + else if (fill !== undefined) attempt(() => host(bytes).fill(fill as number)) + return wrap(bytes) + }, + ], + ["allocUnsafe", 1, (_, args) => wrap(new Uint8Array(size(args[0])))], + ["allocUnsafeSlow", 1, (_, args) => wrap(new Uint8Array(size(args[0])))], + [ + "byteLength", + 2, + (_, args) => + args[0] instanceof Bytes + ? args[0].bytes.byteLength + : Buffer.byteLength(coerceToString(args[0]), encoding(args[1])), + ], + [ + "concat", + 2, + (_, args) => { + const list = args[0] + if (!(list instanceof Arr)) + throw typeError(`Buffer.concat expects an array of Buffers, received ${describeValue(list)}.`) + const parts = list.items.map((item) => { + if (item instanceof Bytes) return item.bytes + throw typeError(`Buffer.concat expects an array of Buffers, received ${describeValue(item)} in it.`) + }) + const total = args[1] === undefined ? parts.reduce((sum, part) => sum + part.length, 0) : size(args[1]) + const joined = new Uint8Array(total) + parts.reduce((offset, part) => { + if (offset < total) joined.set(part.subarray(0, total - offset), offset) + return offset + part.length + }, 0) + return wrap(joined) + }, + ], + ["isBuffer", 1, (_, args) => args[0] instanceof BufferObj], + ["isEncoding", 1, (_, args) => isEncoding(args[0])], + [ + "compare", + 2, + (_, args) => { + if (!(args[0] instanceof Bytes) || !(args[1] instanceof Bytes)) + throw typeError("Buffer.compare expects two Buffers or Uint8Arrays.") + return Buffer.compare(args[0].bytes, args[1].bytes) + }, + ], + ]) + define(buffer, "poolSize", 8192, hidden) + + const view = (thisValue: Value, name: string, args: Array) => { + const bytes = self(thisValue, name).bytes + const length = bytes.length + const clamp = (value: Value, fallback: number) => { + if (value === undefined) return fallback + const index = Math.trunc(coerceToNumber(value)) || 0 + return index < 0 ? Math.max(0, length + index) : Math.min(index, length) + } + return wrap(bytes.subarray(clamp(args[0], 0), clamp(args[1], length))) + } + const delegate = (name: string, length: number): Method => [ + name, + length, + (thisValue, args) => { + const target = host(self(thisValue, name).bytes) + const method = (target as unknown as Record) => unknown>)[name]! + const result = attempt(() => method.apply(target, args.map(argument))) + if (result === target) return thisValue + if (typeof result === "number" || typeof result === "boolean" || typeof result === "string") return result + return undefined + }, + ] + methods(builtins, proto, [ + [ + "toString", + 3, + (thisValue, args) => { + const bytes = self(thisValue, "toString").bytes + const start = args[1] === undefined ? 0 : Math.max(0, coerceToNumber(args[1])) + const end = args[2] === undefined ? bytes.length : Math.min(bytes.length, coerceToNumber(args[2])) + return host(bytes).toString(encoding(args[0]) ?? "utf8", start, end) + }, + ], + [ + "toJSON", + 0, + (thisValue) => + record(builtins.Object, { + type: "Buffer", + data: new Arr(builtins.Array, [...self(thisValue, "toJSON").bytes]), + }), + ], + ["slice", 2, (thisValue, args) => view(thisValue, "slice", args)], + ["subarray", 2, (thisValue, args) => view(thisValue, "subarray", args)], + [ + "equals", + 1, + (thisValue, args) => { + const other = args[0] + if (!(other instanceof Bytes)) + throw typeError( + `The "otherBuffer" argument must be an instance of Buffer or Uint8Array. Received ${describeValue(other)}`, + ) + return Buffer.compare(self(thisValue, "equals").bytes, other.bytes) === 0 + }, + ], + delegate("compare", 5), + delegate("indexOf", 3), + delegate("lastIndexOf", 3), + delegate("includes", 3), + delegate("write", 4), + delegate("fill", 4), + delegate("copy", 4), + ...numeric.map((name) => delegate(name, 2)), + ]) + return buffer +} diff --git a/packages/codemode/test/buffer.test.ts b/packages/codemode/test/buffer.test.ts new file mode 100644 index 000000000000..56c15161fdb9 --- /dev/null +++ b/packages/codemode/test/buffer.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, test } from "bun:test" +import { Effect } from "effect" +import { CodeMode, Extension } from "../src/index.js" + +const value = async (code: string, extensions: ReadonlyArray = []) => { + const result = await Effect.runPromise(CodeMode.make({ extensions }).execute(code)) + if (!result.ok) throw new Error(`expected success, got ${result.error.kind}: ${result.error.message}`) + return result.value +} + +describe("Buffer", () => { + test("converts text in each encoding, defaulting to UTF-8", async () => { + expect( + await value(`const b = Buffer.from("héllo") + return [b.length, b.toString(), String(b), \`\${b}\`, b.toString("hex"), b.toString("base64"), + Buffer.from("aGk=", "base64").toString(), Buffer.from("6869", "hex").toString(), Buffer.byteLength("é")]`), + ).toEqual([6, "héllo", "héllo", "héllo", "68c3a96c6c6f", "aMOpbGxv", "hi", "hi", 2]) + }) + + test("is a Uint8Array, and Buffer.isBuffer tells the two apart", async () => { + expect( + await value(`const b = Buffer.from([1, 2, 3]) + return [b instanceof Uint8Array, Buffer.isBuffer(b), Buffer.isBuffer(new Uint8Array(1)), [...b], b[1], + Object.prototype.toString.call(b)]`), + ).toEqual([true, true, false, [1, 2, 3], 2, "[object Uint8Array]"]) + }) + + test("concatenates, slices over the same memory, compares and searches like Node", async () => { + expect( + await value(`const b = Buffer.concat([Buffer.from("ab"), Buffer.from("cd")]) + const view = b.subarray(1, 3) + view[0] = 120 + return [b.toString(), view.toString(), b.equals(Buffer.from("axcd")), Buffer.compare(Buffer.from("a"), Buffer.from("b")), + b.includes("cd"), b.indexOf("c"), Buffer.alloc(3, 1).toString("hex"), Buffer.alloc(4, "ab").toString()]`), + ).toEqual(["axcd", "xc", true, -1, true, 2, "010101", "abab"]) + }) + + test("reads and writes numbers", async () => { + expect( + await value(`const b = Buffer.alloc(4) + b.writeUInt16BE(0x1234, 0) + b.writeInt8(-1, 2) + return [b.readUInt16BE(0), b.readUInt16LE(0), b.readInt8(2), b.toString("hex")]`), + ).toEqual([0x1234, 0x3412, -1, "1234ff00"]) + }) + + test("serializes like Node and rejects unknown encodings with Node's code", async () => { + expect( + await value(`let code + try { Buffer.from("x", "nope") } catch (e) { code = [e.name, e.code] } + return [JSON.stringify(Buffer.from("hi")), code]`), + ).toEqual(['{"type":"Buffer","data":[104,105]}', ["TypeError", "ERR_UNKNOWN_ENCODING"]]) + }) + + test("crosses extensions as a Buffer in both directions", async () => { + const seen: Array = [] + const bytes = Extension.make({ + name: "bytes", + globals: { + read: () => Buffer.from("from host"), + write: (value: unknown) => void seen.push(Buffer.isBuffer(value)), + }, + }) + expect( + await value(`const b = read() + write(Buffer.from("x")) + return [Buffer.isBuffer(b), b.toString()]`, [bytes]), + ).toEqual([true, "from host"]) + expect(seen).toEqual([true]) + }) + + test("is importable from the buffer module", async () => { + expect(await value(`const { Buffer: B } = await import("node:buffer"); return B.from("ok").toString()`)).toBe("ok") + }) +}) diff --git a/packages/codemode/test/snapshot.test.ts b/packages/codemode/test/snapshot.test.ts index 0587bdfa45ab..40cc6f4ee2d7 100644 --- a/packages/codemode/test/snapshot.test.ts +++ b/packages/codemode/test/snapshot.test.ts @@ -21,6 +21,11 @@ const programs: Record = { const decoder = new TextDecoder('utf8', { fatal: true }) const start = decoder.decode(new Uint8Array([0xef, 0xbb, 0xbf, 0xef]), { stream: true }) return [start, decoder.decode(new Uint8Array([0xbb, 0xbf]))]`, + "a Buffer keeps its type and bytes": ` + const buffer = Buffer.from("héllo") + let total = 0 + for (const byte of buffer) total += byte + return [Buffer.isBuffer(buffer), buffer.toString(), buffer.toString("base64"), total]`, "closures over per-iteration bindings": ` const fns = [] for (let i = 0; i < 5; i++) fns.push(() => i * 2)