diff --git a/packages/ai/src/protocols/utils/gateway-protocol.ts b/packages/ai/src/protocols/utils/gateway-protocol.ts index f6588a9fe2f0..dbf11671b416 100644 --- a/packages/ai/src/protocols/utils/gateway-protocol.ts +++ b/packages/ai/src/protocols/utils/gateway-protocol.ts @@ -1,11 +1,19 @@ -import { Effect, Option, Schema } from "effect" +import { Effect, Option, Schema, SchemaGetter } from "effect" import { Protocol } from "../../route/protocol.js" -import { LLMEvent, mergeJsonRecords, type AIError, type LLMRequest } from "../../schema/index.js" +import { + LLMEvent, + Usage, + mergeJsonRecords, + type AIError, + type LLMRequest, + type ProviderMetadata, +} from "../../schema/index.js" import { JsonObject, lenient } from "../shared.js" interface ParserState { readonly inner: Inner readonly gateway?: Record + readonly providerMetadataKey: string } const GatewayHolder = Schema.Struct({ @@ -18,6 +26,7 @@ const GatewayHolder = Schema.Struct({ const GatewayEvent = Schema.Struct({ ...GatewayHolder.fields, + message: lenient(GatewayHolder), response: lenient(GatewayHolder), choices: lenient( Schema.Array( @@ -29,14 +38,70 @@ const GatewayEvent = Schema.Struct({ }) const decodeGatewayEvent = Schema.decodeUnknownOption(GatewayEvent) +// Gateway bills /v1/messages and /v1/responses as a decimal string. Trimming and matching before Number() +// keeps "" and " " from coercing to 0, which SessionUsage.record would trust as an authoritative free request. +const NonNegativeFinite = Schema.Finite.check(Schema.isGreaterThanOrEqualTo(0)) +const DecimalCost = Schema.Trim.check( + Schema.isPattern(/^(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?$/), +).pipe( + Schema.decodeTo(NonNegativeFinite, { + decode: SchemaGetter.transform((value) => Number(value)), + encode: SchemaGetter.transform((value) => String(value)), + }), +) +const GatewayBilling = Schema.Struct({ + cost: lenient(Schema.Union([NonNegativeFinite, DecimalCost])), +}) +const decodeGatewayBilling = Schema.decodeUnknownOption(GatewayBilling) + +// A terminal frame can arrive without token counts; the billed amount is still authoritative, so it becomes a +// usage carrying only `cost`. Rebuilding through `new Usage` keeps the `visibleOutputTokens` getter the spread drops. +function withGatewayCost(usage: Usage | undefined, cost: number | undefined): Usage | undefined { + if (cost === undefined || usage?.cost !== undefined) return usage + return new Usage(usage === undefined ? { cost } : { ...usage, cost }) +} + +// The Gateway namespaces its metadata under `gateway` on the wire, while core reads the route's +// `providerMetadataKey`. Emit both so either reader finds the same generation and billing data. +function mergeGatewayMetadata( + metadata: ProviderMetadata | undefined, + gateway: Record, + providerMetadataKey: string, +): ProviderMetadata { + const merged: ProviderMetadata = { + ...metadata, + gateway: mergeJsonRecords(metadata?.["gateway"], gateway) ?? gateway, + } + if (providerMetadataKey === "gateway") return merged + return { + ...merged, + [providerMetadataKey]: mergeJsonRecords(metadata?.[providerMetadataKey], { gateway }) ?? { gateway }, + } +} + function attachGatewayMetadata( events: ReadonlyArray, gateway: Record | undefined, + providerMetadataKey: string, ): ReadonlyArray { - if (!gateway || !events.some(LLMEvent.is.finish)) return events - return events.map((event) => - LLMEvent.is.finish(event) ? { ...event, providerMetadata: { ...event.providerMetadata, gateway } } : event, - ) + if (!gateway) return events + if (!events.some((event) => LLMEvent.is.stepFinish(event) || LLMEvent.is.finish(event))) return events + const cost = Option.getOrUndefined(decodeGatewayBilling(gateway))?.cost + return events.map((event) => { + if (LLMEvent.is.stepFinish(event)) + return LLMEvent.stepFinish({ + ...event, + usage: withGatewayCost(event.usage, cost), + providerMetadata: mergeGatewayMetadata(event.providerMetadata, gateway, providerMetadataKey), + }) + if (LLMEvent.is.finish(event)) + return LLMEvent.finish({ + ...event, + usage: withGatewayCost(event.usage, cost), + providerMetadata: mergeGatewayMetadata(event.providerMetadata, gateway, providerMetadataKey), + }) + return event + }) } export function gatewayProtocol( @@ -50,6 +115,7 @@ export function gatewayProtocol( ) { const initial = (request: LLMRequest): ParserState => ({ inner: protocol.stream.initial(request), + providerMetadataKey: request.model.route.providerMetadataKey ?? request.model.provider, }) const onHalt = protocol.stream.onHalt @@ -75,16 +141,23 @@ export function gatewayProtocol( ? mergeJsonRecords( state.gateway, decoded.provider_metadata?.gateway, + decoded.message?.provider_metadata?.gateway, decoded.response?.provider_metadata?.gateway, ...(decoded.choices ?? []).map((choice) => choice.delta?.provider_metadata?.gateway), ) : state.gateway const [inner, events] = yield* protocol.stream.step(state.inner, event) - return [{ inner, gateway }, attachGatewayMetadata(events, gateway)] as const + return [ + { inner, gateway, providerMetadataKey: state.providerMetadataKey }, + attachGatewayMetadata(events, gateway, state.providerMetadataKey), + ] as const }), terminal: protocol.stream.terminal, onHalt: onHalt - ? (state) => onHalt(state.inner).pipe(Effect.map((events) => attachGatewayMetadata(events, state.gateway))) + ? (state) => + onHalt(state.inner).pipe( + Effect.map((events) => attachGatewayMetadata(events, state.gateway, state.providerMetadataKey)), + ) : undefined, }, }) diff --git a/packages/ai/test/provider/vercel-ai-gateway.recorded.test.ts b/packages/ai/test/provider/vercel-ai-gateway.recorded.test.ts index 41d768291081..13a40e265050 100644 --- a/packages/ai/test/provider/vercel-ai-gateway.recorded.test.ts +++ b/packages/ai/test/provider/vercel-ai-gateway.recorded.test.ts @@ -74,6 +74,17 @@ for (const item of [ expect(events.filter(LLMEvent.is.finish).every((event) => event.providerMetadata?.gateway !== undefined)).toBe( true, ) + const stepFinishes = events.filter(LLMEvent.is.stepFinish) + expect(stepFinishes.length).toBeGreaterThan(0) + expect( + stepFinishes.every( + (event) => + event.providerMetadata?.gateway !== undefined && + event.providerMetadata?.["vercel-ai-gateway"]?.gateway !== undefined && + typeof event.usage?.cost === "number" && + event.usage.cost > 0, + ), + ).toBe(true) if (item.reasoning) expect(events.some(LLMEvent.is.reasoningEnd)).toBe(true) }), 120_000, diff --git a/packages/ai/test/provider/vercel-ai-gateway.test.ts b/packages/ai/test/provider/vercel-ai-gateway.test.ts index 6c31b42cad75..4a13af024581 100644 --- a/packages/ai/test/provider/vercel-ai-gateway.test.ts +++ b/packages/ai/test/provider/vercel-ai-gateway.test.ts @@ -1,11 +1,11 @@ import { expect } from "bun:test" -import { ConfigProvider, Effect } from "effect" -import { ai, CacheHint, LLMClient, LLMEvent, Message } from "../../src/effect.js" +import { ConfigProvider, Effect, Stream } from "effect" +import { ai, CacheHint, LanguageModel, LLMClient, LLMEvent, Message, Usage } from "../../src/effect.js" import { VercelAIGateway } from "../../src/providers/vercel-ai-gateway.js" import { compileRequest } from "../../src/route/client.js" import { it } from "../lib/effect.js" import { dynamicResponse } from "../lib/http.js" -import { sseEvents } from "../lib/sse.js" +import { sseEvents, sseRaw } from "../lib/sse.js" import { isRetryable } from "../../src/provider-error.js" it.effect("Gateway defaults preserve full model IDs and select the requested families", () => @@ -301,3 +301,253 @@ it.effect("Gateway authentication supports OIDC fallback and affinity without re ), ), ) + +const gatewayResponseBody = (input: { + readonly id: string + readonly cost: string | number + readonly usage?: Readonly> +}) => + sseEvents( + { + type: "response.completed", + response: { + id: input.id, + output: [], + service_tier: "priority", + ...(input.usage ? { usage: input.usage } : {}), + provider_metadata: { + gateway: { cost: input.cost, generationId: "gen_resp_1", routing: { finalProvider: "openai" } }, + }, + }, + }, + ) + +it.effect( + "Gateway Messages attaches gateway metadata, parsed string cost, and preserves visibleOutputTokens on step-finish and finish", + () => + Effect.gen(function* () { + const expectedGateway = { + generationId: "gen_msg_1", + cost: "0.0207155", + routing: { finalProvider: "anthropic" }, + } + const response = yield* LLMClient.generate( + ai.request({ + model: VercelAIGateway.configure({ apiKey: "fixture" }).messages("anthropic/claude-sonnet-5.5"), + prompt: "Hello", + }), + ).pipe( + Effect.provide( + dynamicResponse((input) => + Effect.sync(() => + input.respond( + sseEvents( + { + type: "message_start", + message: { + usage: { input_tokens: 10, output_tokens: 0 }, + provider_metadata: { gateway: { generationId: "gen_msg_1" } }, + }, + }, + { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }, + { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "Hello" } }, + { type: "content_block_stop", index: 0 }, + { + type: "message_delta", + delta: { stop_reason: "end_turn", stop_sequence: "" }, + usage: { output_tokens: 12, output_tokens_details: { thinking_tokens: 2 } }, + provider_metadata: { + gateway: { cost: "0.0207155", routing: { finalProvider: "anthropic" } }, + }, + }, + { type: "message_stop" }, + ), + { headers: { "content-type": "text/event-stream" } }, + ), + ), + ), + ), + ) + const terminal = response.events.filter((event) => LLMEvent.is.stepFinish(event) || LLMEvent.is.finish(event)) + expect(terminal).toHaveLength(2) + for (const event of terminal) { + expect(event.providerMetadata?.gateway).toEqual(expectedGateway) + expect(event.providerMetadata?.["vercel-ai-gateway"]).toEqual({ + stopSequence: "", + gateway: expectedGateway, + }) + expect(event.usage).toBeInstanceOf(Usage) + expect(event.usage?.cost).toBe(0.0207155) + expect(event.usage?.outputTokens).toBe(12) + expect(event.usage?.reasoningTokens).toBe(2) + expect(event.usage?.visibleOutputTokens).toBe(10) + expect(event.usage?.providerMetadata?.["vercel-ai-gateway"]).toBeDefined() + } + expect(response.usage?.cost).toBe(0.0207155) + }), +) + +it.effect( + "Gateway Responses merges gateway metadata with responseId/serviceTier and supports route providerMetadataKey override", + () => + Effect.gen(function* () { + const gateway = VercelAIGateway.configure({ apiKey: "fixture" }) + const run = (model: LanguageModel) => + LLMClient.stream(ai.request({ model, prompt: "Hello" })).pipe( + Stream.runCollect, + Effect.provide( + dynamicResponse((input) => + Effect.sync(() => + input.respond( + gatewayResponseBody({ + id: "resp_gw_1", + cost: "0.001", + usage: { + input_tokens: 10, + output_tokens: 5, + output_tokens_details: { reasoning_tokens: 2 }, + }, + }), + { headers: { "content-type": "text/event-stream" } }, + ), + ), + ), + ), + ) + const stepFinishOf = (events: Iterable) => { + const stepFinish = Array.from(events).find(LLMEvent.is.stepFinish) + if (stepFinish === undefined) throw new Error("Responses stream did not emit a step-finish event") + return stepFinish + } + const stepFinish = stepFinishOf(yield* run(gateway.responses("openai/gpt-6-luna"))) + expect(stepFinish.providerMetadata?.["vercel-ai-gateway"]).toMatchObject({ + responseId: "resp_gw_1", + serviceTier: "priority", + gateway: { cost: "0.001", generationId: "gen_resp_1", routing: { finalProvider: "openai" } }, + }) + expect(stepFinish.usage?.visibleOutputTokens).toBe(3) + expect(stepFinish.usage?.cost).toBe(0.001) + + const base = gateway.responses("openai/gpt-6-luna") + const custom = LanguageModel.update(base, { route: base.route.with({ provider: "custom-gw" }) }) + const overridden = stepFinishOf(yield* run(custom)) + expect(overridden.providerMetadata?.["custom-gw"]?.gateway).toBeDefined() + }), +) + +it.effect( + "Gateway Chat onHalt preserves numeric usage.cost over string gateway.cost and falls back to gateway.cost when usage.cost is absent", + () => + Effect.gen(function* () { + const run = (gateway: Readonly>, usage: Readonly>) => { + const body = sseRaw( + `data: ${JSON.stringify({ choices: [{ delta: { provider_metadata: { gateway } } }] })}`, + `data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: "stop" }], usage })}`, + ) + return LLMClient.stream( + ai.request({ + model: VercelAIGateway.configure({ apiKey: "fixture" }).chat("meta/muse-spark-1.3"), + prompt: "Hello", + }), + ).pipe( + Stream.runCollect, + Effect.provide( + dynamicResponse((input) => + Effect.sync(() => input.respond(body, { headers: { "content-type": "text/event-stream" } })), + ), + ), + ) + } + const stepFinishOf = (events: Iterable) => { + const stepFinish = Array.from(events).find(LLMEvent.is.stepFinish) + if (stepFinish === undefined) throw new Error("Chat stream did not emit a step-finish event") + return stepFinish + } + + const preserved = stepFinishOf( + yield* run( + { cost: "0.99", routing: { finalProvider: "anthropic" } }, + { prompt_tokens: 10, completion_tokens: 5, cost: 0 }, + ), + ) + expect(preserved.usage?.cost).toBe(0) + expect(preserved.providerMetadata?.["vercel-ai-gateway"]?.gateway).toMatchObject({ + routing: { finalProvider: "anthropic" }, + }) + + const fallback = stepFinishOf( + yield* run( + { cost: "0.0042", routing: { finalProvider: "anthropic" } }, + { prompt_tokens: 10, completion_tokens: 5 }, + ), + ) + expect(fallback.usage?.cost).toBe(0.0042) + }), +) + +it.effect( + "Gateway cost parser accepts valid decimals and rejects empty, negative, or non-finite strings without coercing to zero", + () => + Effect.gen(function* () { + const costSource = (cost: string | number, index: number, withUsage: boolean) => + sseEvents( + { + type: "response.completed", + response: { + id: `resp_cost_${index}`, + output: [], + ...(withUsage ? { usage: { input_tokens: 10, output_tokens: 5 } } : {}), + provider_metadata: { gateway: { cost, routing: { finalProvider: "openai" } } }, + }, + }, + ) + const stepFinishFor = (body: string) => + Effect.gen(function* () { + const events = yield* LLMClient.stream( + ai.request({ + model: VercelAIGateway.configure({ apiKey: "fixture" }).responses("openai/gpt-6-luna"), + prompt: "Hello", + }), + ).pipe( + Stream.runCollect, + Effect.provide( + dynamicResponse((input) => + Effect.sync(() => input.respond(body, { headers: { "content-type": "text/event-stream" } })), + ), + ), + ) + const stepFinish = Array.from(events).find(LLMEvent.is.stepFinish) + if (stepFinish === undefined) throw new Error("Responses stream did not emit a step-finish event") + return stepFinish + }) + + const valid = [ + { cost: "0", expected: 0 }, + { cost: "0.00", expected: 0 }, + { cost: " 0.5 ", expected: 0.5 }, + { cost: "1e-6", expected: 1e-6 }, + { cost: ".25", expected: 0.25 }, + { cost: 0, expected: 0 }, + ] as const + for (const [index, item] of valid.entries()) { + const stepFinish = yield* stepFinishFor(costSource(item.cost, index, true)) + expect(stepFinish.usage?.cost).toBe(item.expected) + } + + const invalid = ["", " ", "-0.01", -1, "NaN", "Infinity", "1e500", "0x10", "$0.02"] as const + for (const [index, cost] of invalid.entries()) { + const stepFinish = yield* stepFinishFor(costSource(cost, index, true)) + expect(stepFinish.usage).toBeInstanceOf(Usage) + expect(stepFinish.usage?.inputTokens).toBe(10) + expect(stepFinish.usage?.outputTokens).toBe(5) + expect(stepFinish.usage?.cost).toBeUndefined() + expect(stepFinish.providerMetadata?.["vercel-ai-gateway"]?.gateway).toMatchObject({ + routing: { finalProvider: "openai" }, + }) + } + + const withoutUsage = yield* stepFinishFor(costSource("0.01", 99, false)) + expect(withoutUsage.usage).toBeInstanceOf(Usage) + expect(withoutUsage.usage?.cost).toBe(0.01) + }), +) diff --git a/packages/core/src/session/transfer.ts b/packages/core/src/session/transfer.ts index e944238a3519..985dd5fb4cf1 100644 --- a/packages/core/src/session/transfer.ts +++ b/packages/core/src/session/transfer.ts @@ -303,6 +303,7 @@ function sanitizeMessage(message: SessionMessage.Info): SessionMessage.Info { return { ...message, metadata: meta, + native: metadata("assistant-native", message.id, message.native), content: message.content.map((content) => { if (content.type === "text") return { diff --git a/packages/core/test/lib/session-runner-harness.ts b/packages/core/test/lib/session-runner-harness.ts new file mode 100644 index 000000000000..a6d4133457e2 --- /dev/null +++ b/packages/core/test/lib/session-runner-harness.ts @@ -0,0 +1,208 @@ +import { type LLMClientService } from "@opencode/ai/route" +import { Agent } from "@opencode/core/agent" +import { Bus } from "@opencode/core/bus" +import { Config } from "@opencode/core/config" +import { Database } from "@opencode/core/database/database" +import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" +import { LayerNodePlatform } from "@opencode/core/effect/app-node-platform" +import { InstructionBuiltIns } from "@opencode/core/instructions/builtins" +import { InstructionDiscovery } from "@opencode/core/instruction-discovery" +import { Instructions } from "@opencode/core/instructions/index" +import { Location } from "@opencode/core/location" +import { LocationServiceMap } from "@opencode/core/location-service-map" +import { McpInstructions } from "@opencode/core/mcp/instructions" +import { Model } from "@opencode/core/model" +import { Permission } from "@opencode/core/permission" +import { Plugin } from "@opencode/core/plugin" +import { PluginHooks } from "@opencode/core/plugin/hooks" +import { IdentityPlugin } from "@opencode/core/plugin/identity" +import { OptimizePlugin } from "@opencode/core/plugin/optimize" +import { PluginSupervisor } from "@opencode/core/plugin/supervisor" +import { Project } from "@opencode/core/project" +import { ProjectTable } from "@opencode/core/project/sql" +import { ReferenceInstructions } from "@opencode/core/reference/instructions" +import { AbsolutePath } from "@opencode/core/schema" +import { Session } from "@opencode/core/session" +import { SessionExecution } from "@opencode/core/session/execution" +import { SessionMessage } from "@opencode/core/session/message" +import { SessionProjector } from "@opencode/core/session/projector" +import { SessionRunCoordinator } from "@opencode/core/session/run-coordinator" +import { SessionRunner } from "@opencode/core/session/runner/index" +import { SessionRunnerLLM } from "@opencode/core/session/runner/llm" +import { SessionRunnerModel } from "@opencode/core/session/runner/model" +import { SessionTable } from "@opencode/core/session/sql" +import { SessionStore } from "@opencode/core/session/store" +import { SkillInstructions } from "@opencode/core/skill/instructions" +import { Snapshot } from "@opencode/core/snapshot" +import { Tool } from "@opencode/core/tool" +import { LayerNode } from "@opencode/util/effect/layer-node" +import { Effect, Layer } from "effect" +import { promptLocationNode } from "../fixture/prompt-location" +import { agentHost, host, modelHost, noProviders } from "../plugin/host" +import { permissionLayer } from "./permission" + +const permission = permissionLayer() +const systemContext = Layer.mock(InstructionBuiltIns.Service, { load: () => Effect.succeed(Instructions.empty) }) +const instructionContext = Layer.mock(InstructionDiscovery.Service, { + project: true, + global: true, + claude: true, + load: () => Effect.succeed(Instructions.empty), +}) +const skillInstructions = Layer.mock(SkillInstructions.Service, { load: () => Effect.succeed(Instructions.empty) }) +const referenceInstructions = Layer.mock(ReferenceInstructions.Service, { + load: () => Effect.succeed(Instructions.empty), +}) +const mcpInstructions = Layer.mock(McpInstructions.Service, { load: () => Effect.succeed(Instructions.empty) }) +const config = Config.testLayer() +const promptModels = Layer.mock(Model.Service, { + get: () => Effect.undefined, + all: () => Effect.succeed([]), + available: () => Effect.succeed([]), + default: () => Effect.undefined, + small: () => Effect.undefined, +}) + +const runnerLayer = (llmClient: Layer.Layer, models: Layer.Layer) => + AppNodeBuilder.build(SessionRunnerLLM.node, [ + Snapshot.node.replace(Snapshot.noopLayer), + LayerNodePlatform.llmClient.replace(llmClient), + SessionRunnerModel.node.replace(models), + InstructionBuiltIns.node.replace(systemContext), + InstructionDiscovery.node.replace(instructionContext), + Location.node.replace(Location.boundNode({ directory: AbsolutePath.make("/project") })), + SkillInstructions.node.replace(skillInstructions), + ReferenceInstructions.node.replace(referenceInstructions), + McpInstructions.node.replace(mcpInstructions), + Config.node.replace(config), + Permission.node.replace(permission), + PluginSupervisor.node.replace(Layer.empty), + Plugin.node.replace(Layer.mock(Plugin.Service, { awaitActivation: Effect.void })), + ]) + +const execution = (llmClient: Layer.Layer, models: Layer.Layer) => + Layer.effect( + SessionExecution.Service, + Effect.gen(function* () { + const sessionRunner = yield* SessionRunner.Service + const coordinator = yield* SessionRunCoordinator.make({ + drain: (sessionID, force) => sessionRunner.drain({ sessionID, force }).pipe(Effect.asVoid), + }) + return SessionExecution.Service.of({ + active: coordinator.active, + isActive: coordinator.isActive, + resume: coordinator.run, + wake: coordinator.wake, + interrupt: (sessionID) => coordinator.interrupt(sessionID), + awaitIdle: coordinator.awaitIdle, + }) + }), + ).pipe(Layer.provide(runnerLayer(llmClient, models)), Layer.orDie) + +/** + * Session runner test layer shared by the recorded-transport and Gateway end-to-end suites. + * Trailing nodes add services the suite exercises (for example `SessionTransfer.node`) without + * growing another runner stack. + */ +export const sessionRunnerTestLayer = []>( + options: { + readonly llmClient: Layer.Layer + readonly models: Layer.Layer + }, + ...nodes: Nodes +) => + AppNodeBuilder.build( + LayerNode.group([ + Database.node, + Bus.node, + SessionProjector.node, + SessionStore.node, + Agent.node, + Model.node, + PluginHooks.node, + Tool.node, + SessionRunnerModel.node, + InstructionBuiltIns.node, + InstructionDiscovery.node, + SkillInstructions.node, + ReferenceInstructions.node, + Config.node, + Snapshot.node, + SessionRunnerLLM.node, + Session.node, + ...nodes, + ]), + [ + Bus.node.replace(Bus.configured({ persist: true })), + LocationServiceMap.node.replace(promptLocationNode), + LayerNodePlatform.llmClient.replace(options.llmClient), + Permission.node.replace(permission), + Model.node.replace(promptModels), + SessionRunnerModel.node.replace(options.models), + InstructionBuiltIns.node.replace(systemContext), + InstructionDiscovery.node.replace(instructionContext), + Location.node.replace(Location.boundNode({ directory: AbsolutePath.make("/project") })), + SkillInstructions.node.replace(skillInstructions), + ReferenceInstructions.node.replace(referenceInstructions), + Config.node.replace(config), + Snapshot.node.replace(Snapshot.noopLayer), + PluginSupervisor.node.replace(Layer.empty), + Plugin.node.replace(Layer.mock(Plugin.Service, { awaitActivation: Effect.void })), + SessionExecution.node.replace(execution(options.llmClient, options.models)), + ], + ) + +/** Seed the global project and one session, then admit its prompt without resuming the drain. */ +export const seedSession = (sessionID: Session.ID) => + Effect.gen(function* () { + const agents = yield* Agent.Service + const models = yield* Model.Service + const hooks = yield* PluginHooks.Service + yield* agents.transform((editor) => + editor.update(Agent.ID.make("build"), (agent) => { + agent.mode = "primary" + agent.permissions.push({ action: "execute", resource: "*", effect: "deny" }) + }), + ) + const pluginHost = host({ + agent: agentHost(agents), + model: modelHost(models), + provider: noProviders, + session: { hook: (name, callback) => hooks.register("session", name, callback) }, + }) + yield* Effect.forEach(OptimizePlugin.Plugins, (plugin) => plugin.effect(pluginHost), { discard: true }) + yield* IdentityPlugin.Plugin.effect(pluginHost) + const { db } = yield* Database.Service + yield* db + .insert(ProjectTable) + .values({ id: Project.ID.global, worktree: AbsolutePath.make("/project"), sandboxes: [] }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionTable) + .values({ + id: sessionID, + project_id: Project.ID.global, + slug: "test", + directory: "/project", + title: "test", + version: "test", + }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + const session = yield* Session.Service + const prompt = yield* session.prompt({ + sessionID, + text: "Say hello in one short sentence.", + resume: false, + }) + return { session, prompt } + }) + +export const assistant = (messages: ReadonlyArray) => { + const message = messages.find((item): item is SessionMessage.Assistant => item.type === "assistant") + if (!message) throw new Error("Expected an assistant message") + return message +} diff --git a/packages/core/test/session-create.test.ts b/packages/core/test/session-create.test.ts index 1701d99de655..2a63dd550bec 100644 --- a/packages/core/test/session-create.test.ts +++ b/packages/core/test/session-create.test.ts @@ -1467,6 +1467,45 @@ describe("SessionTransfer", () => { }), ) + it.effect("preserves assistant native metadata by default and redacts it on sanitize", () => + Effect.gen(function* () { + const session = yield* Session.Service + const transfer = yield* SessionTransfer.Service + const template = yield* session.create({ location }) + const sessionID = Session.ID.create() + const assistantID = SessionMessage.ID.create() + const native = { + gateway: { + routing: { finalProvider: "anthropic", clientSessionId: "secret-fingerprint" }, + generationId: "gen_123", + }, + } + + yield* transfer.import({ + data: { + info: { ...template, id: sessionID }, + messages: [ + SessionMessage.Assistant.make({ + id: assistantID, + type: "assistant", + agent: Agent.ID.make("build"), + model: Model.Ref.make({ id: Model.ID.make("model"), providerID: Provider.ID.make("provider") }), + content: [], + native, + time: { created: DateTime.makeUnsafe(1), completed: DateTime.makeUnsafe(2) }, + }), + ], + }, + location, + }) + + expect((yield* transfer.export({ sessionID })).messages.at(-1)).toMatchObject({ native }) + expect((yield* transfer.export({ sessionID, sanitize: true })).messages.at(-1)).toMatchObject({ + native: { redacted: `assistant-native:${assistantID}` }, + }) + }), + ) + it.effect("imports projected messages and reserves their aggregate sequence", () => Effect.gen(function* () { const session = yield* Session.Service diff --git a/packages/core/test/session-runner-recorded.test.ts b/packages/core/test/session-runner-recorded.test.ts index b0c1fac750a2..819d52bc145b 100644 --- a/packages/core/test/session-runner-recorded.test.ts +++ b/packages/core/test/session-runner-recorded.test.ts @@ -1,51 +1,16 @@ import { HttpRecorder } from "@opencode/http-recorder" import { OpenAIChat } from "@opencode/ai/protocols/openai-chat" -import { Auth, LLMClient, type LLMClientService, RequestExecutor } from "@opencode/ai/route" -import { Model } from "@opencode/core/model" +import { Auth, LLMClient, RequestExecutor } from "@opencode/ai/route" import { Database } from "@opencode/core/database/database" -import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" -import { LayerNodePlatform } from "@opencode/core/effect/app-node-platform" -import { LayerNode } from "@opencode/util/effect/layer-node" -import { Bus } from "@opencode/core/bus" import { EventTable } from "@opencode/core/event/sql" -import { Permission } from "@opencode/core/permission" -import { Agent } from "@opencode/core/agent" -import { Config } from "@opencode/core/config" -import { Project } from "@opencode/core/project" -import { ProjectTable } from "@opencode/core/project/sql" -import { AbsolutePath } from "@opencode/core/schema" import { Session } from "@opencode/core/session" -import { Snapshot } from "@opencode/core/snapshot" -import { SessionProjector } from "@opencode/core/session/projector" -import { SessionExecution } from "@opencode/core/session/execution" -import { SessionRunCoordinator } from "@opencode/core/session/run-coordinator" -import { SessionRunner } from "@opencode/core/session/runner/index" -import { SessionRunnerLLM } from "@opencode/core/session/runner/llm" import { SessionRunnerModel } from "@opencode/core/session/runner/model" -import { Tool } from "@opencode/core/tool" -import { SessionTable } from "@opencode/core/session/sql" -import { SessionStore } from "@opencode/core/session/store" -import { Location } from "@opencode/core/location" -import { InstructionBuiltIns } from "@opencode/core/instructions/builtins" -import { InstructionDiscovery } from "@opencode/core/instruction-discovery" -import { Instructions } from "@opencode/core/instructions/index" -import { SkillInstructions } from "@opencode/core/skill/instructions" -import { ReferenceInstructions } from "@opencode/core/reference/instructions" -import { McpInstructions } from "@opencode/core/mcp/instructions" -import { PluginSupervisor } from "@opencode/core/plugin/supervisor" -import { Plugin } from "@opencode/core/plugin" -import { PluginHooks } from "@opencode/core/plugin/hooks" -import { OptimizePlugin } from "@opencode/core/plugin/optimize" -import { IdentityPlugin } from "@opencode/core/plugin/identity" import { describe, expect } from "bun:test" import { eq } from "drizzle-orm" import { Effect, Layer, Option, Tracer } from "effect" import path from "node:path" import { testEffect } from "./lib/effect" -import { LocationServiceMap } from "@opencode/core/location-service-map" -import { promptLocationNode } from "./fixture/prompt-location" -import { permissionLayer } from "./lib/permission" -import { agentHost, modelHost, host, noProviders } from "./plugin/host" +import { seedSession, sessionRunnerTestLayer } from "./lib/session-runner-harness" const cassetteName = "session-runner/openai-chat-streams-text" const cassetteDirectory = path.resolve(import.meta.dir, "fixtures/recordings") @@ -56,7 +21,6 @@ if (process.env.RECORD === "true") { const cassette = HttpRecorder.layerFetch(cassetteName, { directory: cassetteDirectory }) const executor = RequestExecutor.layer.pipe(Layer.provide(cassette)) const client = LLMClient.layer.pipe(Layer.provide(executor)) -const permission = permissionLayer() const model = OpenAIChat.route .with({ endpoint: { baseURL: "https://api.openai.com/v1" }, @@ -75,100 +39,6 @@ const models = Layer.mock(SessionRunnerModel.Service)({ }), ), }) -const systemContext = Layer.mock(InstructionBuiltIns.Service, { load: () => Effect.succeed(Instructions.empty) }) -const instructionContext = Layer.mock(InstructionDiscovery.Service, { - project: true, - global: true, - claude: true, - load: () => Effect.succeed(Instructions.empty), -}) -const skillInstructions = Layer.mock(SkillInstructions.Service, { load: () => Effect.succeed(Instructions.empty) }) -const referenceInstructions = Layer.mock(ReferenceInstructions.Service, { - load: () => Effect.succeed(Instructions.empty), -}) -const mcpInstructions = Layer.mock(McpInstructions.Service, { load: () => Effect.succeed(Instructions.empty) }) -const config = Config.testLayer() -const promptModels = Layer.mock(Model.Service, { - get: () => Effect.undefined, - all: () => Effect.succeed([]), - available: () => Effect.succeed([]), - default: () => Effect.undefined, - small: () => Effect.undefined, -}) -const runnerLayer = (llmClient: Layer.Layer) => - AppNodeBuilder.build(SessionRunnerLLM.node, [ - Snapshot.node.replace(Snapshot.noopLayer), - LayerNodePlatform.llmClient.replace(llmClient), - SessionRunnerModel.node.replace(models), - InstructionBuiltIns.node.replace(systemContext), - InstructionDiscovery.node.replace(instructionContext), - Location.node.replace(Location.boundNode({ directory: AbsolutePath.make("/project") })), - SkillInstructions.node.replace(skillInstructions), - ReferenceInstructions.node.replace(referenceInstructions), - McpInstructions.node.replace(mcpInstructions), - Config.node.replace(config), - Permission.node.replace(permission), - PluginSupervisor.node.replace(Layer.empty), - Plugin.node.replace(Layer.mock(Plugin.Service, { awaitActivation: Effect.void })), - ]) -const execution = (llmClient: Layer.Layer) => - Layer.effect( - SessionExecution.Service, - Effect.gen(function* () { - const sessionRunner = yield* SessionRunner.Service - const coordinator = yield* SessionRunCoordinator.make({ - drain: (sessionID, force) => sessionRunner.drain({ sessionID, force }).pipe(Effect.asVoid), - }) - return SessionExecution.Service.of({ - active: coordinator.active, - isActive: coordinator.isActive, - resume: coordinator.run, - wake: coordinator.wake, - interrupt: (sessionID) => coordinator.interrupt(sessionID), - awaitIdle: coordinator.awaitIdle, - }) - }), - ).pipe(Layer.provide(runnerLayer(llmClient)), Layer.orDie) -const testLayer = (llmClient: Layer.Layer) => - AppNodeBuilder.build( - LayerNode.group([ - Database.node, - Bus.node, - SessionProjector.node, - SessionStore.node, - Agent.node, - Model.node, - PluginHooks.node, - Tool.node, - SessionRunnerModel.node, - InstructionBuiltIns.node, - InstructionDiscovery.node, - SkillInstructions.node, - ReferenceInstructions.node, - Config.node, - Snapshot.node, - SessionRunnerLLM.node, - Session.node, - ]), - [ - Bus.node.replace(Bus.configured({ persist: true })), - LocationServiceMap.node.replace(promptLocationNode), - LayerNodePlatform.llmClient.replace(llmClient), - Permission.node.replace(permission), - Model.node.replace(promptModels), - SessionRunnerModel.node.replace(models), - InstructionBuiltIns.node.replace(systemContext), - InstructionDiscovery.node.replace(instructionContext), - Location.node.replace(Location.boundNode({ directory: AbsolutePath.make("/project") })), - SkillInstructions.node.replace(skillInstructions), - ReferenceInstructions.node.replace(referenceInstructions), - Config.node.replace(config), - Snapshot.node.replace(Snapshot.noopLayer), - PluginSupervisor.node.replace(Layer.empty), - Plugin.node.replace(Layer.mock(Plugin.Service, { awaitActivation: Effect.void })), - SessionExecution.node.replace(execution(llmClient)), - ], - ) const spans: Tracer.NativeSpan[] = [] const tracer = Tracer.make({ span(options) { @@ -177,55 +47,15 @@ const tracer = Tracer.make({ return span }, }) -const it = testEffect(testLayer(client).pipe(Layer.provide(Layer.succeed(Tracer.Tracer, tracer)))) +const it = testEffect( + sessionRunnerTestLayer({ llmClient: client, models }).pipe(Layer.provide(Layer.succeed(Tracer.Tracer, tracer))), +) const sessionID = Session.ID.make("ses_runner_recorded") describe("SessionRunnerLLM recorded", () => { it.effect("executes one recorded prompt through the recorded HTTP transport", () => Effect.gen(function* () { - const agents = yield* Agent.Service - const models = yield* Model.Service - const hooks = yield* PluginHooks.Service - yield* agents.transform((editor) => - editor.update(Agent.ID.make("build"), (agent) => { - agent.mode = "primary" - agent.permissions.push({ action: "execute", resource: "*", effect: "deny" }) - }), - ) - const pluginHost = host({ - agent: agentHost(agents), - model: modelHost(models), - provider: noProviders, - session: { hook: (name, callback) => hooks.register("session", name, callback) }, - }) - yield* Effect.forEach(OptimizePlugin.Plugins, (plugin) => plugin.effect(pluginHost), { discard: true }) - yield* IdentityPlugin.Plugin.effect(pluginHost) - const { db } = yield* Database.Service - yield* db - .insert(ProjectTable) - .values({ id: Project.ID.global, worktree: AbsolutePath.make("/project"), sandboxes: [] }) - .onConflictDoNothing() - .run() - .pipe(Effect.orDie) - yield* db - .insert(SessionTable) - .values({ - id: sessionID, - project_id: Project.ID.global, - slug: "test", - directory: "/project", - title: "test", - version: "test", - }) - .onConflictDoNothing() - .run() - .pipe(Effect.orDie) - const session = yield* Session.Service - const prompt = yield* session.prompt({ - sessionID, - text: "Say hello in one short sentence.", - resume: false, - }) + const { session, prompt } = yield* seedSession(sessionID) yield* session.resume(sessionID) @@ -236,6 +66,7 @@ describe("SessionRunnerLLM recorded", () => { expect(messages[1]?.type === "assistant" ? messages[1].content : []).toMatchObject([ { type: "text", text: "Hello!" }, ]) + const { db } = yield* Database.Service expect( (yield* db .select({ type: EventTable.type }) diff --git a/packages/core/test/session-runner-vercel-gateway.test.ts b/packages/core/test/session-runner-vercel-gateway.test.ts new file mode 100644 index 000000000000..1c16644d428a --- /dev/null +++ b/packages/core/test/session-runner-vercel-gateway.test.ts @@ -0,0 +1,173 @@ +import { VercelAIGateway } from "@opencode/ai/providers/vercel-ai-gateway" +import { LanguageModel } from "@opencode/ai/effect" +import { LLMClient, RequestExecutor } from "@opencode/ai/route" +import { Session } from "@opencode/core/session" +import { SessionRunnerModel } from "@opencode/core/session/runner/model" +import { SessionTransfer } from "@opencode/core/session/transfer" +import { Money } from "@opencode/schema/money" +import { Provider } from "@opencode/schema/provider" +import { describe, expect } from "bun:test" +import { Effect, Layer } from "effect" +import { HttpClient, HttpClientResponse } from "effect/http" +import { testEffect } from "./lib/effect" +import { assistant, seedSession, sessionRunnerTestLayer } from "./lib/session-runner-harness" + +const sessionID = Session.ID.make("ses_runner_vercel_gateway") + +// The Gateway bills in USD strings while the catalog is a per-million-token price. Keep them far apart +// so a test that prices from the catalog can never look like it used the billed amount. +const catalog = (model: LanguageModel) => + SessionRunnerModel.resolved(model, { + capabilities: { tools: true, input: ["text"], output: ["text"] }, + cost: [ + { + input: Money.USDPerMillionTokens.make(10), + output: Money.USDPerMillionTokens.make(50), + cache: { read: Money.USDPerMillionTokens.zero, write: Money.USDPerMillionTokens.zero }, + }, + ], + limit: { context: 200_000, output: 20_000 }, + }) + +const gateway = VercelAIGateway.configure({ apiKey: "fixture" }) +const messages = gateway.messages("anthropic/claude-sonnet-5.5") +// A route-provider alias: the catalog says `vercel` while the Gateway route still reads its own metadata key. +const alias = LanguageModel.update(messages, { provider: Provider.ID.make("vercel") }) + +const sse = (frames: ReadonlyArray) => + frames.map((frame) => `data: ${JSON.stringify(frame)}\n\n`).join("") + +const respond = (body: string) => + HttpClient.make((request) => + Effect.succeed( + HttpClientResponse.fromWeb( + request, + new Response(body, { status: 200, headers: { "content-type": "text/event-stream" } }), + ), + ), + ) + +const llmClient = (body: string) => + LLMClient.layer.pipe( + Layer.provide(RequestExecutor.layer), + Layer.provide(Layer.succeed(HttpClient.HttpClient, respond(body))), + ) + +const suite = (model: LanguageModel, body: string) => + testEffect( + sessionRunnerTestLayer( + { + llmClient: llmClient(body), + models: Layer.mock(SessionRunnerModel.Service)({ resolve: () => Effect.succeed(catalog(model)) }), + }, + SessionTransfer.node, + ), + ) + +describe("SessionRunnerLLM Vercel AI Gateway", () => { + suite( + alias, + sse([ + { + type: "message_start", + message: { + usage: { input_tokens: 1000, output_tokens: 0 }, + provider_metadata: { gateway: { generationId: "gen_01MOCKGENERATIONID" } }, + }, + }, + { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }, + { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "Hello" } }, + { type: "content_block_stop", index: 0 }, + { + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { output_tokens: 100 }, + provider_metadata: { + gateway: { + cost: "0.0207155", + routing: { finalProvider: "anthropic", resolvedProvider: "anthropic" }, + }, + }, + }, + { type: "message_stop" }, + ]), + ).effect("persists the billed Messages cost and Gateway routing", () => + Effect.gen(function* () { + const { session } = yield* seedSession(sessionID) + yield* session.resume(sessionID) + + const info = yield* session.get(sessionID) + expect(info.cost).toBe(Money.USD.make(0.0207155)) + + const message = assistant(yield* session.messages({ sessionID, order: "asc" })) + expect(message.cost).toBe(Money.USD.make(0.0207155)) + expect(message.tokens).toEqual({ input: 1000, output: 100, reasoning: 0, cache: { read: 0, write: 0 } }) + expect(message.finish).toBe("stop") + expect(message.native).toEqual({ + gateway: { + cost: "0.0207155", + generationId: "gen_01MOCKGENERATIONID", + routing: { finalProvider: "anthropic", resolvedProvider: "anthropic" }, + }, + }) + + const transfer = yield* SessionTransfer.Service + const exported = yield* transfer.export({ sessionID }) + expect(exported.info.cost).toBe(Money.USD.make(0.0207155)) + const exportedMessage = assistant(exported.messages) + expect(exportedMessage.cost).toBe(Money.USD.make(0.0207155)) + expect(exportedMessage.tokens).toEqual({ + input: 1000, + output: 100, + reasoning: 0, + cache: { read: 0, write: 0 }, + }) + expect(exportedMessage.finish).toBe("stop") + expect(exportedMessage.native).toEqual({ + gateway: { + cost: "0.0207155", + generationId: "gen_01MOCKGENERATIONID", + routing: { finalProvider: "anthropic", resolvedProvider: "anthropic" }, + }, + }) + }), + ) + + suite( + messages, + sse([ + { + type: "message_start", + message: { + usage: { input_tokens: 1000, output_tokens: 0 }, + provider_metadata: { gateway: { generationId: "gen_empty_cost" } }, + }, + }, + { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }, + { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "Hello" } }, + { type: "content_block_stop", index: 0 }, + { + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { output_tokens: 100 }, + provider_metadata: { + gateway: { cost: "", routing: { finalProvider: "vertex" } }, + }, + }, + { type: "message_stop" }, + ]), + ).effect("falls back to the catalog price when Gateway cost is empty and keeps routing", () => + Effect.gen(function* () { + const { session } = yield* seedSession(sessionID) + yield* session.resume(sessionID) + + const message = assistant(yield* session.messages({ sessionID, order: "asc" })) + expect(message.cost).toBe(Money.USD.make(0.015)) + expect(message.native?.gateway).toMatchObject({ + cost: "", + routing: { finalProvider: "vertex" }, + generationId: "gen_empty_cost", + }) + }), + ) +})