Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 81 additions & 8 deletions packages/ai/src/protocols/utils/gateway-protocol.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,19 @@
import { Effect, Option, Schema } from "effect"
import { Effect, Option, Schema, SchemaGetter } from "effect"
import { Protocol } from "../../route/protocol.js"
import { LLMEvent, mergeJsonRecords, type AIError, type LLMRequest } from "../../schema/index.js"
import {
LLMEvent,
Usage,
mergeJsonRecords,
type AIError,
type LLMRequest,
type ProviderMetadata,
} from "../../schema/index.js"
import { JsonObject, lenient } from "../shared.js"

interface ParserState<Inner> {
readonly inner: Inner
readonly gateway?: Record<string, unknown>
readonly providerMetadataKey: string
}

const GatewayHolder = Schema.Struct({
Expand All @@ -18,6 +26,7 @@ const GatewayHolder = Schema.Struct({

const GatewayEvent = Schema.Struct({
...GatewayHolder.fields,
message: lenient(GatewayHolder),
response: lenient(GatewayHolder),
choices: lenient(
Schema.Array(
Expand All @@ -29,14 +38,70 @@ const GatewayEvent = Schema.Struct({
})
const decodeGatewayEvent = Schema.decodeUnknownOption(GatewayEvent)

// Gateway bills /v1/messages and /v1/responses as a decimal string. Trimming and matching before Number()
// keeps "" and " " from coercing to 0, which SessionUsage.record would trust as an authoritative free request.
const NonNegativeFinite = Schema.Finite.check(Schema.isGreaterThanOrEqualTo(0))
const DecimalCost = Schema.Trim.check(
Schema.isPattern(/^(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?$/),
).pipe(
Schema.decodeTo(NonNegativeFinite, {
decode: SchemaGetter.transform((value) => Number(value)),
encode: SchemaGetter.transform((value) => String(value)),
}),
)
const GatewayBilling = Schema.Struct({
cost: lenient(Schema.Union([NonNegativeFinite, DecimalCost])),
})
const decodeGatewayBilling = Schema.decodeUnknownOption(GatewayBilling)

// A terminal frame can arrive without token counts; the billed amount is still authoritative, so it becomes a
// usage carrying only `cost`. Rebuilding through `new Usage` keeps the `visibleOutputTokens` getter the spread drops.
function withGatewayCost(usage: Usage | undefined, cost: number | undefined): Usage | undefined {
if (cost === undefined || usage?.cost !== undefined) return usage
return new Usage(usage === undefined ? { cost } : { ...usage, cost })
}

// The Gateway namespaces its metadata under `gateway` on the wire, while core reads the route's
// `providerMetadataKey`. Emit both so either reader finds the same generation and billing data.
function mergeGatewayMetadata(
metadata: ProviderMetadata | undefined,
gateway: Record<string, unknown>,
providerMetadataKey: string,
): ProviderMetadata {
const merged: ProviderMetadata = {
...metadata,
gateway: mergeJsonRecords(metadata?.["gateway"], gateway) ?? gateway,
}
if (providerMetadataKey === "gateway") return merged
return {
...merged,
[providerMetadataKey]: mergeJsonRecords(metadata?.[providerMetadataKey], { gateway }) ?? { gateway },
}
}

function attachGatewayMetadata(
events: ReadonlyArray<LLMEvent>,
gateway: Record<string, unknown> | undefined,
providerMetadataKey: string,
): ReadonlyArray<LLMEvent> {
if (!gateway || !events.some(LLMEvent.is.finish)) return events
return events.map((event) =>
LLMEvent.is.finish(event) ? { ...event, providerMetadata: { ...event.providerMetadata, gateway } } : event,
)
if (!gateway) return events
if (!events.some((event) => LLMEvent.is.stepFinish(event) || LLMEvent.is.finish(event))) return events
const cost = Option.getOrUndefined(decodeGatewayBilling(gateway))?.cost
return events.map((event) => {
if (LLMEvent.is.stepFinish(event))
return LLMEvent.stepFinish({
...event,
usage: withGatewayCost(event.usage, cost),
providerMetadata: mergeGatewayMetadata(event.providerMetadata, gateway, providerMetadataKey),
})
if (LLMEvent.is.finish(event))
return LLMEvent.finish({
...event,
usage: withGatewayCost(event.usage, cost),
providerMetadata: mergeGatewayMetadata(event.providerMetadata, gateway, providerMetadataKey),
})
return event
})
}

export function gatewayProtocol<Body, Event, State>(
Expand All @@ -50,6 +115,7 @@ export function gatewayProtocol<Body, Event, State>(
) {
const initial = (request: LLMRequest): ParserState<State> => ({
inner: protocol.stream.initial(request),
providerMetadataKey: request.model.route.providerMetadataKey ?? request.model.provider,
})
const onHalt = protocol.stream.onHalt

Expand All @@ -75,16 +141,23 @@ export function gatewayProtocol<Body, Event, State>(
? mergeJsonRecords(
state.gateway,
decoded.provider_metadata?.gateway,
decoded.message?.provider_metadata?.gateway,
decoded.response?.provider_metadata?.gateway,
...(decoded.choices ?? []).map((choice) => choice.delta?.provider_metadata?.gateway),
)
: state.gateway
const [inner, events] = yield* protocol.stream.step(state.inner, event)
return [{ inner, gateway }, attachGatewayMetadata(events, gateway)] as const
return [
{ inner, gateway, providerMetadataKey: state.providerMetadataKey },
attachGatewayMetadata(events, gateway, state.providerMetadataKey),
] as const
}),
terminal: protocol.stream.terminal,
onHalt: onHalt
? (state) => onHalt(state.inner).pipe(Effect.map((events) => attachGatewayMetadata(events, state.gateway)))
? (state) =>
onHalt(state.inner).pipe(
Effect.map((events) => attachGatewayMetadata(events, state.gateway, state.providerMetadataKey)),
)
: undefined,
},
})
Expand Down
11 changes: 11 additions & 0 deletions packages/ai/test/provider/vercel-ai-gateway.recorded.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,17 @@ for (const item of [
expect(events.filter(LLMEvent.is.finish).every((event) => event.providerMetadata?.gateway !== undefined)).toBe(
true,
)
const stepFinishes = events.filter(LLMEvent.is.stepFinish)
expect(stepFinishes.length).toBeGreaterThan(0)
expect(
stepFinishes.every(
(event) =>
event.providerMetadata?.gateway !== undefined &&
event.providerMetadata?.["vercel-ai-gateway"]?.gateway !== undefined &&
typeof event.usage?.cost === "number" &&
event.usage.cost > 0,
),
).toBe(true)
if (item.reasoning) expect(events.some(LLMEvent.is.reasoningEnd)).toBe(true)
}),
120_000,
Expand Down
Loading
Loading