diff --git a/packages/core/src/plugin/provider.ts b/packages/core/src/plugin/provider.ts index 5fbc6796af12..8f4266d34392 100644 --- a/packages/core/src/plugin/provider.ts +++ b/packages/core/src/plugin/provider.ts @@ -19,6 +19,7 @@ import { NeonPlugin } from "./provider/neon.js" import { NvidiaPlugin } from "./provider/nvidia.js" import { OllamaPlugin } from "./provider/ollama.js" import { OpenAIPlugin } from "./provider/openai.js" +import { OpenAICompatibleLimitsPlugin } from "./provider/openai-compatible-limits.js" import { OpenAIContextCapsPlugin } from "./provider/openai-context-caps.js" import { SnowflakeCortexPlugin } from "./provider/snowflake-cortex.js" import { OpencodePlugin } from "./provider/opencode.js" @@ -54,6 +55,7 @@ export const ProviderPlugins: PluginInternal.InternalPlugin[] = [ NvidiaPlugin, OllamaPlugin, OpencodePlugin, + OpenAICompatibleLimitsPlugin, SnowflakeCortexPlugin, OpenAIPlugin, OpenAIContextCapsPlugin, diff --git a/packages/core/src/plugin/provider/local.ts b/packages/core/src/plugin/provider/local.ts index 3935461be2da..2b3f236f67d8 100644 --- a/packages/core/src/plugin/provider/local.ts +++ b/packages/core/src/plugin/provider/local.ts @@ -20,6 +20,14 @@ export type LocalModel = { readonly context?: number | null } +/** + * Provider IDs whose inventory a local discovery plugin owns. Populated by `createLocalProviderPlugin` at module + * load, so generic discovery (for example OpenAI-compatible limit probing) can leave these providers to their + * dedicated plugin instead of hitting their servers a second time. + */ +const ownedProviderIDs = new Set() +export const localProviderIDs: ReadonlySet = ownedProviderIDs + export type LocalDiscoveryClient = ReturnType export function createLocalProviderPlugin(input: { @@ -34,6 +42,7 @@ export function createLocalProviderPlugin(input: { const discovery = new Map() const discoveryLock = Semaphore.makeUnsafe(1) const suffix = input.stripPathSuffix ?? /\/v1$/ + ownedProviderIDs.add(input.providerID) return function make(origin?: string, interval: Duration.Input = "30 seconds") { return define({ diff --git a/packages/core/src/plugin/provider/openai-compatible-limits.ts b/packages/core/src/plugin/provider/openai-compatible-limits.ts new file mode 100644 index 000000000000..1199b5d217e8 --- /dev/null +++ b/packages/core/src/plugin/provider/openai-compatible-limits.ts @@ -0,0 +1,334 @@ +import { createHash } from "node:crypto" +import { isRecord } from "@opencode/util/record" +import { define } from "@opencode/plugin/effect/plugin" +import { Effect, Option, Schedule, Semaphore, Stream } from "effect" +import { HttpClient, HttpClientRequest } from "effect/http" +import type { Value as CredentialValue } from "@opencode/schema/credential" +import { Bus } from "../../bus.js" +import { Credential } from "../../credential.js" +import { IntegrationConnection } from "../../integration/connection.js" +import { Model } from "../../model.js" +import { Provider } from "../../provider.js" +import { localProviderIDs } from "./local.js" + +// Fill missing context limits for generic OpenAI-compatible providers from the +// provider's own `GET {baseURL}/models` advertisement. +// +// Custom models synthesized without a models.dev catalog entry carry the +// built-in placeholder limits, so auto-compaction fires against a guess +// instead of the real window. Catalog-backed models — bundled providers, +// config models inherited from a canonical catalog provider, integration +// models with their own cost rows, and providers a dedicated discovery +// plugin owns (LM Studio, Ollama, vLLM) — keep their real values and are +// neither probed nor filled. +// +// Auth for the probe, highest first: a configured `authorization` header, the +// provider's active connection credential (keys added via `opencode auth` or +// integrations live in connections and resolve at request time), then +// `settings.apiKey`. +// +// Precedence, highest first: +// 1. `limit` fields set in the config, applied by the config provider +// transform registered after this plugin +// 2. limits already present from models.dev or dedicated discovery plugins +// 3. the provider's advertisement, filling placeholder fields only +// +// Fail-open: an unreachable, erroring, or malformed endpoint is skipped and +// keeps its previously discovered limits; discovery never blocks the catalog. +// Discovery runs at startup, whenever the catalog or the active credential +// changes, and on the schedule below, so failed endpoints retry once a +// minute and successes refresh at their TTL instead of waiting for a catalog +// change. + +const FETCH_TIMEOUT = "4 seconds" +const CACHE_TTL_MS = 5 * 60_000 +// Failed lookups retry at most once per minute so a dead endpoint cannot slow +// every catalog rebuild behind the fetch timeout. +const FAILURE_TTL_MS = 60_000 +const OPENAI_COMPATIBLE = "@opencode/ai/providers/openai-compatible" + +export interface AdvertisedLimit { + readonly context?: number | undefined + readonly input?: number | undefined + readonly output?: number | undefined +} + +type Endpoint = { + readonly providerID: Provider.ID + readonly baseURL: string + // A configured authorization header outranks the connection credential and the settings key. + readonly authorization?: string | undefined + readonly apiKey?: string | undefined +} + +type CacheEntry = { + readonly at: number + readonly ttl: number + /** Fingerprint of the connection credential the cached fetch ran under; `""` when none was active. */ + readonly connection: string + readonly value: Record +} + +// Shared across Locations so every catalog rebuild reuses one fetch per endpoint. +const cache = new Map() + +const digest = (value: string) => createHash("sha256").update(value).digest("hex").slice(0, 16) + +// The key covers everything known synchronously: provider, URL, and the auth +// set directly in settings, so a rotated `apiKey` or `authorization` header +// never reuses the previous credential's result. The active connection's +// identity rides in the entry and is compared at discovery time. +const endpointKey = (endpoint: Endpoint) => + `${endpoint.providerID}\n${endpoint.baseURL}\n${ + endpoint.authorization ?? (endpoint.apiKey ? digest(endpoint.apiKey) : "") + }` + +const freshEntry = (endpoint: Endpoint) => { + const entry = cache.get(endpointKey(endpoint)) + return entry !== undefined && Date.now() - entry.at < entry.ttl ? entry : undefined +} + +// Model.Info's built-in limits are placeholders from catalog creation, not +// measured values, so an advertisement may replace them per field. +const placeholders: Model.Info["limit"] = Model.Info.default( + Provider.ID.make("placeholder"), + Model.ID.make("placeholder"), +).limit + +// Models.dev entries carry a release date and cost rows; models synthesized +// from config or an integration without catalog backing do not. +const fromCatalog = (model: Model.Info) => model.time.released !== 0 || model.cost.length > 0 + +function positiveInt(value: unknown): value is number { + return typeof value === "number" && Number.isInteger(value) && value > 0 +} + +/** + * Parse an OpenAI-compatible `/models` response body into per-model limits. + * Unknown shapes are ignored; only well-formed entries are returned. + */ +export function parseModelLimits(body: unknown): Record { + if (!isRecord(body) || !Array.isArray(body.data)) return {} + return Object.fromEntries( + body.data.flatMap((item) => { + if (!isRecord(item) || typeof item.id !== "string" || item.id === "") return [] + const limit = { + ...(positiveInt(item.context_length) ? { context: item.context_length } : {}), + ...(positiveInt(item.max_input_tokens) ? { input: item.max_input_tokens } : {}), + ...(positiveInt(item.max_output_tokens) ? { output: item.max_output_tokens } : {}), + } + return Object.keys(limit).length === 0 ? [] : [[item.id, limit] as const] + }), + ) +} + +/** + * Merge one advertised limit into a model's current limit. A field is filled + * only when the advertisement has it and the current value is absent, zero, or + * the built-in placeholder. + */ +export function mergeAdvertisedLimit( + current: Model.Info["limit"], + advertised: AdvertisedLimit, +): Model.Info["limit"] | undefined { + const next = { ...current } + let changed = false + for (const key of ["context", "input", "output"] as const) { + const value = advertised[key] + if (!positiveInt(value)) continue + const existing = current[key] + if (existing !== undefined && existing !== 0 && existing !== placeholders[key]) continue + next[key] = value + changed = true + } + return changed ? next : undefined +} + +const authorized = (request: HttpClientRequest.HttpClientRequest, authorization: string | undefined) => + authorization === undefined + ? request.pipe(HttpClientRequest.acceptJson) + : request.pipe(HttpClientRequest.acceptJson, HttpClientRequest.setHeader("authorization", authorization)) + +const fetchLimits = (http: HttpClient.HttpClient, baseURL: string, authorization: string | undefined) => + Effect.gen(function* () { + const response = yield* http.execute( + authorized(HttpClientRequest.get(`${baseURL.replace(/\/+$/, "")}/models`), authorization), + ) + return parseModelLimits(yield* response.json) + }).pipe(Effect.timeout(FETCH_TIMEOUT)) + +// The active connection credential materializes as a bearer token for this +// package; `external` credentials cannot, so they fall through to the +// settings key like a missing credential. +const bearerFrom = (credential: CredentialValue | undefined) => { + if (!credential) return undefined + if (credential.type === "key") return `Bearer ${credential.key}` + if (credential.type === "oauth") return `Bearer ${credential.access}` + return undefined +} + +// An empty key is absent, matching local providers, so the probe sends no bare `Bearer ` header. +const bearerKey = (apiKey: string | undefined) => (apiKey ? `Bearer ${apiKey}` : undefined) + +// Identifies which connection the cached fetch ran under. The connection's +// identity (not a rotating OAuth access token) keys the cache, plus the stored +// key value for key credentials, which can be edited in place under the same id. +const connectionFingerprint = ( + connection: Parameters[0], + credential: CredentialValue | undefined, +) => { + const identity = IntegrationConnection.key(connection) ?? "" + return credential?.type === "key" ? `${identity}:${digest(credential.key)}` : identity +} + +export const OpenAICompatibleLimitsPlugin = define({ + id: "opencode.provider.openai-compatible-limits", + effect: Effect.fn(function* (ctx) { + const bus = yield* Bus.Service + const http = HttpClient.filterStatusOk(yield* HttpClient.HttpClient) + const endpoints = new Map() + const lock = Semaphore.makeUnsafe(1) + + yield* ctx.model.transform((models) => { + endpoints.clear() + for (const item of models.provider.list()) { + // A dedicated discovery plugin owns this provider's inventory and probes + // its own endpoints; a second /models request here would be redundant. + if (localProviderIDs.has(item.provider.id)) continue + for (const [id, definition] of item.models) { + if ((definition.package ?? item.provider.package) !== OPENAI_COMPATIBLE) continue + if (fromCatalog(definition)) continue + // Model edits cannot create providers, so this also skips unavailable ones. + const current = models.get(item.provider.id, id) + if (current === undefined) continue + // The merged draft can inherit catalog values even when the provider + // record still shows a synthesized model; either provenance wins. + if (fromCatalog(current)) continue + const settings = Provider.mergeOverlay( + Provider.modelSettings(item.provider.settings), + Provider.modelSettings(definition.settings), + ) + const baseURL = settings?.baseURL + if (typeof baseURL !== "string" || baseURL === "") continue + const headers = Provider.mergeHeaders(item.provider.headers, definition.headers) + const apiKey = typeof settings?.apiKey === "string" ? settings.apiKey : undefined + const endpoint = { + providerID: item.provider.id, + baseURL, + authorization: + typeof headers?.authorization === "string" ? headers.authorization : undefined, + apiKey, + } + endpoints.set(endpointKey(endpoint), endpoint) + const advertised = freshEntry(endpoint)?.value[definition.modelID ?? id] + if (advertised === undefined) continue + models.update(item.provider.id, id, (model) => { + const merged = mergeAdvertisedLimit(model.limit, advertised) + if (merged !== undefined) model.limit = merged + }) + } + } + }) + + // Transforms are synchronous, so discovery runs beside the catalog and + // reloads it once new limits land. + const discover = Effect.fn("OpenAICompatibleLimits.discover")(function* () { + yield* lock.withPermit( + Effect.gen(function* () { + // Resolve the active connection first: an entry cached under a + // different credential (for example after `opencode auth login`) is + // stale no matter its TTL. + const stale: Array<{ + endpoint: Endpoint + connection: string + authorization: string | undefined + }> = [] + for (const endpoint of endpoints.values()) { + const connection = yield* ctx.integration.connection + .active(endpoint.providerID) + .pipe(Effect.orElseSucceed(() => undefined)) + const credential = connection + ? yield* ctx.integration.connection + .resolve(connection) + .pipe(Effect.orElseSucceed(() => undefined)) + : undefined + const fingerprint = connectionFingerprint(connection, credential) + const entry = cache.get(endpointKey(endpoint)) + if (entry !== undefined && Date.now() - entry.at < entry.ttl && entry.connection === fingerprint) + continue + stale.push({ + endpoint, + connection: fingerprint, + authorization: + endpoint.authorization ?? bearerFrom(credential) ?? bearerKey(endpoint.apiKey), + }) + } + if (stale.length === 0) return + const results = yield* Effect.all( + stale.map((item) => Effect.option(fetchLimits(http, item.endpoint.baseURL, item.authorization))), + { concurrency: "unbounded" }, + ) + let changed = false + for (const [index, item] of stale.entries()) { + const result = results[index] + const key = endpointKey(item.endpoint) + const previous = cache.get(key) + if (Option.isNone(result)) { + // Keep the last successful inventory through transient outages and retry soon. + cache.set(key, { + at: Date.now(), + ttl: FAILURE_TTL_MS, + connection: item.connection, + value: previous?.value ?? {}, + }) + continue + } + cache.set(key, { + at: Date.now(), + ttl: CACHE_TTL_MS, + connection: item.connection, + value: result.value, + }) + if (previous === undefined || !sameLimits(previous.value, result.value)) changed = true + } + if (changed) yield* ctx.model.reload() + }), + ) + }) + + // Failed endpoints retry once a minute and successes refresh at their TTL, + // on a schedule like the local provider plugins, instead of only when the + // catalog changes. + yield* discover().pipe(Effect.ignore, Effect.repeat(Schedule.spaced(FAILURE_TTL_MS)), Effect.forkScoped) + yield* bus + .subscribe([Model.Event.Updated]) + .pipe( + Stream.runForEach(() => discover().pipe(Effect.ignore)), + Effect.forkScoped({ startImmediately: true }), + ) + // A credential switch can change what /models advertises; revalidate at + // once instead of waiting out the cache TTL. + yield* bus + .subscribe(Credential.Event.Switched) + .pipe( + Stream.filter((event) => + [...endpoints.values()].some( + (endpoint) => endpoint.providerID === Provider.ID.make(event.data.integrationID), + ), + ), + Stream.runForEach(() => discover().pipe(Effect.ignore)), + Effect.forkScoped({ startImmediately: true }), + ) + }), +}) + +const sameLimits = (left: Record, right: Record) => { + const ids = Object.keys(left) + if (ids.length !== Object.keys(right).length) return false + return ids.every((id) => { + const a = left[id] + const b = right[id] + if (b === undefined) return false + return a.context === b.context && a.input === b.input && a.output === b.output + }) +} diff --git a/packages/core/test/plugin/provider-openai-compatible-limits.test.ts b/packages/core/test/plugin/provider-openai-compatible-limits.test.ts new file mode 100644 index 000000000000..9112621ab294 --- /dev/null +++ b/packages/core/test/plugin/provider-openai-compatible-limits.test.ts @@ -0,0 +1,326 @@ +import { Config } from "@opencode/core/config" +import { ConfigProviderPlugin } from "@opencode/core/config/plugin/provider" +import { Model } from "@opencode/core/model" +import { Plugin } from "@opencode/core/plugin" +import { PluginHost } from "@opencode/core/plugin/host" +import { + mergeAdvertisedLimit, + OpenAICompatibleLimitsPlugin, + parseModelLimits, +} from "@opencode/core/plugin/provider/openai-compatible-limits" +import { ProviderPlugins } from "@opencode/core/plugin/provider" +import { Provider } from "@opencode/core/provider" +import { Bus } from "@opencode/core/bus" +import { Document, type Entry, Event, Info } from "@opencode/schema/config" +import { describe, expect, test } from "bun:test" +import { Effect, Schema } from "effect" +import { testEffect } from "../lib/effect" +import { PluginTestLayer } from "./fixture" + +const it = testEffect(PluginTestLayer) +const decode = Schema.decodeUnknownSync(Info) + +const addPlugins = Effect.fn(function* (entries: Entry[]) { + const plugin = yield* Plugin.Service + const host = yield* PluginHost.make(plugin) + const config = Config.testLayer(entries) + return yield* Effect.gen(function* () { + yield* OpenAICompatibleLimitsPlugin.effect(host) + yield* ConfigProviderPlugin.Plugin.effect(host) + return yield* Config.Test + }).pipe(Effect.provide(config)) +}) + +function eventually( + effect: Effect.Effect, + predicate: (value: A) => boolean, + remaining = 3000, +): Effect.Effect { + return Effect.gen(function* () { + const value = yield* effect + if (predicate(value)) return value + if (remaining === 0) return yield* Effect.fail(new Error("Timed out waiting for value")) + yield* Effect.promise(() => Bun.sleep(1)) + return yield* eventually(effect, predicate, remaining - 1) + }) +} + +function configuration(id: string, baseURL: string, models: Record, apiKey = "secret"): Entry { + return new Document({ + type: "document", + info: decode({ + providers: { + [id]: { + name: "Proxy", + package: "@opencode/ai/providers/openai-compatible", + settings: { baseURL, apiKey }, + models, + }, + }, + }), + }) +} + +describe("OpenAICompatibleLimitsPlugin", () => { + test("is registered as a built-in provider plugin", () => { + expect(OpenAICompatibleLimitsPlugin.id).toBe("opencode.provider.openai-compatible-limits") + expect(ProviderPlugins.map((item) => item.id)).toContain("opencode.provider.openai-compatible-limits") + }) + + describe("parseModelLimits", () => { + test("reads limits from an OpenAI-compatible /models body", () => { + expect( + parseModelLimits({ + data: [ + { + id: "factory-capable", + context_length: 1_000_000, + max_input_tokens: 1_000_000, + max_output_tokens: 131_072, + }, + { id: "factory-efficient", context_length: 262_144, max_input_tokens: 230_144 }, + { id: "no-limits" }, + ], + }), + ).toEqual({ + "factory-capable": { context: 1_000_000, input: 1_000_000, output: 131_072 }, + "factory-efficient": { context: 262_144, input: 230_144 }, + }) + }) + + test("ignores malformed entries and non-array bodies", () => { + expect(parseModelLimits({ data: [{ context_length: 100 }, { id: "", context_length: 100 }, { id: "x" }] })).toEqual({}) + expect(parseModelLimits({})).toEqual({}) + expect(parseModelLimits(null)).toEqual({}) + expect(parseModelLimits({ data: [{ id: "neg", context_length: -1 }, { id: "float", context_length: 1.5 }] })).toEqual({}) + }) + }) + + describe("mergeAdvertisedLimit", () => { + test("fills placeholder and absent fields from the advertisement", () => { + expect( + mergeAdvertisedLimit( + { context: 200_000, output: 32_000 }, + { context: 1_000_000, input: 1_000_000, output: 131_072 }, + ), + ).toEqual({ context: 1_000_000, input: 1_000_000, output: 131_072 }) + expect(mergeAdvertisedLimit({ context: 0, output: 0 }, { context: 8_192 })).toEqual({ context: 8_192, output: 0 }) + }) + + test("never overwrites real limits", () => { + expect( + mergeAdvertisedLimit( + { context: 128_000, input: 120_000, output: 4_096 }, + { context: 1_000_000, input: 1_000_000, output: 131_072 }, + ), + ).toBeUndefined() + }) + + test("fills per field, keeping partial real data", () => { + expect(mergeAdvertisedLimit({ context: 128_000, output: 32_000 }, { context: 999_999, output: 131_072 })).toEqual({ + context: 128_000, + output: 131_072, + }) + }) + + test("does nothing when the advertisement has no usable fields", () => { + expect(mergeAdvertisedLimit({ context: 200_000, output: 32_000 }, {})).toBeUndefined() + }) + }) + + it.live( + "fills placeholder limits from the advertisement and keeps configured limits", + () => + Effect.acquireUseRelease( + Effect.sync(() => { + const requests: Array<{ authorization: string | null; path: string }> = [] + return { + requests, + server: Bun.serve({ + port: 0, + fetch: (request) => { + requests.push({ + authorization: request.headers.get("authorization"), + path: new URL(request.url).pathname, + }) + return Response.json({ + data: [ + { + id: "auto-model", + context_length: 1_000_000, + max_input_tokens: 1_000_000, + max_output_tokens: 131_072, + }, + { id: "manual-model", context_length: 999_999_999 }, + ], + }) + }, + }), + } + }), + ({ requests, server }) => + Effect.gen(function* () { + const modelState = yield* Model.Service + const providerID = Provider.ID.make("proxy") + yield* addPlugins([ + configuration("proxy", `${server.url.origin}/v1`, { + "auto-model": {}, + "manual-model": { limit: { context: 555_000 } }, + }), + ]) + + const auto = yield* eventually( + modelState.get(providerID, Model.ID.make("auto-model")), + (model) => model?.limit.context === 1_000_000, + ) + expect(auto?.limit).toEqual({ context: 1_000_000, input: 1_000_000, output: 131_072 }) + expect(requests).toContainEqual({ authorization: "Bearer secret", path: "/v1/models" }) + + const manual = yield* modelState.get(providerID, Model.ID.make("manual-model")) + expect(manual?.limit.context).toBe(555_000) + }), + ({ server }) => Effect.promise(() => server.stop(true)), + ), + ) + + it.live("treats an empty apiKey as absent and sends no authorization header", () => + Effect.acquireUseRelease( + Effect.sync(() => { + const authorizations: Array = [] + const server = Bun.serve({ + port: 0, + fetch: (request) => { + authorizations.push(request.headers.get("authorization")) + return Response.json({ data: [{ id: "open-model", context_length: 300_000 }] }) + }, + }) + return { authorizations, server } + }), + ({ authorizations, server }) => + Effect.gen(function* () { + const modelState = yield* Model.Service + yield* addPlugins([configuration("open-proxy", `${server.url.origin}/v1`, { "open-model": {} }, "")]) + + yield* eventually( + modelState.get(Provider.ID.make("open-proxy"), Model.ID.make("open-model")), + (model) => model?.limit.context === 300_000, + ) + expect(authorizations.length).toBeGreaterThan(0) + expect(authorizations.every((value) => value === null)).toBe(true) + }), + ({ server }) => Effect.promise(() => server.stop(true)), + ), + ) + + it.live("fails open on erroring endpoints and still applies live ones", () => + Effect.acquireUseRelease( + Effect.sync(() => { + const dead = Bun.serve({ port: 0, fetch: () => new Response("nope", { status: 500 }) }) + const live = Bun.serve({ + port: 0, + fetch: () => Response.json({ data: [{ id: "live-model", context_length: 200_001 }] }), + }) + return { dead, live } + }), + ({ dead, live }) => + Effect.gen(function* () { + const modelState = yield* Model.Service + yield* addPlugins([ + configuration("live-proxy", `${live.url.origin}/v1`, { "live-model": {} }), + configuration("dead-proxy", `${dead.url.origin}/v1`, { "dead-model": {} }), + ]) + + expect( + yield* eventually( + modelState.get(Provider.ID.make("live-proxy"), Model.ID.make("live-model")), + (model) => model?.limit.context === 200_001, + ), + ).toBeDefined() + expect( + yield* eventually( + modelState.get(Provider.ID.make("dead-proxy"), Model.ID.make("dead-model")), + (model) => model !== undefined, + ), + ).toMatchObject({ limit: { context: 200_000, output: 32_000 } }) + }), + ({ dead, live }) => Effect.promise(() => Promise.all([dead.stop(true), live.stop(true)])), + ), + ) + + it.live("does not probe providers owned by a dedicated discovery plugin", () => + Effect.acquireUseRelease( + Effect.sync(() => { + const requests: string[] = [] + const server = Bun.serve({ + port: 0, + fetch: (request) => { + requests.push(new URL(request.url).pathname) + return Response.json({ data: [{ id: "local-model", context_length: 1_000_000 }] }) + }, + }) + return { requests, server } + }), + ({ requests, server }) => + Effect.gen(function* () { + const modelState = yield* Model.Service + yield* addPlugins([configuration("vllm", `${server.url.origin}/v1`, { "local-model": {} })]) + + const model = yield* eventually( + modelState.get(Provider.ID.make("vllm"), Model.ID.make("local-model")), + (model) => model !== undefined, + ) + // LM Studio, Ollama, and vLLM keep the inventory their own plugin discovers. + yield* Effect.promise(() => Bun.sleep(200)) + expect(requests).toEqual([]) + expect(model?.limit).toMatchObject({ context: 200_000, output: 32_000 }) + }), + ({ server }) => Effect.promise(() => server.stop(true)), + ), + ) + + it.live("refetches when the settings apiKey changes instead of serving the cached result", () => + Effect.acquireUseRelease( + Effect.sync(() => { + const authorizations: Array = [] + const server = Bun.serve({ + port: 0, + fetch: (request) => { + const authorization = request.headers.get("authorization") + authorizations.push(authorization) + return Response.json({ + data: [ + { + id: "rotating-model", + context_length: authorization === "Bearer one" ? 111_111 : 222_222, + }, + ], + }) + }, + }) + return { authorizations, server } + }), + ({ authorizations, server }) => + Effect.gen(function* () { + const modelState = yield* Model.Service + const bus = yield* Bus.Service + const providerID = Provider.ID.make("rotating") + const origin = `${server.url.origin}/v1` + const models = { "rotating-model": {} } + const test = yield* addPlugins([configuration("rotating", origin, models, "one")]) + + yield* eventually( + modelState.get(providerID, Model.ID.make("rotating-model")), + (model) => model?.limit.context === 111_111, + ) + yield* test.setEntries([configuration("rotating", origin, models, "two")]) + yield* bus.publish(Event.Updated, {}) + yield* eventually( + modelState.get(providerID, Model.ID.make("rotating-model")), + (model) => model?.limit.context === 222_222, + ) + expect(authorizations).toContain("Bearer two") + }), + ({ server }) => Effect.promise(() => server.stop(true)), + ), + ) +}) diff --git a/packages/web/src/content/docs/providers.mdx b/packages/web/src/content/docs/providers.mdx index 2a37fe50a1aa..686e86caa9d3 100644 --- a/packages/web/src/content/docs/providers.mdx +++ b/packages/web/src/content/docs/providers.mdx @@ -2667,6 +2667,31 @@ Configuration details: The `limit` fields allow OpenCode to understand how much context you have left. Standard providers pull these from models.dev automatically. +##### Automatic limit discovery + +For custom `@ai-sdk/openai-compatible` providers, OpenCode reads context limits +directly from the provider: it requests `GET {baseURL}/models` and fills in +`context`, `input`, and `output` for any model that advertises +`context_length`, `max_input_tokens`, or `max_output_tokens`, fields that many +OpenAI-compatible servers already publish on their model list. Without them, a +custom model keeps OpenCode's default context guess, so auto-compaction fires +at the wrong point. + +Precedence, highest first: + +1. The `limit` you set in the config, per field +2. Values from models.dev or dedicated provider discovery +3. The provider's advertisement, which only replaces the default placeholders + +Discovery fails open: if the endpoint is unreachable, errors, or returns +malformed data, OpenCode keeps whatever limits it already has and never blocks +startup. Failed lookups retry about once a minute and successful ones refresh +every five minutes, so an endpoint that was down when OpenCode started catches +up on its own. Providers with their own discovery integration (LM Studio, +Ollama, vLLM) are left to that integration and are never probed here. A manual +`limit` in the config is still the right choice when the server does not +advertise limits or advertises wrong ones. + --- ## Troubleshooting