From 1aa788b085a57499cbe404141cb74cb118fbd189 Mon Sep 17 00:00:00 2001 From: Lance Woodson <65194841+woodsonl@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:38:37 -0500 Subject: [PATCH 1/4] feat(core): discover context limits for custom OpenAI-compatible providers Custom OpenAI-compatible providers are not in the models.dev catalog, so their models keep the built-in placeholder limits and auto-compaction fires against a guess instead of the real window. Request GET {baseURL}/models once per endpoint and fill context, input, and output from the advertised context_length, max_input_tokens, and max_output_tokens when those fields are still placeholders. Configured limits win per field through the config provider transform that runs after this plugin, and limits from models.dev or dedicated discovery plugins are never replaced. Discovery fails open: an unreachable or malformed endpoint keeps the last known limits and never blocks the catalog. Ports #53236 to V2. --- packages/core/src/plugin/provider.ts | 2 + .../provider/openai-compatible-limits.ts | 205 +++++++++++++++++ .../provider-openai-compatible-limits.test.ts | 216 ++++++++++++++++++ packages/web/src/content/docs/providers.mdx | 21 ++ 4 files changed, 444 insertions(+) create mode 100644 packages/core/src/plugin/provider/openai-compatible-limits.ts create mode 100644 packages/core/test/plugin/provider-openai-compatible-limits.test.ts diff --git a/packages/core/src/plugin/provider.ts b/packages/core/src/plugin/provider.ts index 5fbc6796af12..8f4266d34392 100644 --- a/packages/core/src/plugin/provider.ts +++ b/packages/core/src/plugin/provider.ts @@ -19,6 +19,7 @@ import { NeonPlugin } from "./provider/neon.js" import { NvidiaPlugin } from "./provider/nvidia.js" import { OllamaPlugin } from "./provider/ollama.js" import { OpenAIPlugin } from "./provider/openai.js" +import { OpenAICompatibleLimitsPlugin } from "./provider/openai-compatible-limits.js" import { OpenAIContextCapsPlugin } from "./provider/openai-context-caps.js" import { SnowflakeCortexPlugin } from "./provider/snowflake-cortex.js" import { OpencodePlugin } from "./provider/opencode.js" @@ -54,6 +55,7 @@ export const ProviderPlugins: PluginInternal.InternalPlugin[] = [ NvidiaPlugin, OllamaPlugin, OpencodePlugin, + OpenAICompatibleLimitsPlugin, SnowflakeCortexPlugin, OpenAIPlugin, OpenAIContextCapsPlugin, diff --git a/packages/core/src/plugin/provider/openai-compatible-limits.ts b/packages/core/src/plugin/provider/openai-compatible-limits.ts new file mode 100644 index 000000000000..8203bac4d235 --- /dev/null +++ b/packages/core/src/plugin/provider/openai-compatible-limits.ts @@ -0,0 +1,205 @@ +import { isRecord } from "@opencode/util/record" +import { define } from "@opencode/plugin/effect/plugin" +import { Effect, Option, Semaphore, Stream } from "effect" +import { HttpClient, HttpClientRequest } from "effect/http" +import { Bus } from "../../bus.js" +import { Model } from "../../model.js" +import { Provider } from "../../provider.js" + +// Fill missing context limits for generic OpenAI-compatible providers from the +// provider's own `GET {baseURL}/models` advertisement. +// +// Custom providers on this package are not in the models.dev catalog, so their +// models carry the built-in placeholder limits and auto-compaction fires +// against a guess instead of the real window. Servers commonly publish +// `context_length` / `max_input_tokens` / `max_output_tokens` per model on +// `/models`; discovery fills the placeholder fields from that response. +// +// Precedence, highest first: +// 1. `limit` fields set in the config, applied by the config provider +// transform registered after this plugin +// 2. limits already present from models.dev or dedicated discovery plugins +// 3. the provider's advertisement, filling placeholder fields only +// +// Fail-open: an unreachable, erroring, or malformed endpoint is skipped and +// keeps its previously discovered limits; discovery never blocks the catalog. + +const FETCH_TIMEOUT = "4 seconds" +const CACHE_TTL_MS = 5 * 60_000 +// Failed lookups retry at most once per minute so a dead endpoint cannot slow +// every catalog rebuild behind the fetch timeout. +const FAILURE_TTL_MS = 60_000 +const OPENAI_COMPATIBLE = "@opencode/ai/providers/openai-compatible" + +export interface AdvertisedLimit { + readonly context?: number | undefined + readonly input?: number | undefined + readonly output?: number | undefined +} + +type Endpoint = { + readonly baseURL: string + readonly authorization?: string | undefined +} + +type CacheEntry = { + readonly at: number + readonly ttl: number + readonly value: Record +} + +// Shared across Locations so every catalog rebuild reuses one fetch per endpoint. +const cache = new Map() + +const endpointKey = (endpoint: Endpoint) => `${endpoint.baseURL}\n${endpoint.authorization ?? ""}` + +const freshEntry = (endpoint: Endpoint) => { + const entry = cache.get(endpointKey(endpoint)) + return entry !== undefined && Date.now() - entry.at < entry.ttl ? entry : undefined +} + +// Model.Info's built-in limits are placeholders from catalog creation, not +// measured values, so an advertisement may replace them per field. +const placeholders: Model.Info["limit"] = Model.Info.default( + Provider.ID.make("placeholder"), + Model.ID.make("placeholder"), +).limit + +function positiveInt(value: unknown): value is number { + return typeof value === "number" && Number.isInteger(value) && value > 0 +} + +/** + * Parse an OpenAI-compatible `/models` response body into per-model limits. + * Unknown shapes are ignored; only well-formed entries are returned. + */ +export function parseModelLimits(body: unknown): Record { + if (!isRecord(body) || !Array.isArray(body.data)) return {} + return Object.fromEntries( + body.data.flatMap((item) => { + if (!isRecord(item) || typeof item.id !== "string" || item.id === "") return [] + const limit = { + ...(positiveInt(item.context_length) ? { context: item.context_length } : {}), + ...(positiveInt(item.max_input_tokens) ? { input: item.max_input_tokens } : {}), + ...(positiveInt(item.max_output_tokens) ? { output: item.max_output_tokens } : {}), + } + return Object.keys(limit).length === 0 ? [] : [[item.id, limit] as const] + }), + ) +} + +/** + * Merge one advertised limit into a model's current limit. A field is filled + * only when the advertisement has it and the current value is absent, zero, or + * the built-in placeholder. + */ +export function mergeAdvertisedLimit( + current: Model.Info["limit"], + advertised: AdvertisedLimit, +): Model.Info["limit"] | undefined { + const next = { ...current } + let changed = false + for (const key of ["context", "input", "output"] as const) { + const value = advertised[key] + if (!positiveInt(value)) continue + const existing = current[key] + if (existing !== undefined && existing !== 0 && existing !== placeholders[key]) continue + next[key] = value + changed = true + } + return changed ? next : undefined +} + +const authorized = (request: HttpClientRequest.HttpClientRequest, authorization: string | undefined) => + authorization === undefined + ? request.pipe(HttpClientRequest.acceptJson) + : request.pipe(HttpClientRequest.acceptJson, HttpClientRequest.setHeader("authorization", authorization)) + +const fetchLimits = (http: HttpClient.HttpClient, endpoint: Endpoint) => + Effect.gen(function* () { + const response = yield* http.execute( + authorized(HttpClientRequest.get(`${endpoint.baseURL.replace(/\/+$/, "")}/models`), endpoint.authorization), + ) + return parseModelLimits(yield* response.json) + }).pipe(Effect.timeout(FETCH_TIMEOUT)) + +export const OpenAICompatibleLimitsPlugin = define({ + id: "opencode.provider.openai-compatible-limits", + effect: Effect.fn(function* (ctx) { + const bus = yield* Bus.Service + const http = HttpClient.filterStatusOk(yield* HttpClient.HttpClient) + const endpoints = new Map() + const lock = Semaphore.makeUnsafe(1) + + yield* ctx.model.transform((models) => { + endpoints.clear() + for (const item of models.provider.list()) { + for (const [id, definition] of item.models) { + if ((definition.package ?? item.provider.package) !== OPENAI_COMPATIBLE) continue + // Model edits cannot create providers, so this also skips unavailable ones. + if (models.get(item.provider.id, id) === undefined) continue + const settings = Provider.mergeOverlay( + Provider.modelSettings(item.provider.settings), + Provider.modelSettings(definition.settings), + ) + const baseURL = settings?.baseURL + if (typeof baseURL !== "string" || baseURL === "") continue + const headers = Provider.mergeHeaders(item.provider.headers, definition.headers) + const apiKey = typeof settings?.apiKey === "string" ? settings.apiKey : undefined + const endpoint = { + baseURL, + authorization: + (typeof headers?.authorization === "string" ? headers.authorization : undefined) ?? + (apiKey === undefined ? undefined : `Bearer ${apiKey}`), + } + endpoints.set(endpointKey(endpoint), endpoint) + const advertised = freshEntry(endpoint)?.value[definition.modelID ?? id] + if (advertised === undefined) continue + models.update(item.provider.id, id, (model) => { + const merged = mergeAdvertisedLimit(model.limit, advertised) + if (merged !== undefined) model.limit = merged + }) + } + } + }) + + // Transforms are synchronous, so discovery runs beside the catalog and + // reloads it once new limits land. + const discover = Effect.fn("OpenAICompatibleLimits.discover")(function* () { + yield* lock.withPermit( + Effect.gen(function* () { + const stale = [...endpoints.values()].filter((endpoint) => freshEntry(endpoint) === undefined) + const results = yield* Effect.all( + stale.map((endpoint) => Effect.option(fetchLimits(http, endpoint))), + { concurrency: "unbounded" }, + ) + let changed = false + for (const [index, endpoint] of stale.entries()) { + const result = results[index] + const previous = cache.get(endpointKey(endpoint)) + if (Option.isNone(result)) { + // Keep the last successful inventory through transient outages and retry soon. + cache.set(endpointKey(endpoint), { + at: Date.now(), + ttl: FAILURE_TTL_MS, + value: previous?.value ?? {}, + }) + continue + } + cache.set(endpointKey(endpoint), { at: Date.now(), ttl: CACHE_TTL_MS, value: result.value }) + changed = true + } + if (changed) yield* ctx.model.reload() + }), + ) + }) + + yield* discover().pipe(Effect.ignore, Effect.forkScoped) + yield* bus + .subscribe([Model.Event.Updated]) + .pipe( + Stream.runForEach(() => discover().pipe(Effect.ignore)), + Effect.forkScoped({ startImmediately: true }), + ) + }), +}) diff --git a/packages/core/test/plugin/provider-openai-compatible-limits.test.ts b/packages/core/test/plugin/provider-openai-compatible-limits.test.ts new file mode 100644 index 000000000000..d04b31e67f7f --- /dev/null +++ b/packages/core/test/plugin/provider-openai-compatible-limits.test.ts @@ -0,0 +1,216 @@ +import { Config } from "@opencode/core/config" +import { ConfigProviderPlugin } from "@opencode/core/config/plugin/provider" +import { Model } from "@opencode/core/model" +import { Plugin } from "@opencode/core/plugin" +import { PluginHost } from "@opencode/core/plugin/host" +import { + mergeAdvertisedLimit, + OpenAICompatibleLimitsPlugin, + parseModelLimits, +} from "@opencode/core/plugin/provider/openai-compatible-limits" +import { ProviderPlugins } from "@opencode/core/plugin/provider" +import { Provider } from "@opencode/core/provider" +import { Document, type Entry, Info } from "@opencode/schema/config" +import { describe, expect, test } from "bun:test" +import { Effect, Schema } from "effect" +import { testEffect } from "../lib/effect" +import { PluginTestLayer } from "./fixture" + +const it = testEffect(PluginTestLayer) +const decode = Schema.decodeUnknownSync(Info) + +const addPlugins = Effect.fn(function* (entries: Entry[]) { + const plugin = yield* Plugin.Service + const host = yield* PluginHost.make(plugin) + const config = Config.testLayer(entries) + yield* OpenAICompatibleLimitsPlugin.effect(host).pipe(Effect.provide(config)) + yield* ConfigProviderPlugin.Plugin.effect(host).pipe(Effect.provide(config)) +}) + +function eventually( + effect: Effect.Effect, + predicate: (value: A) => boolean, + remaining = 3000, +): Effect.Effect { + return Effect.gen(function* () { + const value = yield* effect + if (predicate(value)) return value + if (remaining === 0) return yield* Effect.fail(new Error("Timed out waiting for value")) + yield* Effect.promise(() => Bun.sleep(1)) + return yield* eventually(effect, predicate, remaining - 1) + }) +} + +function configuration(id: string, baseURL: string, models: Record): Entry { + return new Document({ + type: "document", + info: decode({ + providers: { + [id]: { + name: "Proxy", + package: "@opencode/ai/providers/openai-compatible", + settings: { baseURL, apiKey: "secret" }, + models, + }, + }, + }), + }) +} + +describe("OpenAICompatibleLimitsPlugin", () => { + test("is registered as a built-in provider plugin", () => { + expect(OpenAICompatibleLimitsPlugin.id).toBe("opencode.provider.openai-compatible-limits") + expect(ProviderPlugins.map((item) => item.id)).toContain("opencode.provider.openai-compatible-limits") + }) + + describe("parseModelLimits", () => { + test("reads limits from an OpenAI-compatible /models body", () => { + expect( + parseModelLimits({ + data: [ + { + id: "factory-capable", + context_length: 1_000_000, + max_input_tokens: 1_000_000, + max_output_tokens: 131_072, + }, + { id: "factory-efficient", context_length: 262_144, max_input_tokens: 230_144 }, + { id: "no-limits" }, + ], + }), + ).toEqual({ + "factory-capable": { context: 1_000_000, input: 1_000_000, output: 131_072 }, + "factory-efficient": { context: 262_144, input: 230_144 }, + }) + }) + + test("ignores malformed entries and non-array bodies", () => { + expect(parseModelLimits({ data: [{ context_length: 100 }, { id: "", context_length: 100 }, { id: "x" }] })).toEqual({}) + expect(parseModelLimits({})).toEqual({}) + expect(parseModelLimits(null)).toEqual({}) + expect(parseModelLimits({ data: [{ id: "neg", context_length: -1 }, { id: "float", context_length: 1.5 }] })).toEqual({}) + }) + }) + + describe("mergeAdvertisedLimit", () => { + test("fills placeholder and absent fields from the advertisement", () => { + expect( + mergeAdvertisedLimit( + { context: 200_000, output: 32_000 }, + { context: 1_000_000, input: 1_000_000, output: 131_072 }, + ), + ).toEqual({ context: 1_000_000, input: 1_000_000, output: 131_072 }) + expect(mergeAdvertisedLimit({ context: 0, output: 0 }, { context: 8_192 })).toEqual({ context: 8_192, output: 0 }) + }) + + test("never overwrites real limits", () => { + expect( + mergeAdvertisedLimit( + { context: 128_000, input: 120_000, output: 4_096 }, + { context: 1_000_000, input: 1_000_000, output: 131_072 }, + ), + ).toBeUndefined() + }) + + test("fills per field, keeping partial real data", () => { + expect(mergeAdvertisedLimit({ context: 128_000, output: 32_000 }, { context: 999_999, output: 131_072 })).toEqual({ + context: 128_000, + output: 131_072, + }) + }) + + test("does nothing when the advertisement has no usable fields", () => { + expect(mergeAdvertisedLimit({ context: 200_000, output: 32_000 }, {})).toBeUndefined() + }) + }) + + it.live( + "fills placeholder limits from the advertisement and keeps configured limits", + () => + Effect.acquireUseRelease( + Effect.sync(() => { + const requests: Array<{ authorization: string | null; path: string }> = [] + return { + requests, + server: Bun.serve({ + port: 0, + fetch: (request) => { + requests.push({ + authorization: request.headers.get("authorization"), + path: new URL(request.url).pathname, + }) + return Response.json({ + data: [ + { + id: "auto-model", + context_length: 1_000_000, + max_input_tokens: 1_000_000, + max_output_tokens: 131_072, + }, + { id: "manual-model", context_length: 999_999_999 }, + ], + }) + }, + }), + } + }), + ({ requests, server }) => + Effect.gen(function* () { + const modelState = yield* Model.Service + const providerID = Provider.ID.make("proxy") + yield* addPlugins([ + configuration("proxy", `${server.url.origin}/v1`, { + "auto-model": {}, + "manual-model": { limit: { context: 555_000 } }, + }), + ]) + + const auto = yield* eventually( + modelState.get(providerID, Model.ID.make("auto-model")), + (model) => model?.limit.context === 1_000_000, + ) + expect(auto?.limit).toEqual({ context: 1_000_000, input: 1_000_000, output: 131_072 }) + expect(requests).toContainEqual({ authorization: "Bearer secret", path: "/v1/models" }) + + const manual = yield* modelState.get(providerID, Model.ID.make("manual-model")) + expect(manual?.limit.context).toBe(555_000) + }), + ({ server }) => Effect.promise(() => server.stop(true)), + ), + ) + + it.live("fails open on erroring endpoints and still applies live ones", () => + Effect.acquireUseRelease( + Effect.sync(() => { + const dead = Bun.serve({ port: 0, fetch: () => new Response("nope", { status: 500 }) }) + const live = Bun.serve({ + port: 0, + fetch: () => Response.json({ data: [{ id: "live-model", context_length: 200_001 }] }), + }) + return { dead, live } + }), + ({ dead, live }) => + Effect.gen(function* () { + const modelState = yield* Model.Service + yield* addPlugins([ + configuration("live-proxy", `${live.url.origin}/v1`, { "live-model": {} }), + configuration("dead-proxy", `${dead.url.origin}/v1`, { "dead-model": {} }), + ]) + + expect( + yield* eventually( + modelState.get(Provider.ID.make("live-proxy"), Model.ID.make("live-model")), + (model) => model?.limit.context === 200_001, + ), + ).toBeDefined() + expect( + yield* eventually( + modelState.get(Provider.ID.make("dead-proxy"), Model.ID.make("dead-model")), + (model) => model !== undefined, + ), + ).toMatchObject({ limit: { context: 200_000, output: 32_000 } }) + }), + ({ dead, live }) => Effect.promise(() => Promise.all([dead.stop(true), live.stop(true)])), + ), + ) +}) diff --git a/packages/web/src/content/docs/providers.mdx b/packages/web/src/content/docs/providers.mdx index 2a37fe50a1aa..6196dc642365 100644 --- a/packages/web/src/content/docs/providers.mdx +++ b/packages/web/src/content/docs/providers.mdx @@ -2667,6 +2667,27 @@ Configuration details: The `limit` fields allow OpenCode to understand how much context you have left. Standard providers pull these from models.dev automatically. +##### Automatic limit discovery + +For custom `@ai-sdk/openai-compatible` providers, OpenCode reads context limits +directly from the provider: it requests `GET {baseURL}/models` and fills in +`context`, `input`, and `output` for any model that advertises +`context_length`, `max_input_tokens`, or `max_output_tokens`, fields that many +OpenAI-compatible servers already publish on their model list. Without them, a +custom model keeps OpenCode's default context guess, so auto-compaction fires +at the wrong point. + +Precedence, highest first: + +1. The `limit` you set in the config, per field +2. Values from models.dev or dedicated provider discovery +3. The provider's advertisement, which only replaces the default placeholders + +Discovery fails open: if the endpoint is unreachable, errors, or returns +malformed data, OpenCode keeps whatever limits it already has and never blocks +startup. A manual `limit` in the config is still the right choice when the +server does not advertise limits or advertises wrong ones. + --- ## Troubleshooting From 1ba70362a331cd258c1f58f3b71e738fe8e83bf7 Mon Sep 17 00:00:00 2001 From: Lance Woodson <65194841+woodsonl@users.noreply.github.com> Date: Sun, 11 Oct 2026 11:21:46 -0500 Subject: [PATCH 2/4] fix(core): scope limit discovery to uncataloged models and authorize via connections Discovery matched every provider on the openai-compatible package, so bundled catalog providers (zen free models, models.dev entries mapped to the generic package) were probed on every startup. It also authorized only from settings.apiKey or a configured header, but v2 keys live in connections and resolve at request time, so connection-authenticated gateways got a 401 and never filled limits. Eligible models are now those synthesized without a catalog entry: no release date and no cost rows. Catalog-backed models keep their real values. This also retires the placeholder-value comparison as the only guard: 200000/32000 is a real catalog value for some models, so an equality check cannot tell a default from data. Authorization resolves the provider's active connection credential through ctx.integration.connection, behind a configured authorization header and ahead of settings.apiKey. --- .../provider/openai-compatible-limits.ts | 78 ++++++++++++++++--- 1 file changed, 66 insertions(+), 12 deletions(-) diff --git a/packages/core/src/plugin/provider/openai-compatible-limits.ts b/packages/core/src/plugin/provider/openai-compatible-limits.ts index 8203bac4d235..dc8639609db4 100644 --- a/packages/core/src/plugin/provider/openai-compatible-limits.ts +++ b/packages/core/src/plugin/provider/openai-compatible-limits.ts @@ -2,6 +2,7 @@ import { isRecord } from "@opencode/util/record" import { define } from "@opencode/plugin/effect/plugin" import { Effect, Option, Semaphore, Stream } from "effect" import { HttpClient, HttpClientRequest } from "effect/http" +import type { Value as CredentialValue } from "@opencode/schema/credential" import { Bus } from "../../bus.js" import { Model } from "../../model.js" import { Provider } from "../../provider.js" @@ -9,11 +10,17 @@ import { Provider } from "../../provider.js" // Fill missing context limits for generic OpenAI-compatible providers from the // provider's own `GET {baseURL}/models` advertisement. // -// Custom providers on this package are not in the models.dev catalog, so their -// models carry the built-in placeholder limits and auto-compaction fires -// against a guess instead of the real window. Servers commonly publish -// `context_length` / `max_input_tokens` / `max_output_tokens` per model on -// `/models`; discovery fills the placeholder fields from that response. +// Custom models synthesized without a models.dev catalog entry carry the +// built-in placeholder limits, so auto-compaction fires against a guess +// instead of the real window. Catalog-backed models — bundled providers, +// config models inherited from a canonical catalog provider, and integration +// models with their own inventory — keep their real values and are neither +// probed nor filled. +// +// Auth for the probe, highest first: a configured `authorization` header, the +// provider's active connection credential (keys added via `opencode auth` or +// integrations live in connections and resolve at request time), then +// `settings.apiKey`. // // Precedence, highest first: // 1. `limit` fields set in the config, applied by the config provider @@ -38,8 +45,11 @@ export interface AdvertisedLimit { } type Endpoint = { + readonly providerID: Provider.ID readonly baseURL: string + // A configured authorization header outranks the connection credential and the settings key. readonly authorization?: string | undefined + readonly apiKey?: string | undefined } type CacheEntry = { @@ -51,7 +61,7 @@ type CacheEntry = { // Shared across Locations so every catalog rebuild reuses one fetch per endpoint. const cache = new Map() -const endpointKey = (endpoint: Endpoint) => `${endpoint.baseURL}\n${endpoint.authorization ?? ""}` +const endpointKey = (endpoint: Endpoint) => `${endpoint.providerID}\n${endpoint.baseURL}` const freshEntry = (endpoint: Endpoint) => { const entry = cache.get(endpointKey(endpoint)) @@ -65,6 +75,10 @@ const placeholders: Model.Info["limit"] = Model.Info.default( Model.ID.make("placeholder"), ).limit +// Models.dev entries carry a release date and cost rows; models synthesized +// from config or an integration without catalog backing do not. +const fromCatalog = (model: Model.Info) => model.time.released !== 0 || model.cost.length > 0 + function positiveInt(value: unknown): value is number { return typeof value === "number" && Number.isInteger(value) && value > 0 } @@ -115,14 +129,26 @@ const authorized = (request: HttpClientRequest.HttpClientRequest, authorization: ? request.pipe(HttpClientRequest.acceptJson) : request.pipe(HttpClientRequest.acceptJson, HttpClientRequest.setHeader("authorization", authorization)) -const fetchLimits = (http: HttpClient.HttpClient, endpoint: Endpoint) => +const fetchLimits = (http: HttpClient.HttpClient, baseURL: string, authorization: string | undefined) => Effect.gen(function* () { const response = yield* http.execute( - authorized(HttpClientRequest.get(`${endpoint.baseURL.replace(/\/+$/, "")}/models`), endpoint.authorization), + authorized(HttpClientRequest.get(`${baseURL.replace(/\/+$/, "")}/models`), authorization), ) return parseModelLimits(yield* response.json) }).pipe(Effect.timeout(FETCH_TIMEOUT)) +// The active connection credential materializes as a bearer token for this +// package; `external` credentials cannot, so they fall through to the +// settings key like a missing credential. +const bearerFrom = (credential: CredentialValue | undefined) => { + if (!credential) return undefined + if (credential.type === "key") return `Bearer ${credential.key}` + if (credential.type === "oauth") return `Bearer ${credential.access}` + return undefined +} + +const bearerKey = (apiKey: string | undefined) => (apiKey === undefined ? undefined : `Bearer ${apiKey}`) + export const OpenAICompatibleLimitsPlugin = define({ id: "opencode.provider.openai-compatible-limits", effect: Effect.fn(function* (ctx) { @@ -136,8 +162,13 @@ export const OpenAICompatibleLimitsPlugin = define({ for (const item of models.provider.list()) { for (const [id, definition] of item.models) { if ((definition.package ?? item.provider.package) !== OPENAI_COMPATIBLE) continue + if (fromCatalog(definition)) continue // Model edits cannot create providers, so this also skips unavailable ones. - if (models.get(item.provider.id, id) === undefined) continue + const current = models.get(item.provider.id, id) + if (current === undefined) continue + // The merged draft can inherit catalog values even when the provider + // record still shows a synthesized model; either provenance wins. + if (fromCatalog(current)) continue const settings = Provider.mergeOverlay( Provider.modelSettings(item.provider.settings), Provider.modelSettings(definition.settings), @@ -147,10 +178,11 @@ export const OpenAICompatibleLimitsPlugin = define({ const headers = Provider.mergeHeaders(item.provider.headers, definition.headers) const apiKey = typeof settings?.apiKey === "string" ? settings.apiKey : undefined const endpoint = { + providerID: item.provider.id, baseURL, authorization: - (typeof headers?.authorization === "string" ? headers.authorization : undefined) ?? - (apiKey === undefined ? undefined : `Bearer ${apiKey}`), + typeof headers?.authorization === "string" ? headers.authorization : undefined, + apiKey, } endpoints.set(endpointKey(endpoint), endpoint) const advertised = freshEntry(endpoint)?.value[definition.modelID ?? id] @@ -170,7 +202,29 @@ export const OpenAICompatibleLimitsPlugin = define({ Effect.gen(function* () { const stale = [...endpoints.values()].filter((endpoint) => freshEntry(endpoint) === undefined) const results = yield* Effect.all( - stale.map((endpoint) => Effect.option(fetchLimits(http, endpoint))), + stale.map((endpoint) => + Effect.option( + ctx.integration.connection + .active(endpoint.providerID) + .pipe( + Effect.flatMap((connection) => + connection + ? ctx.integration.connection + .resolve(connection) + .pipe(Effect.orElseSucceed(() => undefined), Effect.map(bearerFrom)) + : Effect.succeed(undefined), + ), + Effect.orElseSucceed(() => undefined), + Effect.flatMap((resolved) => + fetchLimits( + http, + endpoint.baseURL, + endpoint.authorization ?? resolved ?? bearerKey(endpoint.apiKey), + ), + ), + ), + ), + ), { concurrency: "unbounded" }, ) let changed = false From 824385202accff0afb85666729046b63d1d6c03d Mon Sep 17 00:00:00 2001 From: Lance Woodson <65194841+woodsonl@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:21:10 -0500 Subject: [PATCH 3/4] fix(core): send no bearer header for an empty apiKey in limit discovery An empty settings.apiKey produced a bare "Bearer " authorization header on the /models probe. Treat it as absent, as the local provider does, and cover it with a live test. --- .../provider/openai-compatible-limits.ts | 3 +- .../provider-openai-compatible-limits.test.ts | 33 +++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/packages/core/src/plugin/provider/openai-compatible-limits.ts b/packages/core/src/plugin/provider/openai-compatible-limits.ts index dc8639609db4..a64f3df47140 100644 --- a/packages/core/src/plugin/provider/openai-compatible-limits.ts +++ b/packages/core/src/plugin/provider/openai-compatible-limits.ts @@ -147,7 +147,8 @@ const bearerFrom = (credential: CredentialValue | undefined) => { return undefined } -const bearerKey = (apiKey: string | undefined) => (apiKey === undefined ? undefined : `Bearer ${apiKey}`) +// An empty key is absent, matching local providers, so the probe sends no bare `Bearer ` header. +const bearerKey = (apiKey: string | undefined) => (apiKey ? `Bearer ${apiKey}` : undefined) export const OpenAICompatibleLimitsPlugin = define({ id: "opencode.provider.openai-compatible-limits", diff --git a/packages/core/test/plugin/provider-openai-compatible-limits.test.ts b/packages/core/test/plugin/provider-openai-compatible-limits.test.ts index d04b31e67f7f..cb31c5d663d3 100644 --- a/packages/core/test/plugin/provider-openai-compatible-limits.test.ts +++ b/packages/core/test/plugin/provider-openai-compatible-limits.test.ts @@ -41,7 +41,7 @@ function eventually( }) } -function configuration(id: string, baseURL: string, models: Record): Entry { +function configuration(id: string, baseURL: string, models: Record, apiKey = "secret"): Entry { return new Document({ type: "document", info: decode({ @@ -49,7 +49,7 @@ function configuration(id: string, baseURL: string, models: Record { ), ) + it.live("treats an empty apiKey as absent and sends no authorization header", () => + Effect.acquireUseRelease( + Effect.sync(() => { + const authorizations: Array = [] + const server = Bun.serve({ + port: 0, + fetch: (request) => { + authorizations.push(request.headers.get("authorization")) + return Response.json({ data: [{ id: "open-model", context_length: 300_000 }] }) + }, + }) + return { authorizations, server } + }), + ({ authorizations, server }) => + Effect.gen(function* () { + const modelState = yield* Model.Service + yield* addPlugins([configuration("open-proxy", `${server.url.origin}/v1`, { "open-model": {} }, "")]) + + yield* eventually( + modelState.get(Provider.ID.make("open-proxy"), Model.ID.make("open-model")), + (model) => model?.limit.context === 300_000, + ) + expect(authorizations.length).toBeGreaterThan(0) + expect(authorizations.every((value) => value === null)).toBe(true) + }), + ({ server }) => Effect.promise(() => server.stop(true)), + ), + ) + it.live("fails open on erroring endpoints and still applies live ones", () => Effect.acquireUseRelease( Effect.sync(() => { From dafb91b10debba8ebda7cf5aab1b7d66c82844b7 Mon Sep 17 00:00:00 2001 From: Lance Woodson <65194841+woodsonl@users.noreply.github.com> Date: Tue, 27 Oct 2026 02:01:00 -0500 Subject: [PATCH 4/4] fix(core): skip dedicated local providers, schedule limit discovery, key cache by auth Review follow-up on the OpenAI-compatible limit discovery plugin: - LM Studio, Ollama, and vLLM providers (any provider a local discovery plugin owns) are no longer probed or filled; their own plugin owns the inventory and already hits their endpoints. - Discovery now repeats on a 60s schedule like the local provider plugins, so failed endpoints retry about once a minute and successes refresh at their 5-minute TTL instead of both waiting for the next catalog change. A reload only fires when the advertisement changed. - The fetch cache is keyed by the auth known in settings (authorization header or apiKey digest), and entries record the connection identity they were fetched under; a rotated key or a credential switch (for example after `opencode auth login`) refetches instead of serving the previous account's result. Credential switch events revalidate at once. --- packages/core/src/plugin/provider/local.ts | 9 ++ .../provider/openai-compatible-limits.ts | 144 +++++++++++++----- .../provider-openai-compatible-limits.test.ts | 87 ++++++++++- packages/web/src/content/docs/providers.mdx | 8 +- 4 files changed, 208 insertions(+), 40 deletions(-) diff --git a/packages/core/src/plugin/provider/local.ts b/packages/core/src/plugin/provider/local.ts index 3935461be2da..2b3f236f67d8 100644 --- a/packages/core/src/plugin/provider/local.ts +++ b/packages/core/src/plugin/provider/local.ts @@ -20,6 +20,14 @@ export type LocalModel = { readonly context?: number | null } +/** + * Provider IDs whose inventory a local discovery plugin owns. Populated by `createLocalProviderPlugin` at module + * load, so generic discovery (for example OpenAI-compatible limit probing) can leave these providers to their + * dedicated plugin instead of hitting their servers a second time. + */ +const ownedProviderIDs = new Set() +export const localProviderIDs: ReadonlySet = ownedProviderIDs + export type LocalDiscoveryClient = ReturnType export function createLocalProviderPlugin(input: { @@ -34,6 +42,7 @@ export function createLocalProviderPlugin(input: { const discovery = new Map() const discoveryLock = Semaphore.makeUnsafe(1) const suffix = input.stripPathSuffix ?? /\/v1$/ + ownedProviderIDs.add(input.providerID) return function make(origin?: string, interval: Duration.Input = "30 seconds") { return define({ diff --git a/packages/core/src/plugin/provider/openai-compatible-limits.ts b/packages/core/src/plugin/provider/openai-compatible-limits.ts index a64f3df47140..1199b5d217e8 100644 --- a/packages/core/src/plugin/provider/openai-compatible-limits.ts +++ b/packages/core/src/plugin/provider/openai-compatible-limits.ts @@ -1,11 +1,15 @@ +import { createHash } from "node:crypto" import { isRecord } from "@opencode/util/record" import { define } from "@opencode/plugin/effect/plugin" -import { Effect, Option, Semaphore, Stream } from "effect" +import { Effect, Option, Schedule, Semaphore, Stream } from "effect" import { HttpClient, HttpClientRequest } from "effect/http" import type { Value as CredentialValue } from "@opencode/schema/credential" import { Bus } from "../../bus.js" +import { Credential } from "../../credential.js" +import { IntegrationConnection } from "../../integration/connection.js" import { Model } from "../../model.js" import { Provider } from "../../provider.js" +import { localProviderIDs } from "./local.js" // Fill missing context limits for generic OpenAI-compatible providers from the // provider's own `GET {baseURL}/models` advertisement. @@ -13,9 +17,10 @@ import { Provider } from "../../provider.js" // Custom models synthesized without a models.dev catalog entry carry the // built-in placeholder limits, so auto-compaction fires against a guess // instead of the real window. Catalog-backed models — bundled providers, -// config models inherited from a canonical catalog provider, and integration -// models with their own inventory — keep their real values and are neither -// probed nor filled. +// config models inherited from a canonical catalog provider, integration +// models with their own cost rows, and providers a dedicated discovery +// plugin owns (LM Studio, Ollama, vLLM) — keep their real values and are +// neither probed nor filled. // // Auth for the probe, highest first: a configured `authorization` header, the // provider's active connection credential (keys added via `opencode auth` or @@ -30,6 +35,10 @@ import { Provider } from "../../provider.js" // // Fail-open: an unreachable, erroring, or malformed endpoint is skipped and // keeps its previously discovered limits; discovery never blocks the catalog. +// Discovery runs at startup, whenever the catalog or the active credential +// changes, and on the schedule below, so failed endpoints retry once a +// minute and successes refresh at their TTL instead of waiting for a catalog +// change. const FETCH_TIMEOUT = "4 seconds" const CACHE_TTL_MS = 5 * 60_000 @@ -55,13 +64,24 @@ type Endpoint = { type CacheEntry = { readonly at: number readonly ttl: number + /** Fingerprint of the connection credential the cached fetch ran under; `""` when none was active. */ + readonly connection: string readonly value: Record } // Shared across Locations so every catalog rebuild reuses one fetch per endpoint. const cache = new Map() -const endpointKey = (endpoint: Endpoint) => `${endpoint.providerID}\n${endpoint.baseURL}` +const digest = (value: string) => createHash("sha256").update(value).digest("hex").slice(0, 16) + +// The key covers everything known synchronously: provider, URL, and the auth +// set directly in settings, so a rotated `apiKey` or `authorization` header +// never reuses the previous credential's result. The active connection's +// identity rides in the entry and is compared at discovery time. +const endpointKey = (endpoint: Endpoint) => + `${endpoint.providerID}\n${endpoint.baseURL}\n${ + endpoint.authorization ?? (endpoint.apiKey ? digest(endpoint.apiKey) : "") + }` const freshEntry = (endpoint: Endpoint) => { const entry = cache.get(endpointKey(endpoint)) @@ -150,6 +170,17 @@ const bearerFrom = (credential: CredentialValue | undefined) => { // An empty key is absent, matching local providers, so the probe sends no bare `Bearer ` header. const bearerKey = (apiKey: string | undefined) => (apiKey ? `Bearer ${apiKey}` : undefined) +// Identifies which connection the cached fetch ran under. The connection's +// identity (not a rotating OAuth access token) keys the cache, plus the stored +// key value for key credentials, which can be edited in place under the same id. +const connectionFingerprint = ( + connection: Parameters[0], + credential: CredentialValue | undefined, +) => { + const identity = IntegrationConnection.key(connection) ?? "" + return credential?.type === "key" ? `${identity}:${digest(credential.key)}` : identity +} + export const OpenAICompatibleLimitsPlugin = define({ id: "opencode.provider.openai-compatible-limits", effect: Effect.fn(function* (ctx) { @@ -161,6 +192,9 @@ export const OpenAICompatibleLimitsPlugin = define({ yield* ctx.model.transform((models) => { endpoints.clear() for (const item of models.provider.list()) { + // A dedicated discovery plugin owns this provider's inventory and probes + // its own endpoints; a second /models request here would be redundant. + if (localProviderIDs.has(item.provider.id)) continue for (const [id, definition] of item.models) { if ((definition.package ?? item.provider.package) !== OPENAI_COMPATIBLE) continue if (fromCatalog(definition)) continue @@ -201,60 +235,100 @@ export const OpenAICompatibleLimitsPlugin = define({ const discover = Effect.fn("OpenAICompatibleLimits.discover")(function* () { yield* lock.withPermit( Effect.gen(function* () { - const stale = [...endpoints.values()].filter((endpoint) => freshEntry(endpoint) === undefined) + // Resolve the active connection first: an entry cached under a + // different credential (for example after `opencode auth login`) is + // stale no matter its TTL. + const stale: Array<{ + endpoint: Endpoint + connection: string + authorization: string | undefined + }> = [] + for (const endpoint of endpoints.values()) { + const connection = yield* ctx.integration.connection + .active(endpoint.providerID) + .pipe(Effect.orElseSucceed(() => undefined)) + const credential = connection + ? yield* ctx.integration.connection + .resolve(connection) + .pipe(Effect.orElseSucceed(() => undefined)) + : undefined + const fingerprint = connectionFingerprint(connection, credential) + const entry = cache.get(endpointKey(endpoint)) + if (entry !== undefined && Date.now() - entry.at < entry.ttl && entry.connection === fingerprint) + continue + stale.push({ + endpoint, + connection: fingerprint, + authorization: + endpoint.authorization ?? bearerFrom(credential) ?? bearerKey(endpoint.apiKey), + }) + } + if (stale.length === 0) return const results = yield* Effect.all( - stale.map((endpoint) => - Effect.option( - ctx.integration.connection - .active(endpoint.providerID) - .pipe( - Effect.flatMap((connection) => - connection - ? ctx.integration.connection - .resolve(connection) - .pipe(Effect.orElseSucceed(() => undefined), Effect.map(bearerFrom)) - : Effect.succeed(undefined), - ), - Effect.orElseSucceed(() => undefined), - Effect.flatMap((resolved) => - fetchLimits( - http, - endpoint.baseURL, - endpoint.authorization ?? resolved ?? bearerKey(endpoint.apiKey), - ), - ), - ), - ), - ), + stale.map((item) => Effect.option(fetchLimits(http, item.endpoint.baseURL, item.authorization))), { concurrency: "unbounded" }, ) let changed = false - for (const [index, endpoint] of stale.entries()) { + for (const [index, item] of stale.entries()) { const result = results[index] - const previous = cache.get(endpointKey(endpoint)) + const key = endpointKey(item.endpoint) + const previous = cache.get(key) if (Option.isNone(result)) { // Keep the last successful inventory through transient outages and retry soon. - cache.set(endpointKey(endpoint), { + cache.set(key, { at: Date.now(), ttl: FAILURE_TTL_MS, + connection: item.connection, value: previous?.value ?? {}, }) continue } - cache.set(endpointKey(endpoint), { at: Date.now(), ttl: CACHE_TTL_MS, value: result.value }) - changed = true + cache.set(key, { + at: Date.now(), + ttl: CACHE_TTL_MS, + connection: item.connection, + value: result.value, + }) + if (previous === undefined || !sameLimits(previous.value, result.value)) changed = true } if (changed) yield* ctx.model.reload() }), ) }) - yield* discover().pipe(Effect.ignore, Effect.forkScoped) + // Failed endpoints retry once a minute and successes refresh at their TTL, + // on a schedule like the local provider plugins, instead of only when the + // catalog changes. + yield* discover().pipe(Effect.ignore, Effect.repeat(Schedule.spaced(FAILURE_TTL_MS)), Effect.forkScoped) yield* bus .subscribe([Model.Event.Updated]) .pipe( Stream.runForEach(() => discover().pipe(Effect.ignore)), Effect.forkScoped({ startImmediately: true }), ) + // A credential switch can change what /models advertises; revalidate at + // once instead of waiting out the cache TTL. + yield* bus + .subscribe(Credential.Event.Switched) + .pipe( + Stream.filter((event) => + [...endpoints.values()].some( + (endpoint) => endpoint.providerID === Provider.ID.make(event.data.integrationID), + ), + ), + Stream.runForEach(() => discover().pipe(Effect.ignore)), + Effect.forkScoped({ startImmediately: true }), + ) }), }) + +const sameLimits = (left: Record, right: Record) => { + const ids = Object.keys(left) + if (ids.length !== Object.keys(right).length) return false + return ids.every((id) => { + const a = left[id] + const b = right[id] + if (b === undefined) return false + return a.context === b.context && a.input === b.input && a.output === b.output + }) +} diff --git a/packages/core/test/plugin/provider-openai-compatible-limits.test.ts b/packages/core/test/plugin/provider-openai-compatible-limits.test.ts index cb31c5d663d3..9112621ab294 100644 --- a/packages/core/test/plugin/provider-openai-compatible-limits.test.ts +++ b/packages/core/test/plugin/provider-openai-compatible-limits.test.ts @@ -10,7 +10,8 @@ import { } from "@opencode/core/plugin/provider/openai-compatible-limits" import { ProviderPlugins } from "@opencode/core/plugin/provider" import { Provider } from "@opencode/core/provider" -import { Document, type Entry, Info } from "@opencode/schema/config" +import { Bus } from "@opencode/core/bus" +import { Document, type Entry, Event, Info } from "@opencode/schema/config" import { describe, expect, test } from "bun:test" import { Effect, Schema } from "effect" import { testEffect } from "../lib/effect" @@ -23,8 +24,11 @@ const addPlugins = Effect.fn(function* (entries: Entry[]) { const plugin = yield* Plugin.Service const host = yield* PluginHost.make(plugin) const config = Config.testLayer(entries) - yield* OpenAICompatibleLimitsPlugin.effect(host).pipe(Effect.provide(config)) - yield* ConfigProviderPlugin.Plugin.effect(host).pipe(Effect.provide(config)) + return yield* Effect.gen(function* () { + yield* OpenAICompatibleLimitsPlugin.effect(host) + yield* ConfigProviderPlugin.Plugin.effect(host) + return yield* Config.Test + }).pipe(Effect.provide(config)) }) function eventually( @@ -242,4 +246,81 @@ describe("OpenAICompatibleLimitsPlugin", () => { ({ dead, live }) => Effect.promise(() => Promise.all([dead.stop(true), live.stop(true)])), ), ) + + it.live("does not probe providers owned by a dedicated discovery plugin", () => + Effect.acquireUseRelease( + Effect.sync(() => { + const requests: string[] = [] + const server = Bun.serve({ + port: 0, + fetch: (request) => { + requests.push(new URL(request.url).pathname) + return Response.json({ data: [{ id: "local-model", context_length: 1_000_000 }] }) + }, + }) + return { requests, server } + }), + ({ requests, server }) => + Effect.gen(function* () { + const modelState = yield* Model.Service + yield* addPlugins([configuration("vllm", `${server.url.origin}/v1`, { "local-model": {} })]) + + const model = yield* eventually( + modelState.get(Provider.ID.make("vllm"), Model.ID.make("local-model")), + (model) => model !== undefined, + ) + // LM Studio, Ollama, and vLLM keep the inventory their own plugin discovers. + yield* Effect.promise(() => Bun.sleep(200)) + expect(requests).toEqual([]) + expect(model?.limit).toMatchObject({ context: 200_000, output: 32_000 }) + }), + ({ server }) => Effect.promise(() => server.stop(true)), + ), + ) + + it.live("refetches when the settings apiKey changes instead of serving the cached result", () => + Effect.acquireUseRelease( + Effect.sync(() => { + const authorizations: Array = [] + const server = Bun.serve({ + port: 0, + fetch: (request) => { + const authorization = request.headers.get("authorization") + authorizations.push(authorization) + return Response.json({ + data: [ + { + id: "rotating-model", + context_length: authorization === "Bearer one" ? 111_111 : 222_222, + }, + ], + }) + }, + }) + return { authorizations, server } + }), + ({ authorizations, server }) => + Effect.gen(function* () { + const modelState = yield* Model.Service + const bus = yield* Bus.Service + const providerID = Provider.ID.make("rotating") + const origin = `${server.url.origin}/v1` + const models = { "rotating-model": {} } + const test = yield* addPlugins([configuration("rotating", origin, models, "one")]) + + yield* eventually( + modelState.get(providerID, Model.ID.make("rotating-model")), + (model) => model?.limit.context === 111_111, + ) + yield* test.setEntries([configuration("rotating", origin, models, "two")]) + yield* bus.publish(Event.Updated, {}) + yield* eventually( + modelState.get(providerID, Model.ID.make("rotating-model")), + (model) => model?.limit.context === 222_222, + ) + expect(authorizations).toContain("Bearer two") + }), + ({ server }) => Effect.promise(() => server.stop(true)), + ), + ) }) diff --git a/packages/web/src/content/docs/providers.mdx b/packages/web/src/content/docs/providers.mdx index 6196dc642365..686e86caa9d3 100644 --- a/packages/web/src/content/docs/providers.mdx +++ b/packages/web/src/content/docs/providers.mdx @@ -2685,8 +2685,12 @@ Precedence, highest first: Discovery fails open: if the endpoint is unreachable, errors, or returns malformed data, OpenCode keeps whatever limits it already has and never blocks -startup. A manual `limit` in the config is still the right choice when the -server does not advertise limits or advertises wrong ones. +startup. Failed lookups retry about once a minute and successful ones refresh +every five minutes, so an endpoint that was down when OpenCode started catches +up on its own. Providers with their own discovery integration (LM Studio, +Ollama, vLLM) are left to that integration and are never probed here. A manual +`limit` in the config is still the right choice when the server does not +advertise limits or advertises wrong ones. ---