diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 51d820efa..3318e0146 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,8 +4,10 @@ repos: hooks: - id: trailing-whitespace # The plans embed unified diffs, whose blank context lines are a lone space. - exclude: ^docs/superpowers/plans/ + # Golden files hold exact output bytes. A fixer must not change them. + exclude: ^docs/superpowers/plans/|\.golden$ - id: end-of-file-fixer + exclude: \.golden$ - id: check-added-large-files args: ['--maxkb=1024'] - id: check-yaml diff --git a/cmd/agentop/README.md b/cmd/agentop/README.md index 48dae05f0..11170d7e7 100644 --- a/cmd/agentop/README.md +++ b/cmd/agentop/README.md @@ -704,6 +704,19 @@ refusal is believed for an hour, and a session keeps the model it was given. creates the plugin's entry, last in the outbound pipeline. A key or a word containing `$` is refused: the config loader would read it as an environment variable. + - `--api =` (repeatable) names a protocol the server serves and + the path it serves it at: `anthropic-messages`, `chat-completions` or + `responses`. A second `--api` for one protocol replaces the first. With no + `--api`, requests go in the agent's protocol to the agent's path. + - `--auth-header ` names the header that carries the key: `authorization` + sends `Bearer `, any other name sends the raw key. + - `--header =` (repeatable) sets a fixed header on every request to + the server. A value with `$` is refused, as a key is: write + `headers: {: ${NAME}}` by hand. The key headers and the hop-by-hop + headers are refused. + + `agentop server` shows these under the server: its `api`, its auth header, and + the header names, never their values. - **`use --agent `** routes the agent's new sessions to the server, including an agent that has not run yet. **`reset --agent `** stops routing it. Either way a session already running stays where it is: its pin holds diff --git a/cmd/agentop/cmd_server.go b/cmd/agentop/cmd_server.go index 78f315f2b..73105e80b 100644 --- a/cmd/agentop/cmd_server.go +++ b/cmd/agentop/cmd_server.go @@ -33,6 +33,7 @@ const serverUsage = `agentop server — choose the inference server each coding Usage: agentop server [--config PATH] agentop server add --main WORD --helper WORD [--key-stdin] [--yes] [--config PATH] + [--api PROTOCOL=PATH]... [--auth-header NAME] [--header NAME=VALUE]... agentop server remove [--config PATH] agentop server use --agent [--config PATH] agentop server reset --agent [--config PATH] @@ -54,7 +55,9 @@ agent sends it. "add" reads the server's API key at a prompt that does not echo, or from stdin with --key-stdin; it is never an argument. Adding a name that exists asks before replacing its URL, key and models (--yes skips the question), which is also how a -key is rotated. +key is rotated. --api names a protocol the server serves and its path +(anthropic-messages, chat-completions or responses), --auth-header the header +that carries the key, and --header a fixed header for every request. A request goes with the model the agent asked for. Only when the server refuses that model is it sent again, once and before the agent sees the refusal, with @@ -256,8 +259,16 @@ func serverList(args []string, stdout, stderr io.Writer) int { fmt.Fprintf(tw, " %s\t%s\t%s\t%s\n", name, servers.Host(s), resolvedWords(s), strings.Join(agentsOn(c, name), ", ")) } tw.Flush() + // The detail lines go under each row after the table is aligned: a line with no + // tab would restart tabwriter's columns below it. + names := slices.Sorted(maps.Keys(c.Servers)) + i := 0 for line := range strings.Lines(table.String()) { fmt.Fprintln(stdout, strings.TrimRight(line, " \n")) + for _, d := range protocolLines(c.Servers[names[i]]) { + fmt.Fprintln(stdout, " "+d) + } + i++ } fmt.Fprintln(stdout) @@ -267,6 +278,26 @@ func serverList(args []string, stdout, stderr io.Writer) int { return 0 } +// protocolLines is s's api, auth_header and header names, one line each that s has. +// Header values are never shown: they can hold a credential. +func protocolLines(s routerconfig.Server) []string { + var out []string + if len(s.API) > 0 { + var api []string + for _, p := range slices.Sorted(maps.Keys(s.API)) { + api = append(api, p+"="+s.API[p]) + } + out = append(out, "api: "+strings.Join(api, ", ")) + } + if s.AuthHeader != "" { + out = append(out, "auth header: "+s.AuthHeader) + } + if len(s.Headers) > 0 { + out = append(out, "headers: "+strings.Join(slices.Sorted(maps.Keys(s.Headers)), ", ")) + } + return out +} + // resolvedWords is a server's main and helper words with the models they name on its // list now, read with the server's own key, as the router resolves them. A server // whose list cannot be read says why rather than showing a model it may not name. diff --git a/cmd/agentop/cmd_server_test.go b/cmd/agentop/cmd_server_test.go index 09bae4e60..d3ccafa68 100644 --- a/cmd/agentop/cmd_server_test.go +++ b/cmd/agentop/cmd_server_test.go @@ -295,3 +295,41 @@ func TestServer_SaysWhenAServersModelListIsUnavailable(t *testing.T) { t.Errorf("the listing prints a key:\n%s", out) } } + +// Rule C4: the listing shows api, auth_header and the header names, never a header value. +func TestServer_ListsTheAPIAuthHeaderAndHeaderNames(t *testing.T) { + block := strings.Replace(routerBlock, " helper: haiku\n", ` helper: haiku + api: + chat-completions: /v1/chat/completions + anthropic-messages: /v1/messages + auth_header: x-api-key + headers: + X-Client: cortex + User-Agent: bob-shell +`, 1) + path := serverEnv(t, closedAddr(t), block) + code, out, errOut := runServerCmd(t, "", "--config", path) + if code != 0 { + t.Fatalf("exit %d: %s", code, errOut) + } + lines := strings.Split(out, "\n") + want := []string{ + "api: anthropic-messages=/v1/messages, chat-completions=/v1/chat/completions", + "auth header: x-api-key", + "headers: User-Agent, X-Client", + } + if len(lines) < 5 || !strings.HasPrefix(flat(lines[0]), "ete ") || !strings.HasPrefix(flat(lines[4]), "glm ") { + t.Fatalf("the detail lines are not under ete:\n%s", out) + } + for i, w := range want { + if got := flat(lines[i+1]); got != w { + t.Errorf("line %d = %q, want %q", i+2, got, w) + } + } + if strings.Index(lines[0], "main") != strings.Index(lines[4], "main") { + t.Errorf("columns do not line up:\n%s", out) + } + if strings.Contains(out, "bob-shell") || strings.Contains(out, "cortex\n") { + t.Errorf("the listing shows a header value:\n%s", out) + } +} diff --git a/cmd/agentop/cmd_server_write.go b/cmd/agentop/cmd_server_write.go index dbba7fea0..45bdadc36 100644 --- a/cmd/agentop/cmd_server_write.go +++ b/cmd/agentop/cmd_server_write.go @@ -13,6 +13,7 @@ import ( "time" "github.com/charmbracelet/x/term" + "gopkg.in/yaml.v3" "github.com/rossoctl/cortex/cmd/agentop/apiclient" "github.com/rossoctl/cortex/cmd/agentop/edit" @@ -82,6 +83,10 @@ func serverAdd(args []string, stdin io.Reader, stdout, stderr io.Writer) int { var models routerconfig.Server flags.StringVar(&models.Main, "main", "", "a word naming the server's model for an agent's main work, such as opus or glm") flags.StringVar(&models.Helper, "helper", "", "a word naming the server's model for an agent's helper calls, such as haiku") + api, headers := pairFlag{flag: "--api", want: "="}, pairFlag{flag: "--header", want: "="} + flags.Var(&api, "api", "a protocol the server serves and its path, such as chat-completions=/v1/chat/completions (repeatable)") + flags.StringVar(&models.AuthHeader, "auth-header", "", "the header that carries the key, such as x-api-key (default: the agent's)") + flags.Var(&headers, "header", "a fixed header set on every request to the server, as = (repeatable)") pos, code, ok := serverFlags(flags, args, stdout, stderr) if !ok { return code @@ -112,6 +117,11 @@ func serverAdd(args []string, stdin io.Reader, stdout, stderr io.Writer) int { fmt.Fprintf(stderr, "agentop server add: %v\n", err) return 2 } + models.API, models.Headers = api.m, headers.m + if err := checkProtocolFlags(models); err != nil { + fmt.Fprintf(stderr, "agentop server add: %v\n", err) + return 2 + } // Before anything is asked or read: --key-stdin reads stdin as typed, so on a // terminal the key would echo, character by character, onto the screen and into // any recording of it. The prompt without the flag reads it without echo. @@ -186,9 +196,18 @@ func serverAdd(args []string, stdin io.Reader, stdout, stderr io.Writer) int { fmt.Fprintf(stdout, "%s %s → %s\n", strings.TrimPrefix(w.flag, "--"), w.word, m) } - pairs := []string{"url", ep.URL(), "key", key, "main", models.Main, "helper", models.Helper} + value := edit.MapValue("url", ep.URL(), "key", key, "main", models.Main, "helper", models.Helper) + if len(models.API) > 0 { + value.Content = append(value.Content, edit.ScalarValue("api"), sortedMap(models.API)) + } + if models.AuthHeader != "" { + value.Content = append(value.Content, edit.ScalarValue("auth_header"), edit.ScalarValue(models.AuthHeader)) + } + if len(models.Headers) > 0 { + value.Content = append(value.Content, edit.ScalarValue("headers"), sortedMap(models.Headers)) + } ch := edit.ConfigChange{Chain: "outbound", Plugin: routerName, Path: []string{"servers", name}, - Value: edit.MapValue(pairs...), CreatePlugin: true} + Value: value, CreatePlugin: true} done := "Added " + name + "." if replacing { done = "Replaced " + name + "." @@ -217,6 +236,57 @@ func checkWordFlags(models routerconfig.Server) error { return nil } +// pairFlag is a repeatable = flag. A second value for one key replaces the first. +type pairFlag struct { + flag, want string + m map[string]string +} + +func (f *pairFlag) String() string { return "" } + +func (f *pairFlag) Set(s string) error { + k, v, ok := strings.Cut(s, "=") + if !ok || k == "" { + return fmt.Errorf("want %s", f.want) + } + if f.m == nil { + f.m = map[string]string{} + } + f.m[k] = v + return nil +} + +// checkProtocolFlags checks add's --api, --auth-header and --header with the rules the +// router loads them by, and refuses a header value with $, which config.Load would expand. +// The errors never repeat a header value. +func checkProtocolFlags(s routerconfig.Server) error { + if err := routerconfig.CheckAPI(s.API); err != nil { + return fmt.Errorf("--%w", err) + } + if err := routerconfig.CheckAuthHeader(s.AuthHeader); err != nil { + return fmt.Errorf("--auth-header: %w", err) + } + if err := routerconfig.CheckHeaders(s.Headers, s.AuthHeader); err != nil { + return fmt.Errorf("--header: %w", err) + } + for _, name := range slices.Sorted(maps.Keys(s.Headers)) { + if strings.Contains(s.Headers[name], "$") { + return fmt.Errorf("--header %s: the value contains $, which Cortex reads as an environment variable when it "+ + "loads the config; write headers: {%s: ${NAME}} by hand", name, name) + } + } + return nil +} + +// sortedMap is m as a YAML mapping, its keys sorted, so the same flags write the same file. +func sortedMap(m map[string]string) *yaml.Node { + var pairs []string + for _, k := range slices.Sorted(maps.Keys(m)) { + pairs = append(pairs, k, m[k]) + } + return edit.MapValue(pairs...) +} + func serverRemove(args []string, stdout, stderr io.Writer) int { flags := flag.NewFlagSet("server remove", flag.ContinueOnError) cfgPath := flags.String("config", "", "Cortex config file (default ~/.cortex/config.yaml)") diff --git a/cmd/agentop/cmd_server_write_test.go b/cmd/agentop/cmd_server_write_test.go index f218f6d16..b841fb88c 100644 --- a/cmd/agentop/cmd_server_write_test.go +++ b/cmd/agentop/cmd_server_write_test.go @@ -1058,3 +1058,64 @@ func TestServerMove_AnEmptyListingPrunesNothing(t *testing.T) { t.Errorf("config:\n%s", cfg) } } + +// Rule C4: the three fields are written under the server, the maps sorted, and a +// second --api for one protocol replaces the first. +func TestServerAdd_WritesTheAPIAuthHeaderAndHeaders(t *testing.T) { + path := serverEnv(t, newFakeStats(t, 0).addr(), "") + code, _, errOut := runServerCmd(t, "sk-bob", "add", "bob", "https://bob.example.com", "--main", "opus", "--helper", "haiku", + "--api", "chat-completions=/x", "--api", "chat-completions=/v1/chat/completions", "--api", "anthropic-messages=/v1/messages", + "--auth-header", "x-api-key", "--header", "User-Agent=bob-shell", "--header", "X-Client=cortex", + "--key-stdin", "--config", path) + if code != 0 { + t.Fatalf("exit %d: %s", code, errOut) + } + want := ` bob: + url: https://bob.example.com + key: sk-bob + main: opus + helper: haiku + api: + anthropic-messages: /v1/messages + chat-completions: /v1/chat/completions + auth_header: x-api-key + headers: + User-Agent: bob-shell + X-Client: cortex +` + if got := readConfig(t, path); !strings.HasSuffix(got, want) { + t.Errorf("config does not end with the new entry:\n%s", got) + } +} + +// Each bad flag is refused with exit 2 before the key is asked for, and the file is +// not written. +func TestServerAdd_RefusesABadAPIOrHeaderFlagBeforeAskingForAKey(t *testing.T) { + asked := stubKeyPrompt(t, "sk") + path := serverEnv(t, newFakeStats(t, 0).addr(), "") + before := readConfig(t, path) + for _, tc := range []struct { + flags []string + want string + }{ + {[]string{"--api", "bogus=/x"}, "valid protocols: anthropic-messages, chat-completions, responses"}, + {[]string{"--api", "chat-completions"}, "want ="}, + {[]string{"--api", "chat-completions=v1"}, "must be absolute"}, + {[]string{"--auth-header", "Host"}, "cannot carry the key"}, + {[]string{"--header", "X-A=$HOME"}, "--header X-A: the value contains $"}, + {[]string{"--header", "Authorization=x"}, `"Authorization" cannot be set here`}, + {[]string{"--header", "X-A"}, "want ="}, + } { + args := append([]string{"add", "bob", "https://bob.example.com", "--main", "opus", "--helper", "haiku", "--config", path}, tc.flags...) + code, _, errOut := runServerCmd(t, "", args...) + if code != 2 || !strings.Contains(errOut, tc.want) { + t.Errorf("%v: exit %d, stderr:\n%s", tc.flags, code, errOut) + } + if strings.Contains(errOut, "$HOME") { + t.Errorf("%v: stderr repeats the header value:\n%s", tc.flags, errOut) + } + } + if *asked != "" || readConfig(t, path) != before { + t.Error("a bad flag asked for a key or wrote the file") + } +} diff --git a/cmd/agentop/tui/detail_raw.go b/cmd/agentop/tui/detail_raw.go index 2aed1d1e7..cdae49551 100644 --- a/cmd/agentop/tui/detail_raw.go +++ b/cmd/agentop/tui/detail_raw.go @@ -56,6 +56,17 @@ func withRaw(data []byte, raw *pipeline.RawCapture, e *pipeline.SessionEvent) [] case p.Body != "": resp["body"] = bodyView(p.Body, nil) } + if p.Upstream != nil || p.UpstreamBase64 != nil { + text := "" + if p.Upstream != nil { + text = *p.Upstream + } + resp["upstream"] = "server's reply before translation: " + formatBytes(int64(len(text)+len(p.UpstreamBase64))) + resp["upstreamBody"] = bodyView(text, p.UpstreamBase64) + if p.UpstreamTruncated { + resp["upstreamTruncated"] = true + } + } r["response"] = resp } else { r["request"] = bodyView(raw.Request, raw.RequestBase64) @@ -69,6 +80,14 @@ func withRaw(data []byte, raw *pipeline.RawCapture, e *pipeline.SessionEvent) [] default: r["forwarded"] = "forwarded as sent" } + if raw.Translated != nil || raw.TranslatedBase64 != nil { + text := "" + if raw.Translated != nil { + text = *raw.Translated + } + r["translated"] = "sent upstream, translated by Cortex: " + formatBytes(int64(len(text)+len(raw.TranslatedBase64))) + r["translatedBody"] = bodyView(text, raw.TranslatedBase64) + } if len(raw.RequestHeaders) > 0 { m["headers"] = flatHeaders(raw.RequestHeaders) } diff --git a/cmd/agentop/tui/detail_raw_test.go b/cmd/agentop/tui/detail_raw_test.go index badd54e24..98c6f455c 100644 --- a/cmd/agentop/tui/detail_raw_test.go +++ b/cmd/agentop/tui/detail_raw_test.go @@ -215,3 +215,31 @@ func TestDetailRaw_ReopenShowsTheHeldCapture(t *testing.T) { t.Fatalf("the reopened row shows raw %+v, want the held capture", m.detailRow.raw) } } + +// A translated request shows the body Cortex sent upstream, and its reply the server's reply +// before translation, each with its own label: neither is a plugin's rewrite. +func TestDetailRaw_ShowsTheTranslatedHop(t *testing.T) { + tr := `{"model":"c","messages":[{"role":"user","content":[{"type":"text","text":"translated text"}]}]}` + out := rawDetail(t, &pipeline.RawCapture{Request: `{"model":"m"}`, Translated: &tr}) + for _, want := range []string{"sent upstream, translated by Cortex: 95B", "translatedBody", "translated text", "forwarded as sent"} { + if !strings.Contains(out, want) { + t.Errorf("request detail lacks %q:\n%s", want, out) + } + } + + ev := &pipeline.SessionEvent{Seq: 4, Phase: pipeline.SessionResponse, Direction: pipeline.Outbound, Host: "h", + Inference: &pipeline.InferenceExtension{Model: "m"}} + m := detailModel(t, "s1", ev) + up := `{"choices":[{"message":{"content":"chat reply"}}]}` + m.applyDetailEvent(detailEventLoadedMsg{sessionID: "s1", seq: 4, event: ev, raw: &pipeline.RawCapture{ + Response: &pipeline.RawBody{Status: 200, Body: `{}`, Upstream: &up}}}) + m.detailFolds = map[string]foldChoice{"response.raw": {open: true}, "response.raw.response": {open: true}, + "response.raw.response.upstreamBody": {open: true}} + m.renderDetail() + got := stripANSI(strings.Join(lineTexts(m.detailLines), "\n")) + for _, want := range []string{"server's reply before translation: 50B", "upstreamBody"} { + if !strings.Contains(got, want) { + t.Errorf("response detail lacks %q:\n%s", want, got) + } + } +} diff --git a/cmd/agentop/tui/server_route.go b/cmd/agentop/tui/server_route.go index e1e790210..ae58a63e4 100644 --- a/cmd/agentop/tui/server_route.go +++ b/cmd/agentop/tui/server_route.go @@ -5,6 +5,7 @@ import ( "maps" "os" "path/filepath" + "reflect" "strings" "github.com/rossoctl/cortex/cmd/agentop/servers" @@ -88,7 +89,9 @@ func homeTilde(path string) string { // sameRouter reports whether two router configs route alike, so a pipeline refetch repaints the // SERVER columns only when what they show has changed. func sameRouter(a, b routerconfig.Config) bool { - return maps.Equal(a.Servers, b.Servers) && maps.Equal(a.Agents, b.Agents) + // DeepEqual: a Server holds maps (api, headers), so it is not comparable. + return maps.EqualFunc(a.Servers, b.Servers, func(x, y routerconfig.Server) bool { return reflect.DeepEqual(x, y) }) && + maps.Equal(a.Agents, b.Agents) } // localCortexTarget is the config file and stats URL of the Cortex on this machine, when the diff --git a/cmd/agentop/tui/server_route_test.go b/cmd/agentop/tui/server_route_test.go index 98f8cf412..11d10d560 100644 --- a/cmd/agentop/tui/server_route_test.go +++ b/cmd/agentop/tui/server_route_test.go @@ -481,3 +481,15 @@ func TestAgentsPane_TheAgentsNameSurvivesTheServerColumnAt80(t *testing.T) { t.Errorf("the row does not show the agent's name:\n%s", view) } } + +func TestSameRouter_SeesAHeadersChange(t *testing.T) { + cfg := func(ua string) routerconfig.Config { + return routerconfig.Config{Servers: map[string]routerconfig.Server{"bob": {URL: "https://bob.example.com", Headers: map[string]string{"User-Agent": ua}}}} + } + if !sameRouter(cfg("bob-shell"), cfg("bob-shell")) { + t.Error("sameRouter = false for equal configs") + } + if sameRouter(cfg("bob-shell"), cfg("other")) { + t.Error("sameRouter = true for configs with different headers") + } +} diff --git a/core/listener/forwardproxy/inferencerouter_test.go b/core/listener/forwardproxy/inferencerouter_test.go index 75b4d7e3c..f1aac9ad3 100644 --- a/core/listener/forwardproxy/inferencerouter_test.go +++ b/core/listener/forwardproxy/inferencerouter_test.go @@ -1,10 +1,13 @@ package forwardproxy import ( + "bytes" + "context" "encoding/json" "io" "net/http" "net/http/httptest" + "path/filepath" "slices" "strings" "sync" @@ -16,6 +19,7 @@ import ( "github.com/rossoctl/cortex/core/plugins/inferenceparser" "github.com/rossoctl/cortex/core/plugins/inferencerouter" "github.com/rossoctl/cortex/core/session" + "github.com/rossoctl/cortex/core/storage/filestore" ) // seen is every request an origin received, by method and Authorization header — @@ -378,3 +382,247 @@ func TestForwardProxy_ASessionsOverrideMovesARunningSession(t *testing.T) { t.Errorf("ete's own turn received %v, want its own e1 and e2", got) } } + +const routedKey = "s-key" + +// routedServer is an upstream for the router's api tests. It lists claude-sonnet-5 at +// /v1/models, answers every other request with answer, and keeps those requests. +func routedServer(t *testing.T, answer http.HandlerFunc) *chatServer { + t.Helper() + return newChatServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && r.URL.Path == "/v1/models" { + _, _ = io.WriteString(w, `{"data":[{"id":"claude-sonnet-5"}]}`) + return + } + answer(w, r) + }) +} + +// sent are the requests up got, without the router's model-list fetches. +func sent(up *chatServer) []upstreamRequest { + var out []upstreamRequest + for _, r := range up.got() { + if r.path != "/v1/models" { + out = append(out, r) + } + } + return out +} + +// routedProxy is a forward proxy with inference-parser and an inference router that +// sends claude-code and opencode to up, a server with the extra fields given. The +// router's model list for up is claude-sonnet-5 from the start, so that a resend has +// a substitute. +func routedProxy(t *testing.T, up *chatServer, fields string) (*httptest.Server, *session.Store) { + t.Helper() + st, err := filestore.Open(filepath.Join(t.TempDir(), "store.json")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = st.Close() }) + _ = st.Set(context.Background(), inferencerouter.Name+"/models/s", `["claude-sonnet-5"]`, 0) + router := inferencerouter.New() + router.SetStore(st) + if err := router.Configure(json.RawMessage(`{ + "servers": {"s": {"url": "` + up.URL + `", "key": "` + routedKey + `", "main": "sonnet", "helper": "sonnet"` + fields + `}}, + "agents": {"claude-code": "s", "opencode": "s"} + }`)); err != nil { + t.Fatalf("Configure: %v", err) + } + if err := router.Init(context.Background()); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = router.Shutdown(context.Background()) }) + p, err := pipeline.New([]pipeline.Plugin{inferenceparser.NewInferenceParser(), router}) + if err != nil { + t.Fatalf("pipeline.New: %v", err) + } + store := newStore(t) + shared := memstore.New() + t.Cleanup(shared.Close) + srv := &Server{ + OutboundPipeline: pipeline.NewHolder(p), + Sessions: store, + Shared: shared, + SessionIDHeaders: []string{session.ClaudeCodeSessionHeader}, + Client: http.DefaultClient, + } + proxy := httptest.NewServer(srv.Handler()) + t.Cleanup(proxy.Close) + return proxy, store +} + +// claudeCodeRequest is Claude Code's request header set, with its X-Api-Key. +func claudeCodeRequest(r *http.Request) { + r.Header.Set("User-Agent", "claude-cli/2.1.286 (external, cli)") + r.Header.Set("X-Api-Key", "client-key") + r.Header.Set(session.ClaudeCodeSessionHeader, "s1") +} + +// Rule C2 and the auth test: Claude Code's X-Api-Key request reaches a chat server +// with only Authorization: Bearer , and a chat body. +func TestRouterAPI_ChatServerGetsBearerAndAChatBody(t *testing.T) { + up := routedServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, replyFixture(t, "bob_shape1.json")) + }) + proxy, _ := routedProxy(t, up, `, "api": {"chat-completions": "/v1/chat/completions"}`) + resp := postAnthropic(t, proxy, anthropicRequest, claudeCodeRequest) + body := replyText(t, resp) + + if resp.StatusCode != http.StatusOK || !strings.Contains(body, `"type":"message"`) { + t.Errorf("reply = %d %s, want an Anthropic message", resp.StatusCode, body) + } + reqs := sent(up) + if len(reqs) != 1 { + t.Fatalf("upstream got %d requests, want 1", len(reqs)) + } + r := reqs[0] + if r.path != "/v1/chat/completions" || !bytes.Contains(r.body, []byte(`"messages":[{"role":"user","content":[`)) { + t.Errorf("upstream got %s %s, want a chat body at /v1/chat/completions", r.path, r.body) + } + if got := r.header.Values("Authorization"); len(got) != 1 || got[0] != "Bearer "+routedKey { + t.Errorf("Authorization = %q, want only Bearer %s", got, routedKey) + } + for _, h := range []string{"X-Api-Key", "Api-Key", "Anthropic-Version"} { + if got := r.header.Values(h); len(got) != 0 { + t.Errorf("%s = %q, want none", h, got) + } + } +} + +// Rule 4a: nothing goes upstream, and the agent gets an Anthropic api_error. +func TestRouterAPI_ResponsesOnlyServerIsA502(t *testing.T) { + up := routedServer(t, func(w http.ResponseWriter, _ *http.Request) { t.Error("the server got a request") }) + proxy, _ := routedProxy(t, up, `, "api": {"responses": "/v1/responses"}`) + resp := postAnthropic(t, proxy, anthropicRequest, claudeCodeRequest) + body := replyText(t, resp) + + const msg = "Cortex cannot translate anthropic-messages to responses yet. Use a translation server for this pair " + + "(see https://github.com/rossoctl/cortex/blob/main/docs/translation-server.md)." + var got struct { + Type string `json:"type"` + Error struct{ Type, Message string } + } + if resp.StatusCode != http.StatusBadGateway || json.Unmarshal([]byte(body), &got) != nil || + got.Type != "error" || got.Error.Type != "api_error" || got.Error.Message != msg { + t.Errorf("reply = %d %s, want a 502 Anthropic api_error with %q", resp.StatusCode, body, msg) + } + if reqs := sent(up); len(reqs) != 0 { + t.Errorf("upstream got %d requests, want none", len(reqs)) + } +} + +// Rule C1: the agent's protocol is a key, so the request goes untranslated to its path. +func TestRouterAPI_AnthropicPathEntry(t *testing.T) { + up := routedServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"type":"message","role":"assistant","content":[]}`) + }) + proxy, _ := routedProxy(t, up, `, "api": {"anthropic-messages": "/anthropic/v1/messages"}`) + resp := postAnthropic(t, proxy, anthropicRequest, claudeCodeRequest) + _ = replyText(t, resp) + + reqs := sent(up) + if len(reqs) != 1 || reqs[0].path != "/anthropic/v1/messages" || string(reqs[0].body) != anthropicRequest { + t.Fatalf("upstream got %+v, want the body unchanged at /anthropic/v1/messages", reqs) + } + // Not translated, empty auth_header: setKey keeps the agent's header. + if got := reqs[0].header.Get("X-Api-Key"); got != routedKey { + t.Errorf("X-Api-Key = %q, want %s", got, routedKey) + } +} + +// Rule 7: a chat request cannot be decoded, so it is not translated and gets no 502. +func TestRouterAPI_AChatRequestIsNotTranslated(t *testing.T) { + up := routedServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, replyFixture(t, "bob_shape1.json")) + }) + proxy, _ := routedProxy(t, up, `, "api": {"anthropic-messages": "/v1/messages"}`) + const chatBody = `{"model":"claude-opus-5-5","messages":[{"role":"user","content":"hi"}]}` + req, _ := http.NewRequest(http.MethodPost, "http://api.openai.example/v1/chat/completions", strings.NewReader(chatBody)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "opencode/latest/2.0.21/cli") + resp, err := proxyClient(proxy, nil).Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + _, _ = io.ReadAll(resp.Body) + + if resp.StatusCode != http.StatusOK { + t.Errorf("status = %d, want 200", resp.StatusCode) + } + reqs := sent(up) + if len(reqs) != 1 || reqs[0].path != "/v1/chat/completions" || string(reqs[0].body) != chatBody { + t.Errorf("upstream got %+v, want the chat body unchanged at /v1/chat/completions", reqs) + } +} + +// Rule 6: count_tokens routed to a chat server is answered here with a 404. +func TestRouterAPI_CountTokensIsALocal404(t *testing.T) { + up := routedServer(t, func(w http.ResponseWriter, _ *http.Request) { t.Error("the server got a request") }) + proxy, _ := routedProxy(t, up, `, "api": {"chat-completions": "/v1/chat/completions"}`) + req, _ := http.NewRequest(http.MethodPost, up.URL+"/v1/messages/count_tokens", strings.NewReader(anthropicRequest)) + req.Header.Set("Content-Type", "application/json") + claudeCodeRequest(req) + resp, err := proxyClient(proxy, nil).Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + + if resp.StatusCode != http.StatusNotFound || !bytes.Contains(body, []byte(`"not_found_error"`)) { + t.Errorf("reply = %d %s, want a 404 not_found_error", resp.StatusCode, body) + } + if reqs := sent(up); len(reqs) != 0 { + t.Errorf("upstream got %d requests, want none", len(reqs)) + } +} + +// A translated request that the server refuses for its model is resent with the +// router's substitute, as an untranslated one is: LiteLLM's 403 and Bob's 404. +func TestRouterAPI_ATranslatedRefusalIsResent(t *testing.T) { + for _, refusal := range []struct { + name string + status int + fixture string + }{ + {"litellm 403", http.StatusForbidden, "error_403_litellm.json"}, + {"bob 404", http.StatusNotFound, "error_404_bob.json"}, + } { + t.Run(refusal.name, func(t *testing.T) { + var mu sync.Mutex + calls := 0 + up := routedServer(t, func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + calls++ + n := calls + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + if n == 1 { + w.WriteHeader(refusal.status) + _, _ = io.WriteString(w, replyFixture(t, refusal.fixture)) + return + } + _, _ = io.WriteString(w, replyFixture(t, "bob_shape1.json")) + }) + proxy, _ := routedProxy(t, up, `, "api": {"chat-completions": "/v1/chat/completions"}`) + resp := postAnthropic(t, proxy, anthropicRequest, claudeCodeRequest) + body := replyText(t, resp) + + if resp.StatusCode != http.StatusOK || !strings.Contains(body, "391 is not prime") { + t.Errorf("reply = %d %s, want the second answer, translated", resp.StatusCode, body) + } + reqs := sent(up) + if len(reqs) != 2 { + t.Fatalf("upstream got %d requests, want 2", len(reqs)) + } + if !bytes.Contains(reqs[1].body, []byte(`"model":"claude-sonnet-5"`)) || reqs[1].path != "/v1/chat/completions" { + t.Errorf("resent request = %s %s, want a chat body with the substitute", reqs[1].path, reqs[1].body) + } + }) + } +} diff --git a/core/listener/forwardproxy/server.go b/core/listener/forwardproxy/server.go index b43ea0250..98e19d674 100644 --- a/core/listener/forwardproxy/server.go +++ b/core/listener/forwardproxy/server.go @@ -35,6 +35,7 @@ import ( "github.com/rossoctl/cortex/core/spiffe" "github.com/rossoctl/cortex/core/tlsbridge" authtls "github.com/rossoctl/cortex/core/tlsconfig" + "github.com/rossoctl/cortex/core/translate" ) // maxRequestBodySize caps a buffered request body. It is separate from @@ -630,6 +631,8 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne sessionID = s.hydrateSession(pctx, trusted, r.Header, chain, connProcOf(r.Context())) } + // tr is the conversion a plugin asked for with pctx.Translate, nil for none. + var tr *translation if !skipped { action := s.OutboundPipeline.Run(r.Context(), pctx) @@ -666,6 +669,18 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne httpx.WriteRejectionForRequest(w, action, pctx) return } + + // After every request plugin, and before the request row is recorded, so the + // modify/translated record is on that row. A refusal goes nowhere upstream. + var refusal *pipeline.Action + if tr, refusal = startTranslation(pctx); refusal != nil { + s.recordOutboundRejectIn(tl, pctx, *refusal, s.recordingSessionID(sessionID, r.Header, chain)) + httpx.WriteRejectionForRequest(w, *refusal, pctx) + return + } + if tr != nil && capture != nil { + capture.Translated = tr.conv.Body + } } // A WebSocket upgrade is the one hop-by-hop request this proxy forwards @@ -749,6 +764,7 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne // body-rewrite block below and the transport, so leave them untouched. // Mirrors reverseproxy's forwarded-request header sync. httpx.SyncRequestHeaders(r, pctx.Headers) + applyUpstreamHeaders(r, pctx.UpstreamHeaders()) // If a WritesRequestBody plugin rewrote pctx.Body, ship the new bytes // upstream and clear Content-Encoding (see forwardproxy response @@ -756,6 +772,9 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne if pctx.BodyMutated() { httpx.SetRequestBody(r, pctx.Body) } + if tr != nil { + tr.apply(r) + } // The pipeline this request runs on for the rest of the handler: every // capability probe below (encoding, streaming, buffering) must answer @@ -772,7 +791,8 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne bufferResponse := !skipped && pinned.NeedsResponseBody() && (parsed || pinned.ReadsEveryResponseBody()) // Hop branch point 3: the hop always owns the encoding. - ownEncoding := hopped || bufferResponse || (!skipped && pinned.HasStreamingResponders()) + // A translated reply is converted, so the transport must decode it first. + ownEncoding := hopped || bufferResponse || tr != nil || (!skipped && pinned.HasStreamingResponders()) stripForUpstream(r, ownEncoding, forwardUpgrade) // Clear RequestURI — set by the server but must be empty for client requests @@ -840,7 +860,7 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne // itself, with whatever credential it has now. Left as a request-keyed rule // rather than made cleverer (#1549 review). if err == nil && !skipped && !forwardUpgrade { - resp, err = s.resendIfAsked(r, pctx, client, resp, tl, ownEncoding, capture) + resp, err = s.resendIfAsked(r, pctx, client, resp, tl, ownEncoding, capture, tr) } if err != nil { // The egress guard's refusal, BEFORE the generic 502: a destination that @@ -908,6 +928,22 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne } defer resp.Body.Close() + // The session archive's capture of the response (#1494), only when the request was + // captured, so a response never outlives its pair: the request row keeps its capture + // only for inference (recordCapturedRequest). + var rc *pipeline.ResponseCapture + if capture != nil && pctx.Extensions.Inference != nil { + rc = &pipeline.ResponseCapture{Status: resp.StatusCode, Decoded: resp.Uncompressed} + } + + // Before the wrap: a not-streamed translated reply is read whole there. + headersAt := time.Now() + // Before any reply plugin or recorder reads the body: they all see the reply in + // the agent's protocol. A hop's own 503 returned above and is not converted. + if tr != nil { + tr.wrapReply(resp, rc) + } + // Response phase: populate pctx and run plugins in reverse order. pctx.StatusCode = resp.StatusCode pctx.ResponseHeaders = resp.Header.Clone() @@ -918,7 +954,7 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne // own flow — the moment the delivered response's headers are in hand — // so ttfb means one thing wherever a row reports it. Stays zero on the // error path above, where no headers ever arrived (#1251). - pctx.ResponseHeadersAt = time.Now() + pctx.ResponseHeadersAt = headersAt // BytesDown for the response row (#1309), counted HERE and nowhere else. Every // arm below reads this one body — buffered whole, relayed SSE block by block, relayed @@ -939,10 +975,9 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne // // The same wrapper keeps the body for the session archive's capture (#1494): every arm // reads through it, so what it keeps is the body as it came off the wire, whatever arm - // relays it. Only when the request was captured, so a response never outlives its pair: the - // request row keeps its capture only for inference (recordCapturedRequest). - if capture != nil && pctx.Extensions.Inference != nil { - rc := &pipeline.ResponseCapture{Status: resp.StatusCode, Header: redact.Headers(resp.Header), Decoded: resp.Uncompressed} + // relays it. For a translated reply that is the reply in the agent's protocol. + if rc != nil { + rc.Header = redact.Headers(resp.Header) pctx.ResponseCapture = rc resp.Body = bodycount.WrapCapture(resp.Body, &pctx.ResponseBytes, &rc.Body, &rc.Truncated, &rc.Ended, pipeline.MaxResponseCapture) } else { @@ -1260,7 +1295,11 @@ func stripForUpstream(r *http.Request, ownEncoding, keepUpgrade bool) { // id, so the timeline shows both attempts and each pairs with its own answer. A // refused answer that is not sent again goes to the client byte for byte, the bytes // read for the Resender included. -func (s *Server) resendIfAsked(r *http.Request, pctx *pipeline.Context, client *http.Client, resp *http.Response, tl *tunnelLog, ownEncoding bool, capture *pipeline.RequestCapture) (*http.Response, error) { +// +// tr is the translation of the request, nil for none. The Resender reads the raw +// answer, the request is converted again after the Resender changed it, and the +// refused answer's row has the reply in the agent's protocol. +func (s *Server) resendIfAsked(r *http.Request, pctx *pipeline.Context, client *http.Client, resp *http.Response, tl *tunnelLog, ownEncoding bool, capture *pipeline.RequestCapture, tr *translation) (*http.Response, error) { // Both probes and Resend itself answer for the pipeline this request // ran on (Holder.ForPctx): the reload-drain window must not flip the // resend decision onto plugins that never saw the request. @@ -1272,7 +1311,19 @@ func (s *Server) resendIfAsked(r *http.Request, pctx *pipeline.Context, client * // The parse as the refused attempt was sent: a Resender may rewrite the model, // and the refused attempt's row must name the one the upstream refused. sent := pipeline.SnapshotInference(pctx.Extensions.Inference) - if !s.OutboundPipeline.Resend(r.Context(), pctx, resp.StatusCode, head) { + asked := s.OutboundPipeline.Resend(r.Context(), pctx, resp.StatusCode, head) + var conv translate.Converted + if asked && tr != nil { + var convErr error + // A request that cannot be converted after the change is not sent again. + conv, convErr = tr.convert(pctx) + asked = convErr == nil + if !asked { + // The answer goes back for the request sent, so its row names that model. + pctx.Extensions.Inference = sent + } + } + if !asked { resp.Body = struct { io.Reader io.Closer @@ -1285,7 +1336,11 @@ func (s *Server) resendIfAsked(r *http.Request, pctx *pipeline.Context, client * // back at once, because the status of the answer the client gets is the // one OutcomeFromContext must read. resent := pctx.Extensions.Inference - pctx.StatusCode, pctx.ResponseBody, pctx.Extensions.Inference = resp.StatusCode, head, sent + refused := head + if tr != nil { + refused = tr.replyBody(resp, head) + } + pctx.StatusCode, pctx.ResponseBody, pctx.Extensions.Inference = resp.StatusCode, refused, sent if capture != nil && pctx.Extensions.Inference != nil { // The refusal as far as it was read: the resend peeks at most ResendPeekLimit bytes. pctx.ResponseCapture = &pipeline.ResponseCapture{Status: resp.StatusCode, Header: redact.Headers(resp.Header), @@ -1294,11 +1349,23 @@ func (s *Server) resendIfAsked(r *http.Request, pctx *pipeline.Context, client * s.recordOutboundResponseEvent(pctx, resp.StatusCode, nil) pctx.StatusCode, pctx.ResponseBody, pctx.Extensions.Inference = 0, nil, resent pctx.RenewRequestID() + if tr != nil && capture != nil { + // A copy: the refused attempt's row keeps the body it sent. + again := *capture + again.Translated = conv.Body + capture = &again + } s.recordCapturedRequest(tl, pctx, pctx.OutboundSessionID, capture) } again := r.Clone(r.Context()) httpx.SyncRequestHeaders(again, pctx.Headers) - httpx.SetRequestBody(again, pctx.Body) + applyUpstreamHeaders(again, pctx.UpstreamHeaders()) + if tr != nil { + tr.conv = conv + tr.apply(again) + } else { + httpx.SetRequestBody(again, pctx.Body) + } // No keepUpgrade: a resent request is never a forwarded upgrade — the // 101 branch in serveOutbound skips resendIfAsked entirely, and only that // branch leaves the headers unstripped. diff --git a/core/listener/forwardproxy/translate.go b/core/listener/forwardproxy/translate.go new file mode 100644 index 000000000..63d9c98d9 --- /dev/null +++ b/core/listener/forwardproxy/translate.go @@ -0,0 +1,187 @@ +package forwardproxy + +import ( + "bytes" + "errors" + "fmt" + "io" + "net/http" + "strings" + + "github.com/rossoctl/cortex/core/listener/httpx" + "github.com/rossoctl/cortex/core/listener/internal/bodycount" + "github.com/rossoctl/cortex/core/pipeline" + "github.com/rossoctl/cortex/core/translate" + "github.com/rossoctl/cortex/core/translate/pivot" +) + +// translation is a request that the listener converts to the protocol of the +// server, as a plugin asked with pctx.Translate. It goes through the handler as a +// local, not through pctx, so no plugin can change it. +type translation struct { + source, target, path string + conv translate.Converted + dec pivot.ReplyDecoding // the target's reply decoder + enc pivot.ReplyEncoding // the source's reply encoder +} + +// startTranslation converts pctx.Body when a plugin asked for it. It is called after +// every request plugin has run (rule 8). It gives nil, nil when no plugin asked. On a +// refusal it gives the Reject action, with the error body in the agent's protocol; +// nothing must then go upstream. On success it records modify/translated, with the +// fields that the target could not carry (decision 17). +func startTranslation(pctx *pipeline.Context) (*translation, *pipeline.Action) { + src, tgt, path, ok := pctx.TranslateTarget() + if !ok { + return nil, nil + } + t := &translation{source: src, target: tgt, path: path} + conv, err := t.convert(pctx) + if err != nil { + status, errType := http.StatusBadGateway, "api_error" + var re *pivot.RequestError + if errors.As(err, &re) { + status, errType = re.Status, re.Type + } + msg := err.Error() + if re != nil { + msg = re.Message + } + pctx.Record(pipeline.Invocation{ + Plugin: "forward-proxy", + Action: pipeline.ActionDeny, + Phase: pipeline.InvocationPhaseRequest, + Reason: "translation_refused", + Details: map[string]string{"from": src, "to": tgt, "message": msg}, + }) + return nil, &pipeline.Action{Type: pipeline.Reject, Violation: &pipeline.Violation{ + Code: "inference.translation_refused", + Reason: msg, + Status: status, + Body: translate.ErrorBody(src, status, errType, msg), + BodyType: "application/json", + }} + } + details := map[string]string{"from": src, "to": tgt, "path": path} + if len(conv.Dropped) > 0 { + details["dropped"] = strings.Join(conv.Dropped, ",") + } + pctx.Record(pipeline.Invocation{ + Plugin: "forward-proxy", + Action: pipeline.ActionModify, + Phase: pipeline.InvocationPhaseRequest, + Reason: "translated", + Details: details, + }) + t.conv = conv + return t, nil +} + +// convert converts pctx.Body. It removes the source protocol's own headers from +// pctx.Headers (decision 18), so the header sync removes them from the wire. It +// refuses a pair whose reply Cortex cannot convert back, before anything is sent. +func (t *translation) convert(pctx *pipeline.Context) (translate.Converted, error) { + if pctx.Body == nil { + return translate.Converted{}, errors.New("the request body was not buffered, so Cortex cannot translate it") + } + tp, _ := translate.Lookup(t.target) + sp, _ := translate.Lookup(t.source) + var okDec, okEnc bool + t.dec, okDec = tp.(pivot.ReplyDecoding) + t.enc, okEnc = sp.(pivot.ReplyEncoding) + if !okDec || !okEnc { + return translate.Converted{}, errors.New("Cortex cannot convert a " + t.target + " reply to " + t.source) + } + return translate.Request(t.source, t.target, pctx.Body, pctx.Headers) +} + +// applyUpstreamHeaders sets the headers that a plugin gave with +// pctx.SetUpstreamHeaders on the upstream request. Each one replaces a header of +// the same name (rule C3). It runs after the sync of pctx.Headers. +func applyUpstreamHeaders(r *http.Request, h http.Header) { + for k, v := range h { + r.Header[http.CanonicalHeaderKey(k)] = append([]string(nil), v...) + } +} + +// apply puts the converted body and the target path on the upstream request, with +// no query string. pctx.Path does not change: the request row keeps the client's path. +func (t *translation) apply(r *http.Request) { + httpx.SetRequestBody(r, t.conv.Body) + r.URL.Path = t.path + r.URL.RawPath, r.URL.RawQuery, r.URL.ForceQuery = "", "", false +} + +// agentStream tells if the agent gets a stream for an upstream answer with this +// status: only a 2xx answer to a stream request (decision 4). +func (t *translation) agentStream(status int) bool { + return t.conv.Stream && status/100 == 2 +} + +// wrapReply makes resp carry the reply in the agent's protocol. A stream is +// converted as it is read (rule 10). Every later arm then reads the converted bytes. When rc is not nil, +// it keeps the server's reply before the conversion in rc.Upstream (rule 8a). +func (t *translation) wrapReply(resp *http.Response, rc *pipeline.ResponseCapture) { + if rc != nil { + // The count and the end are the converted body's, which the caller's wrapper keeps. + var unused int64 + var ended bool + rc.Upstream = []byte{} // not nil: the reply is converted, also when it is empty + resp.Body = bodycount.WrapCapture(resp.Body, &unused, &rc.Upstream, &rc.UpstreamTruncated, &ended, pipeline.MaxResponseCapture) + } + stream := t.agentStream(resp.StatusCode) + upstreamStream := httpx.IsEventStream(resp.Header.Get("Content-Type")) + if stream { + dec := t.dec.NewReplyDecoder(resp.StatusCode, upstreamStream) + resp.Body = translate.NewReplyReader(resp.Body, dec, t.enc.NewReplyEncoder(true)) + resp.Header.Set("Content-Type", "text/event-stream") + } else { + t.convertWhole(resp, upstreamStream) + resp.Header.Set("Content-Type", "application/json") + } + resp.Header.Del("Content-Length") + resp.Header.Del("Content-Encoding") + resp.ContentLength = -1 + if rc != nil { + rc.Status = resp.StatusCode + } +} + +// convertWhole converts a reply that the agent gets as one body. It reads the +// whole body before the status is sent, so that an error that the decoder finds +// in a 2xx body gives its own status, not the 2xx. +func (t *translation) convertWhole(resp *http.Response, upstreamStream bool) { + upstream := resp.Body + raw, err := io.ReadAll(io.LimitReader(upstream, maxBodySize+1)) + var body io.Reader + switch { + case err != nil: + // The arm that reads the body reports the failed read. + body = &failedRead{err} + case len(raw) > maxBodySize: + resp.StatusCode = http.StatusBadGateway + body = bytes.NewReader(translate.ErrorBody(t.source, http.StatusBadGateway, "api_error", + fmt.Sprintf("the inference server's reply is over the %d-byte buffer limit", maxBodySize))) + default: + status, out := translate.Reply(t.source, t.target, resp.StatusCode, upstreamStream, false, raw) + resp.StatusCode = status + body = bytes.NewReader(out) + } + resp.Body = struct { + io.Reader + io.Closer + }{body, upstream} +} + +// failedRead gives err at the first Read. +type failedRead struct{ err error } + +func (f *failedRead) Read([]byte) (int, error) { return 0, f.err } + +// replyBody converts a whole upstream body, for the row of a refused attempt +// (decision 19). +func (t *translation) replyBody(resp *http.Response, body []byte) []byte { + _, out := translate.Reply(t.source, t.target, resp.StatusCode, + httpx.IsEventStream(resp.Header.Get("Content-Type")), t.agentStream(resp.StatusCode), body) + return out +} diff --git a/core/listener/forwardproxy/translate_test.go b/core/listener/forwardproxy/translate_test.go new file mode 100644 index 000000000..366e32825 --- /dev/null +++ b/core/listener/forwardproxy/translate_test.go @@ -0,0 +1,721 @@ +package forwardproxy + +import ( + "bufio" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "slices" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/rossoctl/cortex/core/pipeline" + "github.com/rossoctl/cortex/core/plugins/inferenceparser" + "github.com/rossoctl/cortex/core/session" +) + +// translator redirects every request to target and asks the listener to translate +// it from Anthropic Messages to chat completions, as the inference router does. When +// translate is false it only redirects. headers, when set, go to SetUpstreamHeaders. +// resend, when set, is the answer to each Resend, and heads keeps what it was shown. +type translator struct { + target string + translate bool + headers http.Header + resend bool + + mu sync.Mutex + heads []string +} + +func (p *translator) Name() string { return "translator" } +func (p *translator) Capabilities() pipeline.PluginCapabilities { + return pipeline.PluginCapabilities{WritesDestination: true, ReadsBody: true, Description: "test"} +} +func (p *translator) OnRequest(_ context.Context, pctx *pipeline.Context) pipeline.Action { + u, _ := url.Parse(p.target) + if err := pctx.Redirect(u); err != nil { + panic(err) + } + if p.translate { + if err := pctx.Translate("anthropic-messages", "chat-completions", "/v1/chat/completions"); err != nil { + panic(err) + } + } + if p.headers != nil { + if err := pctx.SetUpstreamHeaders(p.headers); err != nil { + panic(err) + } + } + return pipeline.Action{Type: pipeline.Continue} +} +func (p *translator) OnResponse(context.Context, *pipeline.Context) pipeline.Action { + return pipeline.Action{Type: pipeline.Continue} +} + +// resendingTranslator is a translator that is also a Resender. When substitute is set, +// each Resend sets the request's model to it. When body is set, each Resend then sets +// the request body to it. +type resendingTranslator struct { + *translator + substitute string + body string +} + +func (p resendingTranslator) Capabilities() pipeline.PluginCapabilities { + c := p.translator.Capabilities() + c.WritesRequestBody = p.substitute != "" || p.body != "" + return c +} + +func (p resendingTranslator) Resend(_ context.Context, pctx *pipeline.Context, _ int, body []byte) bool { + p.mu.Lock() + defer p.mu.Unlock() + p.heads = append(p.heads, string(body)) + if p.substitute != "" { + if err := pctx.SetRequestModel(p.substitute); err != nil { + panic(err) + } + } + if p.body != "" { + pctx.SetBody([]byte(p.body)) + } + return p.resend +} + +// upstreamRequest is one request the chat server got. +type upstreamRequest struct { + path, rawQuery string + header http.Header + body []byte +} + +// chatServer serves answer for every request and keeps what it got. +type chatServer struct { + *httptest.Server + mu sync.Mutex + reqs []upstreamRequest +} + +func newChatServer(t *testing.T, answer http.HandlerFunc) *chatServer { + t.Helper() + c := &chatServer{} + c.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + c.mu.Lock() + c.reqs = append(c.reqs, upstreamRequest{r.URL.Path, r.URL.RawQuery, r.Header.Clone(), body}) + c.mu.Unlock() + answer(w, r) + })) + t.Cleanup(c.Close) + return c +} + +func (c *chatServer) got() []upstreamRequest { + c.mu.Lock() + defer c.mu.Unlock() + return append([]upstreamRequest(nil), c.reqs...) +} + +// replyFixture is a hand-written chat reply from the translate package's testdata. +func replyFixture(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile("../../translate/testdata/reply/" + name) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +const anthropicRequest = `{"model":"claude-opus-5-5","max_tokens":1024,"messages":[{"role":"user","content":"is 391 prime?"}]}` + +const anthropicStreamRequest = `{"model":"claude-opus-5-5","max_tokens":1024,"stream":true,"messages":[{"role":"user","content":"is 391 prime?"}]}` + +// postAnthropic sends an Anthropic Messages request through proxy to the agent's +// provider. edit, when set, changes the request before it is sent. +func postAnthropic(t *testing.T, proxy *httptest.Server, body string, edit func(*http.Request)) *http.Response { + t.Helper() + req, _ := http.NewRequest(http.MethodPost, "http://api.anthropic.example/v1/messages", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Anthropic-Version", "2023-06-01") + if edit != nil { + edit(req) + } + resp, err := proxyClient(proxy, nil).Do(req) + if err != nil { + t.Fatalf("request: %v", err) + } + t.Cleanup(func() { _ = resp.Body.Close() }) + return resp +} + +func replyText(t *testing.T, resp *http.Response) string { + t.Helper() + b, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read reply: %v", err) + } + return string(b) +} + +// translatedInvocation is the forward proxy's modify/translated record on a request +// row of store, or nil. +func translatedInvocation(store *session.Store) (*pipeline.SessionEvent, *pipeline.Invocation) { + for _, s := range store.ListSessions() { + v := store.View(s.ID) + if v == nil { + continue + } + for i := range v.Events { + e := &v.Events[i] + if e.Phase != pipeline.SessionRequest || e.Invocations == nil { + continue + } + for j := range e.Invocations.Outbound { + inv := &e.Invocations.Outbound[j] + if inv.Plugin == "forward-proxy" && inv.Action == pipeline.ActionModify && inv.Reason == "translated" { + return e, inv + } + } + } + } + return nil, nil +} + +// outboundRows are store's outbound rows of phase, in the order they were recorded. +func outboundRows(store *session.Store, phase pipeline.SessionPhase) []pipeline.SessionEvent { + var out []pipeline.SessionEvent + for _, s := range store.ListSessions() { + v := store.View(s.ID) + if v == nil { + continue + } + for _, e := range v.Events { + if e.Direction == pipeline.Outbound && e.Phase == phase { + out = append(out, e) + } + } + } + return out +} + +// waitRows waits for n outbound rows of phase: the last response row can be recorded +// after the client has read the reply. +func waitRows(t *testing.T, store *session.Store, phase pipeline.SessionPhase, n int) []pipeline.SessionEvent { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for { + rows := outboundRows(store, phase) + if len(rows) >= n || time.Now().After(deadline) { + return rows + } + time.Sleep(10 * time.Millisecond) + } +} + +func newStore(t *testing.T) *session.Store { + store := session.New(5*time.Minute, 100, 0) + t.Cleanup(store.Close) + return store +} + +func TestTranslate_StreamedReplyRecordedAndIncremental(t *testing.T) { + chunks := strings.SplitAfter(replyFixture(t, "bob_shape1.sse"), "\n\n") + release := make(chan struct{}) + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + // The ping comment and the first two chunks, then wait. + _, _ = io.WriteString(w, strings.Join(chunks[:3], "")) + w.(http.Flusher).Flush() + <-release + _, _ = io.WriteString(w, strings.Join(chunks[3:], "")) + }) + store := newStore(t) + proxy := newRedirectProxy(t, store, nil, inferenceparser.NewInferenceParser(), &translator{target: up.URL, translate: true}) + + resp := postAnthropic(t, proxy, anthropicStreamRequest, nil) + if ct := resp.Header.Get("Content-Type"); !strings.HasPrefix(ct, "text/event-stream") { + t.Fatalf("Content-Type = %q, want text/event-stream", ct) + } + br := bufio.NewReader(resp.Body) + done := make(chan string, 1) + go func() { + for { + line, err := br.ReadString('\n') + if line == "event: message_start\n" { + done <- line + return + } + if err != nil { + done <- "" + return + } + } + }() + select { + case got := <-done: + if got == "" { + t.Fatal("the reply ended with no message_start") + } + case <-time.After(5 * time.Second): + close(release) + t.Fatal("message_start did not reach the client before the upstream finished") + } + close(release) + rest, _ := io.ReadAll(br) + if !strings.Contains(string(rest), "event: message_stop") { + t.Errorf("the reply has no message_stop: %q", rest) + } + + // The request on the wire is chat-shaped and has no Anthropic headers. + reqs := up.got() + if len(reqs) != 1 || reqs[0].path != "/v1/chat/completions" { + t.Fatalf("upstream requests = %+v, want one to /v1/chat/completions", reqs) + } + if reqs[0].header.Get("Anthropic-Version") != "" { + t.Errorf("upstream saw Anthropic-Version %q", reqs[0].header.Get("Anthropic-Version")) + } + var chatBody struct { + Messages []json.RawMessage `json:"messages"` + } + if err := json.Unmarshal(reqs[0].body, &chatBody); err != nil || len(chatBody.Messages) == 0 { + t.Errorf("upstream body is not a chat request: %s", reqs[0].body) + } + + // The response row has the Anthropic inference. + deadline := time.Now().Add(5 * time.Second) + var inf *pipeline.InferenceExtension + for time.Now().Before(deadline) { + if inf = lastInferenceResponse(t, store); inf != nil && inf.Completion != "" { + break + } + time.Sleep(10 * time.Millisecond) + } + if inf == nil || inf.Completion != "391 is not prime" { + t.Fatalf("response inference = %+v, want completion %q", inf, "391 is not prime") + } + if len(inf.ToolCalls) != 1 || inf.ToolCalls[0].Name != "Bash" { + t.Errorf("tool calls = %+v, want one Bash", inf.ToolCalls) + } + + _, inv := translatedInvocation(store) + if inv == nil { + t.Fatal("no modify/translated invocation on the request row") + } + want := map[string]string{"from": "anthropic-messages", "to": "chat-completions", "path": "/v1/chat/completions"} + for k, v := range want { + if inv.Details[k] != v { + t.Errorf("Details[%q] = %q, want %q", k, inv.Details[k], v) + } + } + if _, ok := inv.Details["dropped"]; ok { + t.Errorf("Details has dropped %q for a request that dropped nothing", inv.Details["dropped"]) + } +} + +func TestTranslate_NoQueryString(t *testing.T) { + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, replyFixture(t, "bob_shape1.json")) + }) + store := newStore(t) + proxy := newRedirectProxy(t, store, nil, &translator{target: up.URL, translate: true}) + resp := postAnthropic(t, proxy, anthropicRequest, func(r *http.Request) { r.URL.RawQuery = "beta=true" }) + _ = replyText(t, resp) + + reqs := up.got() + if len(reqs) != 1 || reqs[0].path != "/v1/chat/completions" || reqs[0].rawQuery != "" { + t.Fatalf("upstream requests = %+v, want /v1/chat/completions with no query", reqs) + } + ev, _ := translatedInvocation(store) + if ev == nil || ev.HTTPPath != "/v1/messages" { + t.Fatalf("request row = %+v, want path /v1/messages", ev) + } +} + +func TestTranslate_UpstreamHeadersReplace(t *testing.T) { + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, replyFixture(t, "bob_shape1.json")) + }) + proxy := newRedirectProxy(t, nil, nil, &translator{target: up.URL, translate: true, + headers: http.Header{"User-Agent": {"bob-shell"}}}) + resp := postAnthropic(t, proxy, anthropicRequest, func(r *http.Request) { r.Header.Set("User-Agent", "claude-cli/2") }) + _ = replyText(t, resp) + + reqs := up.got() + if len(reqs) != 1 { + t.Fatalf("upstream got %d requests, want 1", len(reqs)) + } + if got := reqs[0].header.Values("User-Agent"); len(got) != 1 || got[0] != "bob-shell" { + t.Errorf("upstream User-Agent = %q, want only bob-shell", got) + } +} + +func TestTranslate_GzipUpstream(t *testing.T) { + up := newChatServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + _, _ = io.WriteString(w, replyFixture(t, "bob_shape1.json")) + return + } + w.Header().Set("Content-Encoding", "gzip") + zw := gzip.NewWriter(w) + _, _ = io.WriteString(zw, replyFixture(t, "bob_shape1.json")) + _ = zw.Close() + }) + proxy := newRedirectProxy(t, nil, nil, &translator{target: up.URL, translate: true}) + resp := postAnthropic(t, proxy, anthropicRequest, func(r *http.Request) { r.Header.Set("Accept-Encoding", "gzip") }) + body := replyText(t, resp) + + if ce := resp.Header.Get("Content-Encoding"); ce != "" { + t.Errorf("Content-Encoding = %q, want none", ce) + } + var msg struct { + Type string `json:"type"` + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + } + if err := json.Unmarshal([]byte(body), &msg); err != nil || msg.Type != "message" { + t.Fatalf("reply is not an Anthropic message (%v): %q", err, body) + } + if !strings.Contains(body, "391 is not prime") { + t.Errorf("reply has no text: %s", body) + } +} + +func TestTranslate_AgentStreamsServerAnswersJSON(t *testing.T) { + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, replyFixture(t, "bob_shape1.json")) + }) + proxy := newRedirectProxy(t, nil, nil, &translator{target: up.URL, translate: true}) + resp := postAnthropic(t, proxy, anthropicStreamRequest, nil) + body := replyText(t, resp) + + if ct := resp.Header.Get("Content-Type"); !strings.HasPrefix(ct, "text/event-stream") { + t.Errorf("Content-Type = %q, want text/event-stream", ct) + } + if !strings.HasPrefix(body, "event: message_start\n") || !strings.Contains(body, "event: message_stop") { + t.Errorf("reply is not an Anthropic stream: %q", body) + } +} + +func TestTranslate_ErrorStatus(t *testing.T) { + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = io.WriteString(w, replyFixture(t, "error_429_openai.json")) + }) + proxy := newRedirectProxy(t, nil, nil, &translator{target: up.URL, translate: true}) + resp := postAnthropic(t, proxy, anthropicStreamRequest, nil) + body := replyText(t, resp) + + if resp.StatusCode != http.StatusTooManyRequests { + t.Errorf("status = %d, want 429", resp.StatusCode) + } + var e struct { + Type string `json:"type"` + Error struct { + Type string `json:"type"` + } `json:"error"` + } + if err := json.Unmarshal([]byte(body), &e); err != nil || e.Type != "error" || e.Error.Type != "rate_limit_error" { + t.Errorf("reply = %q, want an Anthropic rate_limit_error body", body) + } +} + +func TestTranslate_DocumentRefused(t *testing.T) { + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) {}) + store := newStore(t) + proxy := newRedirectProxy(t, store, nil, &translator{target: up.URL, translate: true}) + doc := `{"model":"claude-opus-5-5","max_tokens":1024,"messages":[{"role":"user","content":[` + + `{"type":"document","source":{"type":"base64","media_type":"application/pdf","data":"JVBERi0="}}]}]}` + resp := postAnthropic(t, proxy, doc, nil) + body := replyText(t, resp) + + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("status = %d, want 400", resp.StatusCode) + } + if !strings.Contains(body, `"invalid_request_error"`) || !strings.Contains(body, "documents") { + t.Errorf("reply = %q, want an invalid_request_error that names documents", body) + } + if n := len(up.got()); n != 0 { + t.Errorf("upstream got %d requests, want 0", n) + } + denied := waitRows(t, store, pipeline.SessionDenied, 1) + if len(denied) != 1 { + t.Fatalf("%d denied rows, want 1", len(denied)) + } + d := denied[0] + if d.StatusCode != http.StatusBadRequest || d.Error == nil || d.Error.Code != "inference.translation_refused" { + t.Errorf("denied row = status %d error %+v, want 400 inference.translation_refused", d.StatusCode, d.Error) + } + var found bool + if d.Invocations != nil { + for _, inv := range d.Invocations.Outbound { + found = found || (inv.Plugin == "forward-proxy" && inv.Action == pipeline.ActionDeny && inv.Reason == "translation_refused") + } + } + if !found { + t.Errorf("denied row invocations = %+v, want forward-proxy deny/translation_refused", d.Invocations) + } +} + +func TestTranslate_DroppedRecorded(t *testing.T) { + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, replyFixture(t, "bob_shape1.json")) + }) + store := newStore(t) + proxy := newRedirectProxy(t, store, nil, &translator{target: up.URL, translate: true}) + req := `{"model":"claude-opus-5-5","max_tokens":1024,"top_k":5,"metadata":{"user_id":"u"},` + + `"messages":[{"role":"user","content":"hi"}]}` + _ = replyText(t, postAnthropic(t, proxy, req, nil)) + + _, inv := translatedInvocation(store) + if inv == nil { + t.Fatal("no modify/translated invocation") + } + if got := inv.Details["dropped"]; got != "metadata,top_k" { + t.Errorf("dropped = %q, want metadata,top_k", got) + } +} + +// refusedThenAnswered serves a LiteLLM 403 to the first request and the chat reply to +// every later one. +func refusedThenAnswered(t *testing.T) *chatServer { + var calls int + var mu sync.Mutex + return newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + calls++ + n := calls + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + if n == 1 { + w.WriteHeader(http.StatusForbidden) + _, _ = io.WriteString(w, replyFixture(t, "error_403_litellm.json")) + return + } + _, _ = io.WriteString(w, replyFixture(t, "bob_shape1.json")) + }) +} + +func TestTranslate_ResendSeesRawReply(t *testing.T) { + up := refusedThenAnswered(t) + tr := &translator{target: up.URL, translate: true, resend: true, headers: http.Header{"User-Agent": {"bob-shell"}}} + store := newStore(t) + rec := &captures{} + store.AddRecorder(rec) + proxy := newRedirectProxy(t, store, nil, inferenceparser.NewInferenceParser(), resendingTranslator{translator: tr, substitute: "claude-sonnet-5"}) + resp := postAnthropic(t, proxy, anthropicRequest, func(r *http.Request) { r.Header.Set("User-Agent", "claude-cli/2") }) + body := replyText(t, resp) + + if resp.StatusCode != http.StatusOK || !strings.Contains(body, "391 is not prime") { + t.Errorf("reply = %d %q, want the second answer, translated", resp.StatusCode, body) + } + tr.mu.Lock() + heads := append([]string(nil), tr.heads...) + tr.mu.Unlock() + if len(heads) != 1 || !strings.Contains(heads[0], "team_model_access_denied") { + t.Errorf("Resender saw %q, want the raw 403 body", heads) + } + reqs := up.got() + if len(reqs) != 2 { + t.Fatalf("upstream got %d requests, want 2", len(reqs)) + } + for i, r := range reqs { + if r.path != "/v1/chat/completions" || !bytes.Contains(r.body, []byte(`"messages":[{"role":"user","content":[`)) { + t.Errorf("upstream request %d = %s %s, want a chat body", i, r.path, r.body) + } + // The upstream headers go on the resent request too, after the header sync. + if got := r.header.Values("User-Agent"); len(got) != 1 || got[0] != "bob-shell" { + t.Errorf("upstream request %d User-Agent = %q, want only bob-shell", i, got) + } + } + // Each attempt's row holds the chat body that attempt sent: the refused row the + // first model, the resent row the substitute. + caps := rec.all() + if len(caps) != 2 { + t.Fatalf("%d request captures, want 2", len(caps)) + } + var translated []string + for _, c := range caps { + translated = append(translated, string(c.Translated)) + } + sort.Strings(translated) + want := []string{string(reqs[0].body), string(reqs[1].body)} + sort.Strings(want) + if !slices.Equal(translated, want) || want[0] == want[1] { + t.Errorf("Translated = %q, want the two chat bodies sent upstream, %q", translated, want) + } + if !bytes.Contains(reqs[1].body, []byte(`"model":"claude-sonnet-5"`)) { + t.Errorf("resent body = %s, want the substitute model", reqs[1].body) + } + // Decision 19: the refused attempt's row has the refusal in the agent's protocol. + // The raw body's type is team_model_access_denied; Anthropic's for a 403 is + // permission_error. + rows := waitRows(t, store, pipeline.SessionResponse, 2) + if len(rows) != 2 { + t.Fatalf("%d response rows, want 2", len(rows)) + } + refused := rows[0] + if refused.StatusCode != http.StatusForbidden || refused.Error == nil || refused.Error.Message != "permission_error" { + t.Errorf("refused row = status %d error %+v, want 403 with permission_error", refused.StatusCode, refused.Error) + } + if refused.Inference == nil || refused.Inference.Model != "claude-opus-5-5" { + t.Errorf("refused row inference = %+v, want the refused model claude-opus-5-5", refused.Inference) + } + if rows[1].StatusCode != http.StatusOK { + t.Errorf("resent row status = %d, want 200", rows[1].StatusCode) + } +} + +// A Resender asks for a resend, but its change makes a request that cannot be +// converted. The refused answer then goes to the agent, translated, and nothing is +// sent again. +func TestTranslate_ResendNotConvertible(t *testing.T) { + up := refusedThenAnswered(t) + doc := `{"model":"claude-sonnet-5","max_tokens":1024,"messages":[{"role":"user","content":[` + + `{"type":"document","source":{"type":"base64","media_type":"application/pdf","data":"JVBERi0="}}]}]}` + tr := &translator{target: up.URL, translate: true, resend: true} + store := newStore(t) + proxy := newRedirectProxy(t, store, nil, inferenceparser.NewInferenceParser(), + resendingTranslator{translator: tr, substitute: "claude-sonnet-5", body: doc}) + resp := postAnthropic(t, proxy, anthropicRequest, nil) + body := replyText(t, resp) + + if resp.StatusCode != http.StatusForbidden || !strings.Contains(body, `"permission_error"`) { + t.Errorf("reply = %d %q, want the 403 as an Anthropic permission_error", resp.StatusCode, body) + } + if n := len(up.got()); n != 1 { + t.Errorf("upstream got %d requests, want 1", n) + } + rows := waitRows(t, store, pipeline.SessionResponse, 1) + if len(rows) != 1 { + t.Fatalf("%d response rows, want 1", len(rows)) + } + // The row names the model the upstream refused, not the Resender's change. + if r := rows[0]; r.StatusCode != http.StatusForbidden || r.Inference == nil || r.Inference.Model != "claude-opus-5-5" { + t.Errorf("response row = status %d inference %+v, want 403 for claude-opus-5-5", r.StatusCode, r.Inference) + } +} + +func TestTranslate_NotTranslatedPassesBytes(t *testing.T) { + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "ok") + }) + proxy := newRedirectProxy(t, nil, nil, &translator{target: up.URL}) + _ = replyText(t, postAnthropic(t, proxy, anthropicRequest, nil)) + + reqs := up.got() + if len(reqs) != 1 || string(reqs[0].body) != anthropicRequest || reqs[0].path != "/v1/messages" { + t.Fatalf("upstream requests = %+v, want the client body unchanged", reqs) + } +} + +// Rule 8a: the capture of a translated request holds four bodies, so a conversion bug +// can be found from the archive: what the agent sent, what the plugins made of it (the +// input of the conversion), the chat body sent upstream, and the server's raw reply +// beside the Anthropic reply that the agent got. +func TestTranslate_CaptureHoldsFourBodies(t *testing.T) { + sse := replyFixture(t, "bob_shape1.sse") + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, sse) + }) + rewritten := strings.Replace(anthropicStreamRequest, "391", "393", 1) + proxy, rec := proxyWithCaptures(t, inferenceparser.NewInferenceParser(), + &translator{target: up.URL, translate: true}, &bodyMutatorPlugin{newBody: []byte(rewritten)}) + + resp := postAnthropic(t, proxy, anthropicStreamRequest, nil) + reply := replyText(t, resp) + if !strings.Contains(reply, "event: message_stop") { + t.Fatalf("reply = %q, want an Anthropic stream", reply) + } + + res := rec.responses() + reqs := rec.all() + if len(reqs) != 1 || len(res) != 1 { + t.Fatalf("%d request and %d response captures, want 1 and 1", len(reqs), len(res)) + } + c := reqs[0] + if string(c.Original) != anthropicStreamRequest { + t.Errorf("Original = %q, want the agent's request", c.Original) + } + if string(c.Forwarded) != rewritten { + t.Errorf("Forwarded = %q, want the plugin's body", c.Forwarded) + } + sent := up.got() + if len(sent) != 1 || !bytes.Equal(c.Translated, sent[0].body) || !strings.Contains(string(c.Translated), "is 393 prime") { + t.Errorf("Translated = %q, want the chat body sent upstream", c.Translated) + } + for _, r := range res { + if string(r.Body) != reply { + t.Errorf("Response.Body = %q, want the Anthropic reply the agent got", r.Body) + } + if string(r.Upstream) != sse || r.UpstreamTruncated { + t.Errorf("Response.Upstream = %q (truncated %v), want the server's chat stream", r.Upstream, r.UpstreamTruncated) + } + } +} + +func TestTranslate_NotStreamedErrorStatus(t *testing.T) { + for _, c := range []struct { + name, body string + want int + }{ + {"error body at 200", replyFixture(t, "chunk_error.json"), http.StatusInternalServerError}, + {"not JSON", "not json", http.StatusBadGateway}, + // Valid JSON: only the limit makes it an error. + {"over the limit", replyFixture(t, "bob_shape1.json") + strings.Repeat(" ", maxBodySize), http.StatusBadGateway}, + } { + t.Run(c.name, func(t *testing.T) { + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, c.body) + }) + store := newStore(t) + proxy := newRedirectProxy(t, store, nil, &translator{target: up.URL, translate: true}) + resp := postAnthropic(t, proxy, anthropicRequest, nil) + body := replyText(t, resp) + + if resp.StatusCode != c.want || !strings.HasPrefix(body, `{"type":"error"`) { + t.Errorf("reply = %d %q, want %d with an Anthropic error body", resp.StatusCode, body, c.want) + } + if rows := waitRows(t, store, pipeline.SessionResponse, 1); rows[0].StatusCode != c.want { + t.Errorf("response row status = %d, want %d", rows[0].StatusCode, c.want) + } + }) + } +} + +func TestTranslate_EmptyReplyCapturesUpstream(t *testing.T) { + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + }) + proxy, rec := proxyWithCaptures(t, inferenceparser.NewInferenceParser(), &translator{target: up.URL, translate: true}) + _ = replyText(t, postAnthropic(t, proxy, anthropicStreamRequest, nil)) + + res := rec.responses() + if len(res) != 1 { + t.Fatalf("%d response captures, want 1", len(res)) + } + for _, r := range res { + if r.Upstream == nil { + t.Errorf("Upstream is nil, want an empty capture of the server's reply") + } + } +} diff --git a/core/model/capture.go b/core/model/capture.go index 87f0e9937..47d03007e 100644 --- a/core/model/capture.go +++ b/core/model/capture.go @@ -12,6 +12,10 @@ type RequestCapture struct { Original []byte // Forwarded is the body as Cortex sent it upstream, nil when no plugin rewrote it. Forwarded []byte + // Translated is the body as Cortex sent it upstream after it converted it to the + // server's protocol, nil when it did not convert it. Forwarded, or Original when + // Forwarded is nil, is the input of the conversion. + Translated []byte // Header is the request's headers. The producer removes credential headers before it // hands the capture over: the archive stores what it is given. Header http.Header @@ -37,6 +41,12 @@ type ResponseCapture struct { Decoded bool // Truncated is set when the body was longer than MaxResponseCapture. Truncated bool + // Upstream is the server's reply before Cortex converted it to the agent's protocol, up + // to MaxResponseCapture bytes, nil when Cortex did not convert the reply. Body is then the + // converted reply, as the agent got it. + Upstream []byte + // UpstreamTruncated is set when Upstream was longer than MaxResponseCapture. + UpstreamTruncated bool // Incomplete is set when the body did not arrive whole: the stream broke, or the client // went away first. Incomplete bool @@ -54,13 +64,16 @@ const MaxResponseCapture = 10 << 20 // text when it is valid UTF-8, which every JSON API body is, and base64 in the matching *Base64 // field otherwise, so any body survives the trip. type RawCapture struct { - Request string `json:"request,omitempty"` - RequestBase64 []byte `json:"requestBase64,omitempty"` - Forwarded *string `json:"forwarded,omitempty"` - ForwardedBase64 []byte `json:"forwardedBase64,omitempty"` - RequestHeaders http.Header `json:"requestHeaders,omitempty"` - Query string `json:"query,omitempty"` - Response *RawBody `json:"response,omitempty"` + Request string `json:"request,omitempty"` + RequestBase64 []byte `json:"requestBase64,omitempty"` + Forwarded *string `json:"forwarded,omitempty"` + ForwardedBase64 []byte `json:"forwardedBase64,omitempty"` + // Translated is the body after Cortex converted it to the server's protocol. + Translated *string `json:"translated,omitempty"` + TranslatedBase64 []byte `json:"translatedBase64,omitempty"` + RequestHeaders http.Header `json:"requestHeaders,omitempty"` + Query string `json:"query,omitempty"` + Response *RawBody `json:"response,omitempty"` } // RawBody is a captured response. Body is in its Content-Encoding unless Decoded is set, so a @@ -73,4 +86,8 @@ type RawBody struct { Decoded bool `json:"decoded,omitempty"` Truncated bool `json:"truncated,omitempty"` Incomplete bool `json:"incomplete,omitempty"` + // Upstream is the server's reply before Cortex converted it to the agent's protocol. + Upstream *string `json:"upstream,omitempty"` + UpstreamBase64 []byte `json:"upstreamBase64,omitempty"` + UpstreamTruncated bool `json:"upstreamTruncated,omitempty"` } diff --git a/core/parse/inference/dialect.go b/core/parse/inference/dialect.go index 4a8ebee70..69979b253 100644 --- a/core/parse/inference/dialect.go +++ b/core/parse/inference/dialect.go @@ -60,3 +60,25 @@ func dialectFor(path string) dialect { } return dialectNone } + +// chatCompletionsSuffix ends a chat-completions path. The OpenAI dialect also reads +// legacy completions, which have no translation protocol. +const chatCompletionsSuffix = "/chat/completions" + +// ProtocolFor names the translation protocol of path, by the same rule as dialectFor, +// so that the parser and the translator cannot disagree: "anthropic-messages", +// "chat-completions", "responses", or "" for a path with no protocol (legacy +// completions included). +func ProtocolFor(path string) string { + switch dialectFor(path) { + case dialectAnthropic: + return "anthropic-messages" + case dialectOpenAI: + if strings.HasSuffix(path, chatCompletionsSuffix) { + return "chat-completions" + } + case dialectResponses: + return "responses" + } + return "" +} diff --git a/core/parse/inference/dialect_test.go b/core/parse/inference/dialect_test.go index 5d4ea3eb9..ef812dbe0 100644 --- a/core/parse/inference/dialect_test.go +++ b/core/parse/inference/dialect_test.go @@ -47,3 +47,21 @@ func TestDialectFor(t *testing.T) { } } } + +// ProtocolFor names the protocol of a path with the rule that dialectFor uses +// (decision 14). Legacy completions and count_tokens have no protocol. +func TestProtocolFor(t *testing.T) { + for path, want := range map[string]string{ + "/v1/messages": "anthropic-messages", + "/anthropic/v1/messages": "anthropic-messages", + "/v1/chat/completions": "chat-completions", + "/v1/completions": "", + "/v1/responses": "responses", + "/v1/messages/count_tokens": "", + "/v1/models": "", + } { + if got := ProtocolFor(path); got != want { + t.Errorf("ProtocolFor(%q) = %q, want %q", path, got, want) + } + } +} diff --git a/core/pipeline/context.go b/core/pipeline/context.go index c37f87d02..35f8a2ddc 100644 --- a/core/pipeline/context.go +++ b/core/pipeline/context.go @@ -359,6 +359,11 @@ type Context struct { redirectHost string // redirectPath is the path SetRedirectPath set, "" to keep the client's. redirectPath string + // translateSource, translateTarget and translatePath are what Translate set; + // translateTarget is "" when no Translate was accepted. + translateSource, translateTarget, translatePath string + // upstreamHeaders is the clone SetUpstreamHeaders kept. See UpstreamHeaders. + upstreamHeaders http.Header // bodyMutated / responseBodyMutated flag that some plugin's // SetBody / SetResponseBody took effect on this context. Listeners read diff --git a/core/pipeline/destination.go b/core/pipeline/destination.go index d54bf448e..15fbc4b97 100644 --- a/core/pipeline/destination.go +++ b/core/pipeline/destination.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "log/slog" + "net/http" "net/url" "slices" "strings" @@ -139,17 +140,11 @@ func (c *Context) RedirectTarget() (scheme, host string, ok bool) { // went to, and the path the client named is kept in the modify/path_rewritten record. // The listener applies the copy RedirectPath reports. func (c *Context) SetRedirectPath(path string) error { - switch { - case c.inFinish: - return errors.New("pipeline: SetRedirectPath refused in OnFinish: the request has already been sent") - case c.currentPhase != InvocationPhaseRequest: - return fmt.Errorf("pipeline: SetRedirectPath refused in phase %q: it is accepted only from OnRequest", c.currentPhase) - case !c.currentMayRedirect: - return fmt.Errorf("pipeline: plugin %q called SetRedirectPath without declaring WritesDestination", c.currentPlugin) - case !c.redirected: - return errors.New("pipeline: SetRedirectPath refused: no Redirect has taken effect on this request") - case !strings.HasPrefix(path, "/") || strings.ContainsAny(path, "?#") || slices.Contains(strings.Split(path, "/"), ".."): - return errors.New("pipeline: SetRedirectPath path must be absolute, with no query, fragment or \"..\" segment") + if err := c.redirectedGate("SetRedirectPath"); err != nil { + return err + } + if err := checkRedirectPath("SetRedirectPath", path); err != nil { + return err } from := c.Path c.redirectPath, c.Path = path, path @@ -164,6 +159,85 @@ func (c *Context) RedirectPath() (string, bool) { return c.redirectPath, c.redirected && c.redirectPath != "" } +// Translate asks the listener to convert this redirected request from the source +// protocol to the target protocol and send it to path, with no query string. It does +// not change Path: the request row keeps the path the client named. +// +// Gates as SetRedirectPath: OnRequest only, a plugin that declares WritesDestination, +// a Redirect took effect, and path absolute with no query, fragment or ".." segment. +// Also source and target must be non-empty and different. It records nothing: only the +// listener knows what the conversion dropped, so the listener records +// modify/translated. A second call replaces the first. +func (c *Context) Translate(source, target, path string) error { + if err := c.redirectedGate("Translate"); err != nil { + return err + } + if err := checkRedirectPath("Translate", path); err != nil { + return err + } + if source == "" || target == "" || source == target { + return errors.New("pipeline: Translate needs a source and a target protocol that are not empty and not the same") + } + c.translateSource, c.translateTarget, c.translatePath = source, target, path + return nil +} + +// TranslateTarget is what the last accepted Translate set, with ok only when one was +// accepted on a request a Redirect moved. The listener applies it. +func (c *Context) TranslateTarget() (source, target, path string, ok bool) { + if !c.redirected || c.translateTarget == "" { + return "", "", "", false + } + return c.translateSource, c.translateTarget, c.translatePath, true +} + +// SetUpstreamHeaders gives headers that the listener sets on the upstream request +// after every plugin has run, each one replacing a header of the same name. A plugin +// cannot do this through Headers, because a later plugin can change those. Gates as +// SetRedirectPath, without the path. It keeps a clone of h, so a later change to h +// does not show. A second call replaces the first. +func (c *Context) SetUpstreamHeaders(h http.Header) error { + if err := c.redirectedGate("SetUpstreamHeaders"); err != nil { + return err + } + c.upstreamHeaders = h.Clone() + return nil +} + +// UpstreamHeaders is what the last accepted SetUpstreamHeaders set, and nil unless a +// Redirect took effect. +func (c *Context) UpstreamHeaders() http.Header { + if !c.redirected { + return nil + } + return c.upstreamHeaders +} + +// redirectedGate refuses a call that only a WritesDestination plugin may make from +// OnRequest, after a Redirect took effect. call names the method in the error. +func (c *Context) redirectedGate(call string) error { + switch { + case c.inFinish: + return fmt.Errorf("pipeline: %s refused in OnFinish: the request has already been sent", call) + case c.currentPhase != InvocationPhaseRequest: + return fmt.Errorf("pipeline: %s refused in phase %q: it is accepted only from OnRequest", call, c.currentPhase) + case !c.currentMayRedirect: + return fmt.Errorf("pipeline: plugin %q called %s without declaring WritesDestination", c.currentPlugin, call) + case !c.redirected: + return fmt.Errorf("pipeline: %s refused: no Redirect has taken effect on this request", call) + } + return nil +} + +// checkRedirectPath refuses a path that is not absolute, or that has a query, a +// fragment or a ".." segment. The error quotes no part of path. +func checkRedirectPath(call, path string) error { + if !strings.HasPrefix(path, "/") || strings.ContainsAny(path, "?#") || slices.Contains(strings.Split(path, "/"), "..") { + return fmt.Errorf("pipeline: %s path must be absolute, with no query, fragment or \"..\" segment", call) + } + return nil +} + // RequestedHost is the host the client named when a redirect sent the request // somewhere else, and "" otherwise — including when the redirects ended back at // that host. It is what a session event records beside Host. It compares against diff --git a/core/pipeline/translate_test.go b/core/pipeline/translate_test.go new file mode 100644 index 000000000..0125d4366 --- /dev/null +++ b/core/pipeline/translate_test.go @@ -0,0 +1,201 @@ +package pipeline + +import ( + "context" + "net/http" + "testing" +) + +const ( + srcProto = "anthropic-messages" + tgtProto = "chat-completions" +) + +// redirectThen gives a plugin that redirects to b.example and then runs do. +func redirectThen(do func(*Context) error, err *error) func(*Context) { + return func(c *Context) { + if *err = c.Redirect(mustURL("https://b.example")); *err == nil { + *err = do(c) + } + } +} + +func TestTranslate_AcceptedAfterARedirect(t *testing.T) { + var err error + _, pctx := runRequest(t, redirector(true, func(c *Context) { + c.Path = "/v1/messages" + redirectThen(func(c *Context) error { return c.Translate(srcProto, tgtProto, "/v1/chat/completions") }, &err)(c) + }), true) + if err != nil { + t.Fatalf("Translate: %v", err) + } + src, tgt, path, ok := pctx.TranslateTarget() + if !ok || src != srcProto || tgt != tgtProto || path != "/v1/chat/completions" { + t.Errorf("TranslateTarget() = %q, %q, %q, %v", src, tgt, path, ok) + } + if pctx.Path != "/v1/messages" { + t.Errorf("Path = %q, want the client's path: Translate does not change Path", pctx.Path) + } + for _, inv := range outbound(pctx) { + if inv.Reason != "redirected" { + t.Errorf("invocation %+v: Translate must record nothing", inv) + } + } +} + +func TestTranslate_SecondCallReplacesTheFirst(t *testing.T) { + var err error + _, pctx := runRequest(t, redirector(true, redirectThen(func(c *Context) error { + if err := c.Translate(srcProto, tgtProto, "/v1/a"); err != nil { + return err + } + return c.Translate(srcProto, "responses", "/v1/responses") + }, &err)), true) + if err != nil { + t.Fatalf("Translate: %v", err) + } + if _, tgt, path, ok := pctx.TranslateTarget(); !ok || tgt != "responses" || path != "/v1/responses" { + t.Errorf("TranslateTarget() = %q, %q, %v; want the second call", tgt, path, ok) + } +} + +func TestTranslate_RefusedWithoutARedirect(t *testing.T) { + var err error + _, pctx := runRequest(t, redirector(true, func(c *Context) { + err = c.Translate(srcProto, tgtProto, "/v1/chat/completions") + }), true) + if err == nil { + t.Fatal("Translate succeeded with no Redirect in effect") + } + if _, _, _, ok := pctx.TranslateTarget(); ok { + t.Error("TranslateTarget() ok for a request that was not redirected") + } +} + +func TestTranslate_RefusedWithoutTheCapability(t *testing.T) { + var err error + runRequest(t, redirector(false, func(c *Context) { + err = c.Translate(srcProto, tgtProto, "/v1/chat/completions") + }), true) + if err == nil { + t.Fatal("Translate succeeded for a plugin that does not declare WritesDestination") + } +} + +func TestTranslate_RefusedInTheResponsePass(t *testing.T) { + var err, rerr error + plug := redirector(true, func(c *Context) { rerr = c.Redirect(mustURL("https://b.example")) }) + plug.onResponse = func(c *Context) { err = c.Translate(srcProto, tgtProto, "/v1/chat/completions") } + p, pctx := runRequest(t, plug, true) + if rerr != nil { + t.Fatalf("Redirect: %v", rerr) + } + p.RunResponse(context.Background(), pctx) + if err == nil { + t.Fatal("Translate succeeded from OnResponse") + } + if _, _, _, ok := pctx.TranslateTarget(); ok { + t.Error("TranslateTarget() ok after a refusal in OnResponse") + } +} + +func TestTranslate_NothingUnderObserve(t *testing.T) { + var err error + _, pctx := runRequest(t, redirector(true, func(c *Context) { + _ = c.Redirect(mustURL("https://b.example")) + err = c.Translate(srcProto, tgtProto, "/v1/chat/completions") + }), true, WithPolicies(ErrorPolicyObserve)) + if err == nil { + t.Error("Translate succeeded under observe, where the Redirect moved nothing") + } + if _, _, _, ok := pctx.TranslateTarget(); ok { + t.Error("TranslateTarget() ok under observe") + } +} + +func TestTranslate_RefusesBadArguments(t *testing.T) { + for _, tc := range []struct{ src, tgt, path string }{ + {srcProto, tgtProto, "v1/x"}, + {srcProto, tgtProto, "/a?b"}, + {srcProto, tgtProto, "/a#b"}, + {srcProto, tgtProto, "/a/../b"}, + {srcProto, tgtProto, ""}, + {srcProto, srcProto, "/v1/messages"}, + {"", tgtProto, "/v1/chat/completions"}, + {srcProto, "", "/v1/chat/completions"}, + } { + var err error + _, pctx := runRequest(t, redirector(true, redirectThen(func(c *Context) error { + return c.Translate(tc.src, tc.tgt, tc.path) + }, &err)), true) + if err == nil { + t.Errorf("Translate(%q, %q, %q) succeeded", tc.src, tc.tgt, tc.path) + } + if _, _, _, ok := pctx.TranslateTarget(); ok { + t.Errorf("Translate(%q, %q, %q) left a target to apply", tc.src, tc.tgt, tc.path) + } + } +} + +func TestSetUpstreamHeaders_KeepsAClone(t *testing.T) { + var err error + h := http.Header{"User-Agent": {"bob-shell"}} + _, pctx := runRequest(t, redirector(true, redirectThen(func(c *Context) error { + return c.SetUpstreamHeaders(h) + }, &err)), true) + if err != nil { + t.Fatalf("SetUpstreamHeaders: %v", err) + } + h.Set("User-Agent", "changed") + h.Set("X-Later", "1") + got := pctx.UpstreamHeaders() + if got.Get("User-Agent") != "bob-shell" || got.Get("X-Later") != "" { + t.Errorf("UpstreamHeaders() = %v; a later change to the caller's map must not show", got) + } + if len(outbound(pctx)) != 1 { + t.Errorf("invocations = %+v, want only modify/redirected", outbound(pctx)) + } +} + +func TestSetUpstreamHeaders_Refusals(t *testing.T) { + h := http.Header{"User-Agent": {"bob-shell"}} + cases := map[string]func(t *testing.T) (error, *Context){ + "no redirect": func(t *testing.T) (error, *Context) { + var err error + _, pctx := runRequest(t, redirector(true, func(c *Context) { err = c.SetUpstreamHeaders(h) }), true) + return err, pctx + }, + "no capability": func(t *testing.T) (error, *Context) { + var err error + _, pctx := runRequest(t, redirector(false, func(c *Context) { err = c.SetUpstreamHeaders(h) }), true) + return err, pctx + }, + "observe": func(t *testing.T) (error, *Context) { + var err error + _, pctx := runRequest(t, redirector(true, func(c *Context) { + _ = c.Redirect(mustURL("https://b.example")) + err = c.SetUpstreamHeaders(h) + }), true, WithPolicies(ErrorPolicyObserve)) + return err, pctx + }, + "response pass": func(t *testing.T) (error, *Context) { + var err error + plug := redirector(true, func(c *Context) { _ = c.Redirect(mustURL("https://b.example")) }) + plug.onResponse = func(c *Context) { err = c.SetUpstreamHeaders(h) } + p, pctx := runRequest(t, plug, true) + p.RunResponse(context.Background(), pctx) + return err, pctx + }, + } + for name, run := range cases { + t.Run(name, func(t *testing.T) { + err, pctx := run(t) + if err == nil { + t.Error("SetUpstreamHeaders succeeded") + } + if pctx.UpstreamHeaders() != nil { + t.Errorf("UpstreamHeaders() = %v, want nil", pctx.UpstreamHeaders()) + } + }) + } +} diff --git a/core/plugins/inferencerouter/api.go b/core/plugins/inferencerouter/api.go new file mode 100644 index 000000000..dff4fd9ad --- /dev/null +++ b/core/plugins/inferencerouter/api.go @@ -0,0 +1,122 @@ +package inferencerouter + +import ( + "net/http" + "slices" + "strings" + + "github.com/rossoctl/cortex/core/parse/inference" + "github.com/rossoctl/cortex/core/pipeline" + "github.com/rossoctl/cortex/core/translate" +) + +// planKind is what the server's api map does to a request (rules 4 to 7, C1). +type planKind int + +const ( + planAsIs planKind = iota // the agent's protocol, to the agent's path + planPath // the agent's protocol, to the api map's path + planTranslate // converted to target, sent to path + planRefuse // no protocol of the map can be translated to (rule 4a) + planCountTokens // answered here with a 404 (rule 6) +) + +type apiPlan struct { + kind planKind + source, target, path string +} + +// countTokensSuffix ends Anthropic's token-count path. +const countTokensSuffix = "/v1/messages/count_tokens" + +// translationServerDoc is the fallback doc that the rule 4a error names. +const translationServerDoc = "https://github.com/rossoctl/cortex/blob/main/docs/translation-server.md" + +// plan says how a request to path goes to r, from r's api map. +func (r route) plan(path string) apiPlan { + if len(r.api) == 0 { + return apiPlan{kind: planAsIs} + } + if _, ok := r.api[translate.Anthropic]; !ok && strings.HasSuffix(path, countTokensSuffix) { + return apiPlan{kind: planCountTokens} + } + src := inference.ProtocolFor(path) + if p, ok := r.api[src]; ok { + return apiPlan{kind: planPath, path: p} + } + if translate.Decodes(src) { + tgt, ok := translate.Target(apiKeys(r)) + if !ok { + return apiPlan{kind: planRefuse, source: src} + } + return apiPlan{kind: planTranslate, source: src, target: tgt, path: r.api[tgt]} + } + return apiPlan{kind: planAsIs} +} + +// apiKeys are the protocols of r's api map, sorted. +func apiKeys(r route) []string { + keys := make([]string, 0, len(r.api)) + for k := range r.api { + keys = append(keys, k) + } + slices.Sort(keys) + return keys +} + +// cannotTranslate is the rule 4a message (decision 13). +func cannotTranslate(source string, r route) string { + return "Cortex cannot translate " + source + " to " + strings.Join(apiKeys(r), " or ") + + " yet. Use a translation server for this pair (see " + translationServerDoc + ")." +} + +// refusePlan records and gives the local answer for planRefuse and planCountTokens. +func refusePlan(pctx *pipeline.Context, r route, pl apiPlan, details map[string]string) pipeline.Action { + if pl.kind == planCountTokens { + pctx.Record(pipeline.Invocation{Action: pipeline.ActionDeny, Reason: "count_tokens_local", Details: details}) + const msg = "this inference server does not count tokens" + return pipeline.Action{Type: pipeline.Reject, Violation: &pipeline.Violation{ + Code: "inference.count_tokens_unavailable", Reason: msg, Status: http.StatusNotFound, + Body: translate.ErrorBody(translate.Anthropic, http.StatusNotFound, "not_found_error", msg), + BodyType: "application/json", + }} + } + details["from"] = pl.source + pctx.Record(pipeline.Invocation{Action: pipeline.ActionDeny, Reason: "cannot_translate", Details: details}) + msg := cannotTranslate(pl.source, r) + return pipeline.Action{Type: pipeline.Reject, Violation: &pipeline.Violation{ + Code: "inference.translation_unavailable", Reason: msg, Status: http.StatusBadGateway, + Body: translate.ErrorBody(pl.source, http.StatusBadGateway, "api_error", msg), + BodyType: "application/json", + }} +} + +// authHeaderNames are the headers an agent can carry its key in. +var authHeaderNames = []string{"Authorization", "X-Api-Key", "Api-Key"} + +// setAuth puts r's key on the request (rule C2). translatedTo is the target protocol, +// "" when the request is not translated. With no auth_header and no translation, it +// is setKey: the header the agent used. +func setAuth(pctx *pipeline.Context, r route, translatedTo string) { + name := r.authHeader + switch { + case name == "" && translatedTo == translate.Chat: + name = "authorization" + case name == "" && translatedTo == translate.Anthropic: + name = "x-api-key" + case name == "": + setKey(pctx, r.key) + return + } + if pctx.Headers == nil { + pctx.Headers = http.Header{} + } + for _, h := range authHeaderNames { + pctx.Headers.Del(h) + } + if strings.EqualFold(name, "authorization") { + pctx.Headers.Set("Authorization", "Bearer "+r.key) + return + } + pctx.Headers.Set(name, r.key) +} diff --git a/core/plugins/inferencerouter/api_test.go b/core/plugins/inferencerouter/api_test.go new file mode 100644 index 000000000..79eb0e4e1 --- /dev/null +++ b/core/plugins/inferencerouter/api_test.go @@ -0,0 +1,215 @@ +package inferencerouter + +import ( + "encoding/json" + "net/http" + "slices" + "strings" + "testing" + + "github.com/rossoctl/cortex/core/pipeline" + "github.com/rossoctl/cortex/core/plugins/inferencerouter/routerconfig" + "github.com/rossoctl/cortex/core/translate" +) + +// routerconfig keeps its own list, because it must not import core/translate +// (decision 15). This pins the two lists equal. +func TestAPIProtocolsMatchRegistry(t *testing.T) { + if !slices.Equal(routerconfig.APIProtocols, translate.Names()) { + t.Errorf("routerconfig.APIProtocols = %v, translate.Names() = %v; want equal", routerconfig.APIProtocols, translate.Names()) + } +} + +func TestPlan(t *testing.T) { + for _, tc := range []struct { + name string + api map[string]string + path string + want apiPlan + }{ + {"empty map", nil, "/v1/messages", apiPlan{kind: planAsIs}}, + {"chat only", map[string]string{"chat-completions": "/v1/chat/completions"}, "/v1/messages", + apiPlan{kind: planTranslate, source: "anthropic-messages", target: "chat-completions", path: "/v1/chat/completions"}}, + {"anthropic at its own path", map[string]string{"anthropic-messages": "/anthropic/v1/messages"}, "/v1/messages", + apiPlan{kind: planPath, path: "/anthropic/v1/messages"}}, + {"responses only", map[string]string{"responses": "/v1/responses"}, "/v1/messages", + apiPlan{kind: planRefuse, source: "anthropic-messages"}}, + // Rule 7: a chat request cannot be decoded, so it is not translated. + {"no decoder", map[string]string{"anthropic-messages": "/v1/messages", "responses": "/v1/responses"}, "/v1/chat/completions", + apiPlan{kind: planAsIs}}, + {"count_tokens to a chat server", map[string]string{"chat-completions": "/v1/chat/completions"}, "/v1/messages/count_tokens", + apiPlan{kind: planCountTokens}}, + {"count_tokens to an anthropic server", map[string]string{"anthropic-messages": "/v1/messages"}, "/v1/messages/count_tokens", + apiPlan{kind: planAsIs}}, + {"DeepSeek chat", map[string]string{"chat-completions": "/chat/completions", "anthropic-messages": "/anthropic/v1/messages"}, "/v1/chat/completions", + apiPlan{kind: planPath, path: "/chat/completions"}}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := (route{api: tc.api}).plan(tc.path); got != tc.want { + t.Errorf("plan(%q) = %+v, want %+v", tc.path, got, tc.want) + } + }) + } +} + +// apiConfig is one server, "s", with the extra server fields given, and claude-code +// routed to it. +func apiConfig(fields string) string { + return `{ + "servers": {"s": {"url": "https://s.example.com", "key": "s-key", "main": "opus", "helper": "haiku"` + fields + `}}, + "agents": {"claude-code": "s"} + }` +} + +const chatAPI = `, "api": {"chat-completions": "/v1/chat/completions"}` + +// Rule C2: the key goes in one header, and the agent's other auth headers are gone. +func TestRouter_SetAuth(t *testing.T) { + for _, tc := range []struct { + name string + fields string + sent http.Header + want http.Header // every auth header the request must carry + }{ + {"not translated, empty auth_header: setKey", "", http.Header{"X-Api-Key": {"client-key"}}, + http.Header{"X-Api-Key": {"s-key"}}}, + {"translated to chat", chatAPI, http.Header{"X-Api-Key": {"client-key"}, "Api-Key": {"client-key"}}, + http.Header{"Authorization": {"Bearer s-key"}}}, + {"path entry, not translated: setKey", `, "api": {"anthropic-messages": "/x/v1/messages"}`, http.Header{"Authorization": {"Bearer client-key"}}, + http.Header{"Authorization": {"Bearer s-key"}}}, + {"auth_header x-api-key", chatAPI + `, "auth_header": "x-api-key"`, http.Header{"Authorization": {"Bearer client-key"}}, + http.Header{"X-Api-Key": {"s-key"}}}, + {"auth_header api-key", `, "auth_header": "api-key"`, http.Header{"X-Api-Key": {"client-key"}, "Authorization": {"Bearer client-key"}}, + http.Header{"Api-Key": {"s-key"}}}, + {"auth_header authorization", `, "auth_header": "authorization"`, http.Header{"X-Api-Key": {"client-key"}}, + http.Header{"Authorization": {"Bearer s-key"}}}, + } { + t.Run(tc.name, func(t *testing.T) { + p := build(t, apiConfig(tc.fields)) + pctx := request(newStore(t), "api.anthropic.com", claudeUA, "s1") + pctx.Headers = tc.sent + pctx.Headers.Set("User-Agent", claudeUA) + if a := run(t, p, pctx); a.Type != pipeline.Continue { + t.Fatalf("action = %+v, want Continue", a) + } + for _, h := range []string{"Authorization", "X-Api-Key", "Api-Key"} { + if got, want := pctx.Headers.Values(h), tc.want.Values(h); !slices.Equal(got, want) { + t.Errorf("%s = %q, want %q", h, got, want) + } + } + }) + } +} + +// A request translated to Anthropic Messages gets the key in X-Api-Key only. No +// protocol of this issue can be translated to it, so setAuth is called directly. +func TestSetAuth_TranslatedToAnthropic(t *testing.T) { + pctx := &pipeline.Context{Headers: http.Header{"Authorization": {"Bearer client-key"}, "Api-Key": {"client-key"}}} + setAuth(pctx, route{key: "s-key"}, "anthropic-messages") + if got := pctx.Headers; len(got) != 1 || got.Get("X-Api-Key") != "s-key" { + t.Errorf("headers = %v, want only X-Api-Key: s-key", got) + } +} + +// Rule 5: the router does not change Path and does not call SetRedirectPath. +func TestRouter_ATranslatedRequestKeepsItsPath(t *testing.T) { + p := build(t, apiConfig(chatAPI)) + pctx := request(newStore(t), "api.anthropic.com", claudeUA, "s1") + pctx.Path = "/zen/v1/messages" + run(t, p, pctx) + + if pctx.Path != "/zen/v1/messages" { + t.Errorf("Path = %q, want the client's path", pctx.Path) + } + if path, ok := pctx.RedirectPath(); ok { + t.Errorf("RedirectPath = %q, want none for a translated request", path) + } + src, tgt, path, ok := pctx.TranslateTarget() + if !ok || src != "anthropic-messages" || tgt != "chat-completions" || path != "/v1/chat/completions" { + t.Errorf("TranslateTarget = %q %q %q %v, want anthropic-messages to chat-completions at /v1/chat/completions", src, tgt, path, ok) + } + assertRecord(t, pctx, pipeline.ActionModify, "routed", map[string]string{"server": "s"}) +} + +func TestRouter_APathEntrySetsTheRedirectPath(t *testing.T) { + p := build(t, apiConfig(`, "api": {"anthropic-messages": "/anthropic/v1/messages"}`)) + pctx := request(newStore(t), "api.anthropic.com", claudeUA, "s1") + run(t, p, pctx) + + if path, ok := pctx.RedirectPath(); !ok || path != "/anthropic/v1/messages" { + t.Errorf("RedirectPath = %q %v, want /anthropic/v1/messages", path, ok) + } + if _, _, _, ok := pctx.TranslateTarget(); ok { + t.Error("TranslateTarget is set, want no translation") + } +} + +func TestRouter_HeadersReachUpstreamHeaders(t *testing.T) { + p := build(t, apiConfig(`, "headers": {"user-agent": "bob-shell"}`)) + pctx := request(newStore(t), "api.anthropic.com", claudeUA, "s1") + run(t, p, pctx) + + if got := pctx.UpstreamHeaders().Values("User-Agent"); !slices.Equal(got, []string{"bob-shell"}) { + t.Errorf("UpstreamHeaders User-Agent = %q, want bob-shell", got) + } +} + +func TestRouter_NoHeadersSetsNoUpstreamHeaders(t *testing.T) { + p := build(t, apiConfig("")) + pctx := request(newStore(t), "api.anthropic.com", claudeUA, "s1") + run(t, p, pctx) + + if h := pctx.UpstreamHeaders(); h != nil { + t.Errorf("UpstreamHeaders = %v, want nil", h) + } +} + +// Rule 4a, with the decision 13 message. +func TestRouter_RefusesAPairItCannotTranslate(t *testing.T) { + p := build(t, apiConfig(`, "api": {"responses": "/v1/responses"}`)) + pctx := request(newStore(t), "api.anthropic.com", claudeUA, "s1") + a := run(t, p, pctx) + + const msg = "Cortex cannot translate anthropic-messages to responses yet. Use a translation server for this pair " + + "(see https://github.com/rossoctl/cortex/blob/main/docs/translation-server.md)." + if a.Type != pipeline.Reject || a.Violation == nil { + t.Fatalf("action = %+v, want Reject", a) + } + v := a.Violation + if v.Code != "inference.translation_unavailable" || v.Status != http.StatusBadGateway || v.Reason != msg || v.BodyType != "application/json" { + t.Errorf("violation = %+v, want a 502 inference.translation_unavailable with the decision 13 message", v) + } + var body struct { + Type string `json:"type"` + Error struct{ Type, Message string } + } + if err := json.Unmarshal(v.Body, &body); err != nil || body.Type != "error" || body.Error.Type != "api_error" || body.Error.Message != msg { + t.Errorf("body = %s (%v), want an Anthropic api_error with the message", v.Body, err) + } + assertRecord(t, pctx, pipeline.ActionDeny, "cannot_translate", map[string]string{"server": "s", "from": "anthropic-messages"}) +} + +func TestRouter_TheRefusalNamesEveryAPIKey(t *testing.T) { + r := route{api: map[string]string{"responses": "/v1/responses", "anthropic-messages": "/v1/messages"}} + got := cannotTranslate("chat-completions", r) + if !strings.HasPrefix(got, "Cortex cannot translate chat-completions to anthropic-messages or responses yet.") { + t.Errorf("message = %q, want the sorted keys joined with or", got) + } +} + +// Rule 6: count_tokens to a server with no anthropic-messages entry is answered here. +func TestRouter_AnswersCountTokensLocally(t *testing.T) { + p := build(t, apiConfig(chatAPI)) + pctx := request(newStore(t), "s.example.com", claudeUA, "s1") + pctx.Path, pctx.Extensions.Inference = "/v1/messages/count_tokens", nil + a := run(t, p, pctx) + + if a.Type != pipeline.Reject || a.Violation == nil || a.Violation.Status != http.StatusNotFound { + t.Fatalf("action = %+v, want a 404 Reject", a) + } + if !strings.Contains(string(a.Violation.Body), `"not_found_error"`) || + !strings.Contains(string(a.Violation.Body), "this inference server does not count tokens") { + t.Errorf("body = %s, want an Anthropic not_found_error", a.Violation.Body) + } + assertRecord(t, pctx, pipeline.ActionDeny, "count_tokens_local", map[string]string{"server": "s"}) +} diff --git a/core/plugins/inferencerouter/plugin.go b/core/plugins/inferencerouter/plugin.go index 6e0c1526e..9e9247860 100644 --- a/core/plugins/inferencerouter/plugin.go +++ b/core/plugins/inferencerouter/plugin.go @@ -187,6 +187,11 @@ type route struct { // main and helper are the words naming the server's substitute models, resolved // against its model list when one is needed (see substitute). main, helper string + // api, authHeader and headers are the server's api, auth_header and headers + // fields (rules C1 to C3). headers is nil when the server has none. + api map[string]string + authHeader string + headers http.Header } // target is the server's own scheme and host, where every request routed to it is @@ -280,7 +285,15 @@ func (p *Router) Configure(raw json.RawMessage) error { "crosses the network decrypted, its key and prompt included; use https unless the network is trusted", "server", name, "url", ep.URL()) } - servers[name] = route{endpoint: ep, key: s.Key, main: s.Main, helper: s.Helper} + var headers http.Header + for k, v := range s.Headers { + if headers == nil { + headers = http.Header{} + } + headers.Set(k, v) + } + servers[name] = route{endpoint: ep, key: s.Key, main: s.Main, helper: s.Helper, + api: s.API, authHeader: s.AuthHeader, headers: headers} byHost[ep.Key()] = name } // p.agents is keyed by CANONICAL agent name, folded from whatever spelling the @@ -366,11 +379,21 @@ func (p *Router) OnRequest(_ context.Context, pctx *pipeline.Context) pipeline.A // sent it. Redirected is the signal that it moved, and it is this router's // redirect: a pipeline admits one WritesDestination plugin. redirected := redirectErr == nil && pctx.Redirected() + var pl apiPlan var pathErr error - if redirected && captured != "" { - // The table's paths are absolute with no query, so this is refused only by a - // drift between the table and SetRedirectPath's rule. - pathErr = pctx.SetRedirectPath(captured) + if redirected { + // The paths of the table and of the api map are absolute with no query, so + // these are refused only by a drift between the checks and the pipeline's. + switch pl = srv.plan(pctx.Path); pl.kind { + case planPath: + pathErr = pctx.SetRedirectPath(pl.path) + case planTranslate: + pathErr = pctx.Translate(pl.source, pl.target, pl.path) + case planAsIs: + if captured != "" { + pathErr = pctx.SetRedirectPath(captured) + } + } } err := cmp.Or(redirectErr, pathErr) outcome := redirectMoved @@ -397,6 +420,13 @@ func (p *Router) OnRequest(_ context.Context, pctx *pipeline.Context) pipeline.A pctx.Record(pipeline.Invocation{Action: pipeline.ActionObserve, Reason: "would_route", Details: details()}) return cont } + if pl.kind == planRefuse || pl.kind == planCountTokens { + // The session is on this server; only this request is answered here. + if settle { + pin.settle(name) + } + return refusePlan(pctx, srv, pl, details()) + } // The model is chosen only once the request has moved, so an observed router // changes nothing. p.chooseModel(pctx, name) @@ -409,7 +439,17 @@ func (p *Router) OnRequest(_ context.Context, pctx *pipeline.Context) pipeline.A if addressed != name { dropProviderCredentials(pctx.Headers) } - setKey(pctx, srv.key) + translatedTo := "" + if pl.kind == planTranslate { + translatedTo = pl.target + } + setAuth(pctx, srv, translatedTo) + if len(srv.headers) > 0 { + if err := pctx.SetUpstreamHeaders(srv.headers); err != nil { + pctx.Record(pipeline.Invocation{Action: pipeline.ActionDeny, Reason: "redirect_failed", Details: details()}) + return pipeline.Deny(codeUnavailable, fmt.Sprintf("inference-router could not send this request to %q: %v", name, err)) + } + } d := details() if captured != "" { d["captured"] = captured diff --git a/core/plugins/inferencerouter/routerconfig/config.go b/core/plugins/inferencerouter/routerconfig/config.go index 63ddb31ae..3749e003a 100644 --- a/core/plugins/inferencerouter/routerconfig/config.go +++ b/core/plugins/inferencerouter/routerconfig/config.go @@ -42,10 +42,94 @@ type Config struct { // Server is one inference server. type Server struct { - URL string `json:"url" required:"true" description:"scheme://host[:port] of the server; no path."` - Key string `json:"key" required:"true" description:"API key sent to this server in place of the client's."` - Main string `json:"main" required:"true" description:"A word naming this server's model for an agent's main work, a request with tools, sent when the server refuses the model the agent asked for. The newest listed model whose name contains it."` - Helper string `json:"helper" required:"true" description:"A word naming this server's model for an agent's helper calls, a request without tools. Resolved as main is."` + URL string `json:"url" required:"true" description:"scheme://host[:port] of the server; no path."` + Key string `json:"key" required:"true" description:"API key sent to this server in place of the client's."` + Main string `json:"main" required:"true" description:"A word naming this server's model for an agent's main work, a request with tools, sent when the server refuses the model the agent asked for. The newest listed model whose name contains it."` + Helper string `json:"helper" required:"true" description:"A word naming this server's model for an agent's helper calls, a request without tools. Resolved as main is."` + API map[string]string `json:"api" description:"Protocols the server serves, by protocol name, to the path it serves each at. Empty: requests go in the agent's protocol, to the agent's path."` + AuthHeader string `json:"auth_header" description:"Header that carries key. authorization sends Bearer ; any other name sends the raw key. Empty: the agent's header, or the target protocol's usual one for a translated request."` + Headers map[string]string `json:"headers" description:"Fixed headers set on every request to this server after the plugins run. Values may use ${ENV_VAR}."` +} + +// APIProtocols are the protocol names an api key can have. It is translate.Names(), +// copied rather than imported to keep this package free of core/translate; a test in +// the plugin package holds the two equal. +var APIProtocols = []string{"anthropic-messages", "chat-completions", "responses"} + +// CheckAPI reports whether api maps known protocols to paths a request can be sent +// to. Keys are checked in sorted order, so the same map always gives the same first +// error. Each error starts with "api". The path rule is pipeline.Context.SetRedirectPath's. +func CheckAPI(api map[string]string) error { + for _, k := range slices.Sorted(maps.Keys(api)) { + if !slices.Contains(APIProtocols, k) { + return fmt.Errorf("api: unknown protocol %q; valid protocols: %s", k, strings.Join(APIProtocols, ", ")) + } + p := api[k] + if !strings.HasPrefix(p, "/") || strings.ContainsAny(p, "?#") || slices.Contains(strings.Split(p, "/"), "..") { + return fmt.Errorf("api.%s: path %q must be absolute, with no query, fragment or \"..\" segment", k, p) + } + } + return nil +} + +// framingHeaders are the headers neither auth_header nor headers can name: the +// transport sets them, and a fixed value would break the request's framing. +var framingHeaders = []string{ + "host", "content-length", + "connection", "keep-alive", "proxy-connection", "transfer-encoding", "upgrade", "te", "trailer", + "proxy-authenticate", "proxy-authorization", +} + +// authHeaders are the headers the router puts the key in. headers cannot name them. +var authHeaders = []string{"authorization", "x-api-key", "api-key"} + +// CheckAuthHeader reports whether name can carry a server's key. "" is valid: it +// means the agent's header, or the target protocol's usual one. +func CheckAuthHeader(name string) error { + if name == "" { + return nil + } + if !validHeaderName(name) { + return fmt.Errorf("%q is not a valid header name", name) + } + if slices.Contains(framingHeaders, strings.ToLower(name)) { + return fmt.Errorf("%q cannot carry the key: it is a hop-by-hop or framing header", name) + } + return nil +} + +// CheckHeaders reports whether h can be set on every request to a server whose +// auth_header is authHeader. Names are checked in sorted order for a stable first +// error. The errors never repeat a value. +func CheckHeaders(h map[string]string, authHeader string) error { + for _, name := range slices.Sorted(maps.Keys(h)) { + lower := strings.ToLower(name) + switch { + case !validHeaderName(name): + return fmt.Errorf("headers: %q is not a valid header name", name) + case slices.Contains(authHeaders, lower) || (authHeader != "" && lower == strings.ToLower(authHeader)): + return fmt.Errorf("headers: %q cannot be set here: Cortex sets it", name) + case slices.Contains(framingHeaders, lower): + return fmt.Errorf("headers: %q cannot be set here: it is a hop-by-hop or framing header", name) + case strings.ContainsAny(h[name], "\r\n\x00"): + return fmt.Errorf("headers: the value of %q has a CR, LF or NUL", name) + } + } + return nil +} + +// validHeaderName reports whether name is an RFC 9110 token. +func validHeaderName(name string) bool { + if name == "" { + return false + } + for i := 0; i < len(name); i++ { + c := name[i] + if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || strings.IndexByte("!#$%&'*+-.^_`|~", c) >= 0) { + return false + } + } + return true } // CheckWord reports whether word can name a server's main or helper model. A word is @@ -151,6 +235,15 @@ func (c Config) Validate() error { if err := CheckWord(s.Helper); err != nil { return fmt.Errorf("servers.%s.helper: %w", name, err) } + if err := CheckAPI(s.API); err != nil { + return fmt.Errorf("servers.%s.%w", name, err) + } + if err := CheckAuthHeader(s.AuthHeader); err != nil { + return fmt.Errorf("servers.%s.auth_header: %w", name, err) + } + if err := CheckHeaders(s.Headers, s.AuthHeader); err != nil { + return fmt.Errorf("servers.%s.%w", name, err) + } } for _, agent := range slices.Sorted(maps.Keys(c.Agents)) { if err := CheckAgent(agent); err != nil { diff --git a/core/plugins/inferencerouter/routerconfig/config_test.go b/core/plugins/inferencerouter/routerconfig/config_test.go index 4304956ae..2bc40f9ce 100644 --- a/core/plugins/inferencerouter/routerconfig/config_test.go +++ b/core/plugins/inferencerouter/routerconfig/config_test.go @@ -373,3 +373,93 @@ func TestDecode_Sessions(t *testing.T) { }) } } +func TestCheckAPI(t *testing.T) { + for _, api := range []map[string]string{ + nil, + {"chat-completions": "/v1/chat/completions"}, + {"chat-completions": "/chat/completions", "anthropic-messages": "/anthropic/v1/messages"}, + {"responses": "/v1/responses"}, + } { + if err := CheckAPI(api); err != nil { + t.Errorf("CheckAPI(%v) = %v, want nil", api, err) + } + } + for api, want := range map[string]string{ + `{"chat": "/v1/chat/completions"}`: `unknown protocol "chat"; valid protocols: anthropic-messages, chat-completions, responses`, + `{"chat-completions": "v1/chat/completions"}`: `api.chat-completions: path "v1/chat/completions" must be absolute, with no query, fragment or ".." segment`, + `{"chat-completions": "/v1/chat/completions?x=1"}`: `api.chat-completions: path "/v1/chat/completions?x=1" must be absolute`, + `{"chat-completions": "/v1/../chat/completions"}`: `".." segment`, + `{"chat-completions": "/v1/chat#x"}`: `must be absolute`, + `{"anthropic-messages": "x", "chat-completions": "y"}`: `api.anthropic-messages:`, // sorted: first key first + } { + var m map[string]string + if err := json.Unmarshal([]byte(api), &m); err != nil { + t.Fatal(err) + } + err := CheckAPI(m) + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("CheckAPI(%s) = %v, want %q", api, err, want) + } + } +} + +func TestCheckAuthHeader(t *testing.T) { + for _, name := range []string{"", "authorization", "x-api-key", "api-key", "Authorization"} { + if err := CheckAuthHeader(name); err != nil { + t.Errorf("CheckAuthHeader(%q) = %v, want nil", name, err) + } + } + for _, name := range []string{"Host", "Connection", "content-length", "Transfer-Encoding", "TE", "Proxy-Authorization", "bad header", "x:y"} { + if err := CheckAuthHeader(name); err == nil { + t.Errorf("CheckAuthHeader(%q) = nil, want an error", name) + } + } +} + +func TestCheckHeaders(t *testing.T) { + if err := CheckHeaders(map[string]string{"User-Agent": "bob-shell", "X-Team": "a b"}, ""); err != nil { + t.Errorf("CheckHeaders = %v, want nil", err) + } + for name, want := range map[string]string{ + "Authorization": `headers: "Authorization" cannot be set here: Cortex sets it`, + "X-Api-Key": `headers: "X-Api-Key" cannot be set here: Cortex sets it`, + "api-key": `Cortex sets it`, + "Content-Length": `headers: "Content-Length" cannot be set here: it is a hop-by-hop or framing header`, + "Transfer-Encoding": `it is a hop-by-hop or framing header`, + "host": `it is a hop-by-hop or framing header`, + "X-Key": `Cortex sets it`, // the server's own auth_header + "bad name": `not a valid header name`, + } { + err := CheckHeaders(map[string]string{name: "v"}, "x-key") + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("CheckHeaders(%q) = %v, want %q", name, err, want) + } + } + for _, v := range []string{"a\nb", "a\rb", "a\x00b"} { + if err := CheckHeaders(map[string]string{"User-Agent": v}, ""); err == nil { + t.Errorf("CheckHeaders(value %q) = nil, want an error", v) + } + } +} + +func TestDecode_APIAuthHeaderHeaders(t *testing.T) { + c, err := Decode(json.RawMessage(`{"servers": {"bob": {"url": "https://bob.example.com", "key": "k", "main": "m", "helper": "h", + "api": {"chat-completions": "/v1/chat/completions"}, "auth_header": "x-api-key", "headers": {"User-Agent": "bob-shell"}}}}`)) + if err != nil { + t.Fatalf("Decode: %v", err) + } + s := c.Servers["bob"] + if s.API["chat-completions"] != "/v1/chat/completions" || s.AuthHeader != "x-api-key" || s.Headers["User-Agent"] != "bob-shell" { + t.Errorf("decoded %+v", s) + } + for field, want := range map[string]string{ + `"api": {"chat": "/x"}`: "servers.bob.api: unknown protocol", + `"auth_header": "Host"`: "servers.bob.auth_header:", + `"headers": {"Authorization": "Bearer x"}`: "servers.bob.headers: \"Authorization\" cannot be set here", + } { + _, err := Decode(json.RawMessage(`{"servers": {"bob": {"url": "https://bob.example.com", "key": "k", "main": "m", "helper": "h", ` + field + `}}}`)) + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("Decode(%s) = %v, want %q", field, err, want) + } + } +} diff --git a/core/redact/redact.go b/core/redact/redact.go index bc7e448d3..7748ac0ca 100644 --- a/core/redact/redact.go +++ b/core/redact/redact.go @@ -47,6 +47,16 @@ func redactMap(m map[string]any) { m[k] = "[REDACTED]" continue } + // A field named exactly "headers" (the inference router's fixed headers) + // holds values that can be secrets under names that are not. Keep the + // names. Matched exactly, not by suffix: id_headers and remove_headers + // hold header names, which must stay visible. + if h, ok := v.(map[string]any); ok && k == "headers" { + for name := range h { + h[name] = "[REDACTED]" + } + continue + } switch val := v.(type) { case map[string]any: redactMap(val) diff --git a/core/redact/redact_test.go b/core/redact/redact_test.go index e6c6e17ca..8ed247203 100644 --- a/core/redact/redact_test.go +++ b/core/redact/redact_test.go @@ -149,3 +149,27 @@ func TestJSON_NestedArrayRedaction(t *testing.T) { t.Errorf("nested array url = %v, want https://a", obj["url"]) } } + +func TestJSON_RedactsHeadersValuesKeepsNames(t *testing.T) { + in := json.RawMessage(`{"servers":{"bob":{"headers":{"User-Agent":"bob-shell"}}},"session":{"id_headers":["X-Session"]},` + + `"cpex":{"delegated_credential_headers":["X-Cred"]},"script":{"remove_headers":["X-Drop"]}}`) + var m struct { + Servers map[string]struct { + Headers map[string]any `json:"headers"` + } `json:"servers"` + Session map[string]any `json:"session"` + Cpex map[string]any `json:"cpex"` + Script map[string]any `json:"script"` + } + if err := json.Unmarshal(JSON(in), &m); err != nil { + t.Fatal(err) + } + if got := m.Servers["bob"].Headers["User-Agent"]; got != "[REDACTED]" { + t.Errorf("headers.User-Agent = %v, want [REDACTED]", got) + } + for name, got := range map[string]any{"id_headers": m.Session["id_headers"], "delegated_credential_headers": m.Cpex["delegated_credential_headers"], "remove_headers": m.Script["remove_headers"]} { + if s, ok := got.([]any); !ok || len(s) != 1 { + t.Errorf("%s = %v, want it unchanged", name, got) + } + } +} diff --git a/core/session/archive/archive.go b/core/session/archive/archive.go index 0de17e37e..15ea3130f 100644 --- a/core/session/archive/archive.go +++ b/core/session/archive/archive.go @@ -375,9 +375,9 @@ func (a *Archive) RecordCapture(sessionID string, e *model.SessionEvent, c *mode // body read with io.ReadAll can hold up to twice that, and headers are not counted, so the // budget bounds memory within a small factor, not exactly. func captureSize(c *model.RequestCapture) int64 { - n := len(c.Original) + len(c.Forwarded) + n := len(c.Original) + len(c.Forwarded) + len(c.Translated) if c.Response != nil { - n += len(c.Response.Body) + n += len(c.Response.Body) + len(c.Response.Upstream) } return int64(n) } diff --git a/core/session/archive/capture.go b/core/session/archive/capture.go index 75ae020d6..d691d9139 100644 --- a/core/session/archive/capture.go +++ b/core/session/archive/capture.go @@ -27,15 +27,17 @@ import ( // construction. The SHA-256 of the original is stored too, and a rebuild that does not match it // is refused. Exact bytes matter: the prompt cache and thinking signatures are sensitive to them. -// captureRec is a request capture on its event record: the skeletons of the original and the -// forwarded body, and the request's headers and query. Old readers ignore the key. +// captureRec is a request capture on its event record: the skeletons of the original, the +// forwarded and the translated body, and the request's headers and query. Old readers ignore the key. // // Headers and the query are not hashed, so they must be exact by construction. JSON strings // cannot hold invalid UTF-8, which Go's server accepts in a header value, so a header set or // query that holds any is written as bytes instead: HX and QX, base64 on disk. type captureRec struct { - O *bodyRec `json:"o,omitempty"` - W *bodyRec `json:"w,omitempty"` + O *bodyRec `json:"o,omitempty"` + W *bodyRec `json:"w,omitempty"` + // X is the translated body (#1911): the request after Cortex converted it to the server's protocol. + X *bodyRec `json:"x,omitempty"` H http.Header `json:"h,omitempty"` HX map[string][][]byte `json:"hx,omitempty"` Q string `json:"q,omitempty"` @@ -47,7 +49,8 @@ type captureRec struct { } // responseRec is a ResponseCapture on disk. B is the key of the block holding the body, empty for -// an empty body; S is the hex SHA-256 of the body, checked on read. +// an empty body; S is the hex SHA-256 of the body, checked on read. UB, US and UT are the same +// for the server's reply before translation (#1911); US is empty when there is none. type responseRec struct { C int `json:"c"` H http.Header `json:"h,omitempty"` @@ -57,6 +60,9 @@ type responseRec struct { T bool `json:"t,omitempty"` IC bool `json:"ic,omitempty"` D bool `json:"d,omitempty"` + UB string `json:"ub,omitempty"` + US string `json:"us,omitempty"` + UT bool `json:"ut,omitempty"` } // bodyRec is one body's skeleton. P alternates literal bytes and block references, starting and @@ -184,13 +190,11 @@ func splitCapture(c *model.RequestCapture, seen map[string]struct{}) (*captureRe T: r.Truncated, IC: r.Incomplete, D: r.Decoded}} rec.R.H, rec.R.HX = headerRec(r.Header) var fresh []stringRec - if len(r.Body) > 0 { - v := string(r.Body) - rec.R.B = hashOf(v) - if _, ok := seen[rec.R.B]; !ok { - seen[rec.R.B] = struct{}{} - fresh = append(fresh, stringRec{h: rec.R.B, v: v}) - } + rec.R.B, fresh = wholeBlock(r.Body, seen, fresh) + if r.Upstream != nil { + sum := sha256.Sum256(r.Upstream) + rec.R.US, rec.R.UT = hex.EncodeToString(sum[:]), r.UpstreamTruncated + rec.R.UB, fresh = wholeBlock(r.Upstream, seen, fresh) } return rec, fresh } @@ -205,9 +209,29 @@ func splitCapture(c *model.RequestCapture, seen map[string]struct{}) (*captureRe f.W, more = splitBody(c.Forwarded, seen) fresh = append(fresh, more...) } + if c.Translated != nil { + var more []stringRec + f.X, more = splitBody(c.Translated, seen) + fresh = append(fresh, more...) + } return f, fresh } +// wholeBlock is the key of the one block that holds body, empty for an empty body. It adds the +// block to fresh when seen does not have it yet. +func wholeBlock(body []byte, seen map[string]struct{}, fresh []stringRec) (string, []stringRec) { + if len(body) == 0 { + return "", fresh + } + v := string(body) + h := hashOf(v) + if _, ok := seen[h]; !ok { + seen[h] = struct{}{} + fresh = append(fresh, stringRec{h: h, v: v}) + } + return h, fresh +} + // headerRec is h as a capture record holds it: as is, or as bytes in hx when a value is not valid // UTF-8, which a JSON string cannot hold. func headerRec(h http.Header) (http.Header, map[string][][]byte) { @@ -283,17 +307,18 @@ func readCapture(path string, seq uint64) (*model.RequestCapture, error) { return nil, errors.New("archive: capture record has no body") } want := map[string]struct{}{} - if f.R != nil && f.R.B != "" { - want[f.R.B] = struct{}{} - } - if f.O != nil { - for _, h := range f.O.refs() { - want[h] = struct{}{} + if f.R != nil { + for _, h := range []string{f.R.B, f.R.UB} { + if h != "" { + want[h] = struct{}{} + } } } - if f.W != nil { - for _, h := range f.W.refs() { - want[h] = struct{}{} + for _, b := range []*bodyRec{f.O, f.W, f.X} { + if b != nil { + for _, h := range b.refs() { + want[h] = struct{}{} + } } } blocks := map[string]string{} @@ -310,8 +335,18 @@ func readCapture(path string, seq uint64) (*model.RequestCapture, error) { if hex.EncodeToString(sum[:]) != r.S { return nil, errors.New("archive: response body does not match the captured hash") } - return &model.RequestCapture{Response: &model.ResponseCapture{Status: r.C, Header: headerOf(r.H, r.HX), - Body: body, Truncated: r.T, Incomplete: r.IC, Decoded: r.D}}, nil + rc := &model.ResponseCapture{Status: r.C, Header: headerOf(r.H, r.HX), + Body: body, Truncated: r.T, Incomplete: r.IC, Decoded: r.D} + if r.US != "" { + // Never nil: an empty reply from the server is not "no translation". + rc.Upstream = append([]byte{}, blocks[r.UB]...) + sum := sha256.Sum256(rc.Upstream) + if hex.EncodeToString(sum[:]) != r.US { + return nil, errors.New("archive: upstream response body does not match the captured hash") + } + rc.UpstreamTruncated = r.UT + } + return &model.RequestCapture{Response: rc}, nil } c := &model.RequestCapture{Header: headerOf(f.H, f.HX), Query: f.Q} if f.QX != nil { @@ -326,5 +361,10 @@ func readCapture(path string, seq uint64) (*model.RequestCapture, error) { return nil, err } } + if f.X != nil { + if c.Translated, err = f.X.rebuild(blocks); err != nil { + return nil, err + } + } return c, nil } diff --git a/core/session/archive/capture_test.go b/core/session/archive/capture_test.go index fbec55ac0..d8293567b 100644 --- a/core/session/archive/capture_test.go +++ b/core/session/archive/capture_test.go @@ -666,3 +666,43 @@ func TestCapture_EventReadsSkipResponseBodies(t *testing.T) { t.Fatalf("got %+v, err %v", got, err) } } + +// Rule 8a of #1911: a translated request's converted body and the server's raw reply come back +// byte for byte beside the bodies they were converted from and to, and count in the queue's size. +func TestCapture_TranslatedAndUpstreamRoundTrip(t *testing.T) { + long := strings.Repeat("x", 600) + req := &pipeline.RequestCapture{Original: []byte(`{"model":"m","system":"` + long + `"}`), + Translated: []byte(`{"model":"m","messages":[{"role":"system","content":[{"type":"text","text":"` + long + `"}]}]}`)} + res := &pipeline.RequestCapture{Response: &pipeline.ResponseCapture{Status: 200, Body: []byte(`event: message_stop` + "\n\n"), + Upstream: []byte("data: \xff[DONE]\n\n"), UpstreamTruncated: true}} + plain := &pipeline.RequestCapture{Response: &pipeline.ResponseCapture{Status: 200, Body: []byte(`{}`)}} + w, err := createSegment(filepath.Join(t.TempDir(), "sess"), time.Now()) + if err != nil { + t.Fatal(err) + } + for i, c := range []*pipeline.RequestCapture{req, res, plain} { + if err := w.appendCapture(pipeline.SessionEvent{Seq: uint64(i + 1)}, c, time.Now()); err != nil { + t.Fatal(err) + } + } + w.close() + path := filepath.Join(filepath.Dir(w.f.Name()), w.info().File) + got, err := readCapture(path, 1) + if err != nil || got == nil || !bytes.Equal(got.Original, req.Original) || !bytes.Equal(got.Translated, req.Translated) || got.Forwarded != nil { + t.Fatalf("request: got %v, err %v, want the original and translated bodies", got != nil, err) + } + got, err = readCapture(path, 2) + if err != nil || got == nil || got.Response == nil || !bytes.Equal(got.Response.Body, res.Response.Body) || + !bytes.Equal(got.Response.Upstream, res.Response.Upstream) || !got.Response.UpstreamTruncated { + t.Fatalf("response: got %v, err %v, want the body and the upstream body", got != nil, err) + } + if got, err = readCapture(path, 3); err != nil || got.Response.Upstream != nil || got.Response.UpstreamTruncated { + t.Fatalf("a reply that was not converted: got %+v, err %v, want no upstream", got, err) + } + if n := captureSize(req); n != int64(len(req.Original)+len(req.Translated)) { + t.Errorf("captureSize(request) = %d, want the translated body counted", n) + } + if n := captureSize(res); n != int64(len(res.Response.Body)+len(res.Response.Upstream)) { + t.Errorf("captureSize(response) = %d, want the upstream body counted", n) + } +} diff --git a/core/sessionapi/archive_test.go b/core/sessionapi/archive_test.go index 4b127a3c0..6d0be17d5 100644 --- a/core/sessionapi/archive_test.go +++ b/core/sessionapi/archive_test.go @@ -347,3 +347,46 @@ func TestHandleGetEvent_RawAddsTheCapture(t *testing.T) { } } } + +// Rule 8a of #1911: GET raw serves a translated request's converted body and the server's raw +// reply, each as text when it is UTF-8 and as base64 otherwise. +func TestHandleGetEvent_RawAddsTheTranslatedHop(t *testing.T) { + root := t.TempDir() + a, err := archive.Open(root) + if err != nil { + t.Fatal(err) + } + store := session.New(0, 0, 0) + store.AddRecorder(a) + store.AppendCapture("s1", pipeline.SessionEvent{At: time.Now(), Phase: pipeline.SessionRequest}, + &pipeline.RequestCapture{Original: []byte(`{"model":"m"}`), Translated: []byte(`{"model":"c"}`)}) + raw := []byte("data: \xff\n\n") + store.AppendCapture("s1", pipeline.SessionEvent{At: time.Now(), Phase: pipeline.SessionResponse}, + &pipeline.RequestCapture{Response: &pipeline.ResponseCapture{Status: 200, Body: []byte(`{}`), Upstream: raw, UpstreamTruncated: true}}) + if err := a.Close(); err != nil { + t.Fatal(err) + } + if a, err = archive.Open(root); err != nil { + t.Fatal(err) + } + defer a.Close() + ts := httptest.NewServer(New(":0", store, WithArchive(a)).server.Handler) + defer ts.Close() + + var got struct { + Raw *pipeline.RawCapture `json:"raw"` + } + code, body := getBody(t, ts.URL+"/v1/sessions/s1/events/1?raw=1") + if code != http.StatusOK || json.Unmarshal(body, &got) != nil || got.Raw == nil || + got.Raw.Translated == nil || *got.Raw.Translated != `{"model":"c"}` || got.Raw.TranslatedBase64 != nil { + t.Fatalf("GET events/1?raw=1 = %d %s, want the translated body as text", code, body) + } + got.Raw = nil + code, body = getBody(t, ts.URL+"/v1/sessions/s1/events/2?raw=1") + if code != http.StatusOK || json.Unmarshal(body, &got) != nil || got.Raw == nil || got.Raw.Response == nil { + t.Fatalf("GET events/2?raw=1 = %d %s", code, body) + } + if r := got.Raw.Response; r.Body != `{}` || r.Upstream != nil || !bytes.Equal(r.UpstreamBase64, raw) || !r.UpstreamTruncated { + t.Fatalf("response = %+v, want the non-UTF-8 upstream body as base64", r) + } +} diff --git a/core/sessionapi/ingest.go b/core/sessionapi/ingest.go index 93d9b0301..3d36e33cd 100644 --- a/core/sessionapi/ingest.go +++ b/core/sessionapi/ingest.go @@ -245,22 +245,48 @@ type ingestCost struct { // ingestCapture is the wire form of model.RequestCapture, which has no JSON tags. type ingestCapture struct { - Original []byte `json:"original"` - Forwarded []byte `json:"forwarded"` - Header http.Header `json:"header"` - Query string `json:"query"` - Response *ingestResponse `json:"response"` - Dropped bool `json:"dropped"` + Original []byte `json:"original"` + Forwarded []byte `json:"forwarded"` + Translated lenientBytes `json:"translated"` + Header http.Header `json:"header"` + Query string `json:"query"` + Response *ingestResponse `json:"response"` + Dropped bool `json:"dropped"` } type ingestResponse struct { - Status int `json:"status"` - Header http.Header `json:"header"` - Body []byte `json:"body"` - Decoded bool `json:"decoded"` - Truncated bool `json:"truncated"` - Ended bool `json:"ended"` - Incomplete bool `json:"incomplete"` + Status int `json:"status"` + Header http.Header `json:"header"` + Body []byte `json:"body"` + Decoded bool `json:"decoded"` + Truncated bool `json:"truncated"` + Ended bool `json:"ended"` + Incomplete bool `json:"incomplete"` + Upstream lenientBytes `json:"upstream"` + UpstreamTruncated lenientBool `json:"upstreamTruncated"` +} + +// lenientBytes decodes a base64 JSON string as its bytes and any other value as nil, so a +// wrong type drops the key and keeps the line, as model.LenientString does. +type lenientBytes []byte + +func (b *lenientBytes) UnmarshalJSON(data []byte) error { + var v []byte + if json.Unmarshal(data, &v) == nil { + *b = v + } + return nil +} + +// lenientBool decodes a JSON bool as itself and any other value as false. +type lenientBool bool + +func (b *lenientBool) UnmarshalJSON(data []byte) error { + var v bool + if json.Unmarshal(data, &v) == nil { + *b = lenientBool(v) + } + return nil } // handleIngest is registered behind httpx.LoopbackOnly (see New), also when the rest of the @@ -465,7 +491,7 @@ func (in *ingester) capture(l ingestLine, ev *model.SessionEvent) *model.Request defer in.reqCapsMu.Unlock() if c.Response == nil { in.reqCaps[l.RequestID] = time.Now() - return &model.RequestCapture{Original: c.Original, Forwarded: c.Forwarded, + return &model.RequestCapture{Original: c.Original, Forwarded: c.Forwarded, Translated: []byte(c.Translated), Header: redact.Headers(c.Header), Query: redact.Query(c.Query)} } in.sweepLocked(time.Now()) @@ -475,7 +501,8 @@ func (in *ingester) capture(l ingestLine, ev *model.SessionEvent) *model.Request delete(in.reqCaps, l.RequestID) res := c.Response return &model.RequestCapture{Response: &model.ResponseCapture{Status: res.Status, Header: redact.Headers(res.Header), - Body: res.Body, Decoded: res.Decoded, Truncated: res.Truncated, Ended: res.Ended, Incomplete: res.Incomplete}} + Body: res.Body, Decoded: res.Decoded, Truncated: res.Truncated, Ended: res.Ended, Incomplete: res.Incomplete, + Upstream: []byte(res.Upstream), UpstreamTruncated: bool(res.UpstreamTruncated)}} } func (s *Server) handleIngestStats(w http.ResponseWriter, r *http.Request) { diff --git a/core/sessionapi/ingest_test.go b/core/sessionapi/ingest_test.go index 12f2c8ba6..060081c37 100644 --- a/core/sessionapi/ingest_test.go +++ b/core/sessionapi/ingest_test.go @@ -623,3 +623,52 @@ func TestIngest_NonStringSourceKeepsTheRow(t *testing.T) { } } } + +// Rule 8a of #1911: an ingested capture keeps a translated request's converted body and the +// server's raw reply. +func TestIngest_CaptureKeepsTheTranslatedHop(t *testing.T) { + r := newIngestRig(t) + r.post(t, nil, + line(t, map[string]any{"capture": map[string]any{ + "original": []byte(`{"a":1}`), "translated": []byte(`{"b":2}`), + }}), + line(t, map[string]any{"event": responseEvent(nil), "capture": map[string]any{ + "response": map[string]any{"status": 200, "body": []byte("{}"), "upstream": []byte("data: [DONE]\n\n"), + "upstreamTruncated": true, "ended": true}, + }}), + ) + req, res := r.caps.get("r1") + if req == nil || res == nil { + t.Fatalf("captures: request %v, response %v", req, res) + } + if string(req.Translated) != `{"b":2}` { + t.Errorf("translated = %q", req.Translated) + } + if string(res.Upstream) != "data: [DONE]\n\n" || !res.UpstreamTruncated { + t.Errorf("upstream = %q, truncated %v", res.Upstream, res.UpstreamTruncated) + } +} + +// A wrong JSON type in a translated-hop key drops that key, not the line. +func TestIngest_BadTranslatedHopKeysKeepTheRow(t *testing.T) { + for _, bad := range []any{123, "{not base64}", map[string]any{"a": 1}, true} { + r := newIngestRig(t) + r.post(t, nil, + line(t, map[string]any{"capture": map[string]any{"original": []byte(`{"a":1}`), "translated": bad}}), + line(t, map[string]any{"event": responseEvent(nil), "capture": map[string]any{ + "response": map[string]any{"status": 200, "upstream": bad, "upstreamTruncated": bad, "ended": true}, + }}), + ) + if evs := r.events("s1"); len(evs) != 2 { + t.Errorf("value %v: stored %d events, want 2", bad, len(evs)) + continue + } + req, res := r.caps.get("r1") + if req == nil || res == nil { + t.Fatalf("value %v: captures: request %v, response %v", bad, req, res) + } + if wantTrunc := bad == true; req.Translated != nil || res.Upstream != nil || res.UpstreamTruncated != wantTrunc { + t.Errorf("value %v: translated %q, upstream %q, truncated %v; want empty, empty, %v", bad, req.Translated, res.Upstream, res.UpstreamTruncated, wantTrunc) + } + } +} diff --git a/core/sessionapi/raw.go b/core/sessionapi/raw.go index 2187ffc34..76b8cdc12 100644 --- a/core/sessionapi/raw.go +++ b/core/sessionapi/raw.go @@ -43,13 +43,28 @@ func rawOf(c *pipeline.RequestCapture) *pipeline.RawCapture { } r.ForwardedBase64 = f64 } + r.Translated, r.TranslatedBase64 = optionalTextOr64(c.Translated) if p := c.Response; p != nil { - r.Response = &pipeline.RawBody{Status: p.Status, Headers: p.Header, Decoded: p.Decoded, Truncated: p.Truncated, Incomplete: p.Incomplete} + r.Response = &pipeline.RawBody{Status: p.Status, Headers: p.Header, Decoded: p.Decoded, Truncated: p.Truncated, + Incomplete: p.Incomplete, UpstreamTruncated: p.UpstreamTruncated} r.Response.Body, r.Response.BodyBase64 = textOr64(p.Body) + r.Response.Upstream, r.Response.UpstreamBase64 = optionalTextOr64(p.Upstream) } return r } +// optionalTextOr64 is textOr64 for a body that can be absent: nil and nil for a nil body. +func optionalTextOr64(b []byte) (*string, []byte) { + if b == nil { + return nil, nil + } + t, t64 := textOr64(b) + if t64 != nil { + return nil, t64 + } + return &t, nil +} + func textOr64(b []byte) (string, []byte) { if utf8.Valid(b) { return string(b), nil diff --git a/core/translate/chat/reply.go b/core/translate/chat/reply.go index e05c16c5f..082dbb9aa 100644 --- a/core/translate/chat/reply.go +++ b/core/translate/chat/reply.go @@ -14,6 +14,10 @@ import ( // maxErrorBody is the most of a non-2xx body that the decoder keeps. const maxErrorBody = 1 << 20 +// maxReplyBuffer is the most of a 2xx reply that the decoder keeps: a whole +// not-streamed body, or one stream line. It is the forward proxy's body limit. +const maxReplyBuffer = 10 << 20 + // maxErrorMessage is the most of a non-JSON error body that goes into the // error message (Review Focus 4). const maxErrorMessage = 512 @@ -79,6 +83,11 @@ func (d *decoder) Feed(p []byte) []pivot.Event { } d.buf = append(d.buf, p...) if !d.stream { + if len(d.buf) > maxReplyBuffer { + d.out = nil + d.overLimit() + return d.out + } return nil } d.out = nil @@ -91,6 +100,10 @@ func (d *decoder) Feed(p []byte) []pivot.Event { d.line(rest[:i]) rest = rest[i+1:] } + if !d.done && len(rest) > maxReplyBuffer { + d.overLimit() + return d.out + } // Keep only the partial line, in a new array, so that the stream // read so far is not held (rule 10). d.buf = append([]byte(nil), rest...) @@ -125,6 +138,12 @@ func (d *decoder) fail(status int, errType, msg string) { d.done = true } +// overLimit ends a reply that is longer than the decoder keeps. +func (d *decoder) overLimit() { + d.fail(http.StatusBadGateway, "api_error", "the inference server's reply is over the "+strconv.Itoa(maxReplyBuffer)+"-byte buffer limit") + d.buf = nil +} + // finish ends a stream at [DONE] or at the end of the body (decision 5). func (d *decoder) finish() { if d.done { diff --git a/core/translate/chat/reply_test.go b/core/translate/chat/reply_test.go index 159d5165f..6238e33c0 100644 --- a/core/translate/chat/reply_test.go +++ b/core/translate/chat/reply_test.go @@ -362,3 +362,26 @@ func TestReplyDecoder_ToolCallIDsNotStream(t *testing.T) { checkEvents(t, 200, false, body, []pivot.Event{apiError(502, "api_error", msg)}) } } + +func TestReplyDecoder_OverLimit(t *testing.T) { + // Valid JSON, so only the limit makes it an error. + body := `{"choices":[{"index":0,"finish_reason":"stop","message":{"role":"assistant","content":"hi"}}]}` + strings.Repeat(" ", maxReplyBuffer) + d := Protocol{}.NewReplyDecoder(200, false) + evs := append(d.Feed([]byte(body)), d.End()...) + if len(evs) != 1 || evs[0].Kind != pivot.EventError || evs[0].Status != 502 { + t.Errorf("not streamed: events = %+v, want one 502 error", evs) + } + // Bytes after [DONE] are not read, so they give no error. + d = Protocol{}.NewReplyDecoder(200, true) + evs = d.Feed([]byte(sse(chunk(`{"content":"a"}`), finish("stop"), "[DONE]") + strings.Repeat("x", maxReplyBuffer+1))) + for _, ev := range evs { + if ev.Kind == pivot.EventError { + t.Errorf("stream: events = %+v, want no error after [DONE]", evs) + } + } + d = Protocol{}.NewReplyDecoder(200, true) + evs = d.Feed([]byte("data: " + strings.Repeat("x", maxReplyBuffer))) + if len(evs) != 1 || evs[0].Kind != pivot.EventError || evs[0].Status != 502 { + t.Errorf("stream line: events = %+v, want one 502 error", evs) + } +} diff --git a/core/translate/chat/request.go b/core/translate/chat/request.go index 6380d8181..8c9b3d9b8 100644 --- a/core/translate/chat/request.go +++ b/core/translate/chat/request.go @@ -109,8 +109,8 @@ func (Protocol) EncodeRequest(req pivot.Request, h http.Header) ([]byte, []strin if req.Stream { out.StreamOptions = &streamOptions{IncludeUsage: true} } - if len(req.System) > 0 { - out.Messages = append(out.Messages, message{Role: "system", Content: e.textParts(req.System)}) + if parts := e.textParts(req.System); len(parts) > 0 { + out.Messages = append(out.Messages, message{Role: "system", Content: parts}) } for _, m := range req.Messages { e.extra(m.Extra) @@ -156,7 +156,7 @@ func (Protocol) EncodeRequest(req pivot.Request, h http.Header) ([]byte, []strin return body, dropped, nil } -// encoder collects the names of the dropped fields (decision 11). +// encoder collects the names of the dropped fields and blocks (decision 11). type encoder struct { dropped map[string]bool } @@ -175,6 +175,10 @@ func (e *encoder) textParts(blocks []pivot.Block) []part { parts := make([]part, 0, len(blocks)) for _, b := range blocks { e.extra(b.Extra) + if b.Kind != pivot.BlockText { + e.dropped[string(b.Kind)] = true + continue + } parts = append(parts, textPart(b.Text, b.CacheControl)) } return parts @@ -221,6 +225,8 @@ func (e *encoder) user(blocks []pivot.Block) ([]message, error) { toolImages = append(toolImages, imagePart(c)) case pivot.BlockDocument: return nil, errDocument + default: + e.dropped[string(c.Kind)] = true } } if len(text) == 0 { @@ -230,6 +236,8 @@ func (e *encoder) user(blocks []pivot.Block) ([]message, error) { } text[len(text)-1].CacheControl = b.CacheControl msgs = append(msgs, message{Role: "tool", Content: text, ToolCallID: b.ToolUseID}) + default: + e.dropped[string(b.Kind)] = true } } if len(toolImages) > 0 { @@ -266,6 +274,8 @@ func (e *encoder) assistant(blocks []pivot.Block) message { m.ThinkingBlocks = append(m.ThinkingBlocks, thinkingBlock{Type: "thinking", Thinking: &t, Signature: b.Signature}) case pivot.BlockRedactedThinking: m.ThinkingBlocks = append(m.ThinkingBlocks, thinkingBlock{Type: "redacted_thinking", Data: b.Data}) + default: + e.dropped[string(b.Kind)] = true } } if text.Len() > 0 { diff --git a/core/translate/chat/request_test.go b/core/translate/chat/request_test.go index 7d575faaa..0ddda77c3 100644 --- a/core/translate/chat/request_test.go +++ b/core/translate/chat/request_test.go @@ -211,3 +211,23 @@ func TestEncodeRequest_PrefixStable(t *testing.T) { t.Errorf("moved marker:\n got %s\nwant %s", got, wantMoved) } } + +func TestEncodeRequest_UnknownBlocksDropped(t *testing.T) { + req := pivot.Request{Model: "m", MaxTokens: 10, Messages: []pivot.Message{ + {Role: "user", Content: []pivot.Block{ + {Kind: pivot.BlockText, Text: "hi"}, + {Kind: "search_result"}, + {Kind: pivot.BlockToolResult, ToolUseID: "t", Content: []pivot.Block{{Kind: "web_fetch_result"}}}, + }}, + {Role: "assistant", Content: []pivot.Block{{Kind: "server_tool_use"}}}, + }} + _, dropped, err := Protocol{}.EncodeRequest(req, http.Header{}) + if err != nil { + t.Fatal(err) + } + for _, k := range []string{"search_result", "server_tool_use", "web_fetch_result"} { + if !strings.Contains(strings.Join(dropped, ","), k) { + t.Errorf("dropped = %v, want it to name %s", dropped, k) + } + } +} diff --git a/core/translate/convert.go b/core/translate/convert.go new file mode 100644 index 000000000..789905356 --- /dev/null +++ b/core/translate/convert.go @@ -0,0 +1,92 @@ +package translate + +import ( + "fmt" + "net/http" + + "github.com/rossoctl/cortex/core/translate/pivot" +) + +// Converted is a request converted to the target protocol. +type Converted struct { + Body []byte + Dropped []string // the fields and blocks that the target could not carry (rule H4) + Stream bool // the agent asked for a stream (decision 4) +} + +// Request converts a request body from the source protocol to the target +// protocol, and removes the source protocol's own headers from h. A +// refusal to send to the agent is a *pivot.RequestError. +func Request(source, target string, body []byte, h http.Header) (Converted, error) { + dec, ok := protocols[source].(pivot.RequestDecoder) + if !ok { + return Converted{}, fmt.Errorf("translate: protocol %q cannot decode a request", source) + } + enc, ok := protocols[target].(pivot.RequestEncoder) + if !ok { + return Converted{}, fmt.Errorf("translate: protocol %q cannot encode a request", target) + } + req, err := dec.DecodeRequest(body) + if err != nil { + return Converted{}, err + } + out, dropped, err := enc.EncodeRequest(req, h) + if err != nil { + return Converted{}, err + } + return Converted{Body: out, Dropped: dropped, Stream: req.Stream}, nil +} + +// replyCodecs gives the decoder of the target's reply and the encoder of +// the source's reply. false when a direction is missing. +func replyCodecs(source, target string, status int, upstreamStream, agentStream bool) (pivot.ReplyDecoder, pivot.ReplyEncoder, bool) { + dec, ok := protocols[target].(pivot.ReplyDecoding) + if !ok { + return nil, nil, false + } + enc, ok := protocols[source].(pivot.ReplyEncoding) + if !ok { + return nil, nil, false + } + return dec.NewReplyDecoder(status, upstreamStream), enc.NewReplyEncoder(agentStream), true +} + +// Reply converts a whole reply body of the target protocol to the reply +// that the source protocol's agent expects. upstreamStream tells if the +// body is SSE; agentStream tells if the agent asked for a stream. When a +// direction is missing, it gives a 502 error body in the source protocol +// (nil for an unknown source). It also gives the status of the converted +// reply: the status of its error event, or status when it has none. +func Reply(source, target string, status int, upstreamStream, agentStream bool, body []byte) (int, []byte) { + dec, enc, ok := replyCodecs(source, target, status, upstreamStream, agentStream) + if !ok { + return http.StatusBadGateway, ErrorBody(source, http.StatusBadGateway, "api_error", + "Cortex cannot convert a "+target+" reply to "+source) + } + evs := append(dec.Feed(body), dec.End()...) + for _, ev := range evs { + if ev.Kind == pivot.EventError && ev.Status != 0 { + status = ev.Status + } + } + out := encodeAll(enc, nil, evs) + return status, append(out, enc.End()...) +} + +// encodeAll appends the encoded events to out. +func encodeAll(enc pivot.ReplyEncoder, out []byte, evs []pivot.Event) []byte { + for _, ev := range evs { + out = append(out, enc.Encode(ev)...) + } + return out +} + +// ErrorBody gives an error body in the protocol's shape, or nil for an +// unknown protocol. +func ErrorBody(protocol string, status int, errType, message string) []byte { + p, ok := protocols[protocol] + if !ok { + return nil + } + return p.EncodeError(status, errType, message) +} diff --git a/core/translate/golden_test.go b/core/translate/golden_test.go new file mode 100644 index 000000000..95daa2663 --- /dev/null +++ b/core/translate/golden_test.go @@ -0,0 +1,180 @@ +package translate + +import ( + "bytes" + "flag" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +var update = flag.Bool("update", false, "rewrite golden files") + +// checkGolden compares got to the golden file, or writes it with -update. +func checkGolden(t *testing.T, path string, got []byte) { + t.Helper() + if *update { + if err := os.WriteFile(path, got, 0o644); err != nil { + t.Fatal(err) + } + return + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatalf("%v (run with -update to write it)", err) + } + if !bytes.Equal(got, want) { + t.Errorf("%s:\n got %s\nwant %s", path, got, want) + } +} + +// requestGolden gives the golden form of one request conversion: the +// first line names the dropped fields, the second line is the body. A +// refusal gives one "error:" line. +func requestGolden(in []byte) []byte { + c, err := Request(Anthropic, Chat, in, nil) + if err != nil { + return []byte("error: " + err.Error() + "\n") + } + return []byte("dropped: " + strings.Join(c.Dropped, ",") + "\n" + string(c.Body) + "\n") +} + +// TestRequestGolden converts each hand-written Anthropic request in +// testdata/request to chat completions (rules 11-19, 20a). +func TestRequestGolden(t *testing.T) { + inputs, _ := filepath.Glob("testdata/request/*.json") + if len(inputs) == 0 { + t.Fatal("no request inputs") + } + for _, in := range inputs { + t.Run(filepath.Base(in), func(t *testing.T) { + body, err := os.ReadFile(in) + if err != nil { + t.Fatal(err) + } + checkGolden(t, strings.TrimSuffix(in, ".json")+".golden", requestGolden(body)) + }) + } +} + +func encodeChat(t *testing.T, in []byte) string { + t.Helper() + c, err := Request(Anthropic, Chat, in, nil) + if err != nil { + t.Fatal(err) + } + return string(c.Body) +} + +// TestDeterministic checks rule 9a on the full multi-turn request. +func TestDeterministic(t *testing.T) { + in, err := os.ReadFile("testdata/request/full_multiturn.json") + if err != nil { + t.Fatal(err) + } + first := encodeChat(t, in) + if again := encodeChat(t, in); again != first { + t.Fatalf("two encodings differ:\n%s\n%s", first, again) + } + + // One more turn: the bytes up to the end of the old last message stay. + const marker = `,"cache_control":{"type":"ephemeral"}` + const oldLast = `"text":"Thanks. Is 391 prime?"` + lastOut := `{"role":"user","content":[{"type":"text",` + oldLast + marker + `}]}` + end := strings.Index(first, lastOut) + if end < 0 { + t.Fatalf("the last old message is not in the output:\n%s", first) + } + end += len(lastOut) + lastIn := oldLast + marker + `}]}` + longer := bytes.Replace(in, []byte(lastIn), []byte(lastIn+ + `,{"role":"assistant","content":[{"type":"text","text":"No: 17 x 23."}]}`+ + `,{"role":"user","content":[{"type":"text","text":"And 397?"}]}`), 1) + if bytes.Equal(longer, in) { + t.Fatal("the fixture does not hold the last user turn") + } + grown := encodeChat(t, longer) + if !strings.HasPrefix(grown, first[:end]) { + t.Errorf("one more turn changed the old bytes:\n old %s\n new %s", first[:end], grown) + } + + // Move the cache marker to the new last turn: only the marker bytes + // change. + moved := bytes.Replace(longer, []byte(oldLast+marker), []byte(oldLast), 1) + moved = bytes.Replace(moved, []byte(`"text":"And 397?"`), []byte(`"text":"And 397?"`+marker), 1) + movedOut := encodeChat(t, moved) + if movedOut == grown { + t.Fatal("moving the marker did not change the output") + } + if strings.ReplaceAll(movedOut, marker, "") != strings.ReplaceAll(grown, marker, "") { + t.Errorf("moving the marker changed more than the marker:\n%s\n%s", grown, movedOut) + } +} + +// replyCase is one hand-written chat-completions reply in testdata/reply. +// A .sse input is a stream. The status is 200, or N for a name that +// starts with "error_N_". +type replyCase struct { + path string + status int + stream bool +} + +func replyCases(t *testing.T) []replyCase { + t.Helper() + inputs, _ := filepath.Glob("testdata/reply/*.sse") + more, _ := filepath.Glob("testdata/reply/*.json") + inputs = append(inputs, more...) + if len(inputs) == 0 { + t.Fatal("no reply inputs") + } + var out []replyCase + for _, in := range inputs { + c := replyCase{path: in, status: 200, stream: strings.HasSuffix(in, ".sse")} + if rest, ok := strings.CutPrefix(filepath.Base(in), "error_"); ok { + n, err := strconv.Atoi(strings.SplitN(rest, "_", 2)[0]) + if err != nil { + t.Fatalf("%s: no status in the name", in) + } + c.status = n + } + out = append(out, c) + } + return out +} + +// agentForms gives the reply shapes that the agent asks for: the same +// shape as the upstream for a 2xx reply, both shapes for an error. +func (c replyCase) agentForms() []bool { + if c.status >= 200 && c.status < 300 { + return []bool{c.stream} + } + return []bool{true, false} +} + +func goldenName(c replyCase, agentStream bool) string { + if agentStream { + return c.path + ".to-sse.golden" + } + return c.path + ".to-json.golden" +} + +// TestReplyGolden converts each chat-completions reply to the Anthropic +// reply (rules 20-24). +func TestReplyGolden(t *testing.T) { + for _, c := range replyCases(t) { + body, err := os.ReadFile(c.path) + if err != nil { + t.Fatal(err) + } + for _, agentStream := range c.agentForms() { + name := goldenName(c, agentStream) + t.Run(filepath.Base(name), func(t *testing.T) { + _, out := Reply(Anthropic, Chat, c.status, c.stream, agentStream, body) + checkGolden(t, name, out) + }) + } + } +} diff --git a/core/translate/reader.go b/core/translate/reader.go new file mode 100644 index 000000000..da860fa7c --- /dev/null +++ b/core/translate/reader.go @@ -0,0 +1,55 @@ +package translate + +import ( + "io" + + "github.com/rossoctl/cortex/core/translate/pivot" +) + +// readSize is the most that one Read takes from the upstream body. +const readSize = 32 << 10 + +// replyReader converts an upstream reply body as it is read (rule 10). +type replyReader struct { + body io.ReadCloser + dec pivot.ReplyDecoder + enc pivot.ReplyEncoder + buf []byte // upstream bytes of one Read + pending []byte // converted bytes not read yet + err error // the upstream error, after pending is read +} + +// NewReplyReader gives the converted reply as a stream. Each Read takes +// at most 32 KiB from body and returns as soon as it has converted bytes: +// it never waits for the whole stream. At the upstream EOF it ends the +// decoder and the encoder. An upstream read error other than EOF comes +// after the bytes already converted. Close closes body. +func NewReplyReader(body io.ReadCloser, dec pivot.ReplyDecoder, enc pivot.ReplyEncoder) io.ReadCloser { + return &replyReader{body: body, dec: dec, enc: enc} +} + +func (r *replyReader) Read(p []byte) (int, error) { + for len(r.pending) == 0 { + if r.err != nil { + return 0, r.err + } + if r.buf == nil { + r.buf = make([]byte, readSize) + } + n, err := r.body.Read(r.buf) + r.pending = encodeAll(r.enc, r.pending, r.dec.Feed(r.buf[:n])) + switch { + case err == io.EOF: + r.pending = encodeAll(r.enc, r.pending, r.dec.End()) + r.pending = append(r.pending, r.enc.End()...) + r.err = io.EOF + case err != nil: + r.err = err + } + } + n := copy(p, r.pending) + r.pending = r.pending[n:] + return n, nil +} + +func (r *replyReader) Close() error { return r.body.Close() } diff --git a/core/translate/reader_test.go b/core/translate/reader_test.go new file mode 100644 index 000000000..55eeea677 --- /dev/null +++ b/core/translate/reader_test.go @@ -0,0 +1,102 @@ +package translate + +import ( + "bytes" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + "testing/iotest" + "time" + + "github.com/rossoctl/cortex/core/translate/anthropic" + "github.com/rossoctl/cortex/core/translate/chat" +) + +func newReader(r io.Reader, status int, upstreamStream, agentStream bool) io.ReadCloser { + return NewReplyReader(io.NopCloser(r), + chat.Protocol{}.NewReplyDecoder(status, upstreamStream), + anthropic.Protocol{}.NewReplyEncoder(agentStream)) +} + +// TestReplyReader_EveryByteSplit reads each reply input one byte per +// Read, so the input splits at every byte boundary. The output must equal +// the golden. +func TestReplyReader_EveryByteSplit(t *testing.T) { + for _, c := range replyCases(t) { + body, err := os.ReadFile(c.path) + if err != nil { + t.Fatal(err) + } + for _, agentStream := range c.agentForms() { + name := goldenName(c, agentStream) + t.Run(filepath.Base(name), func(t *testing.T) { + want, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + got, err := io.ReadAll(newReader(iotest.OneByteReader(bytes.NewReader(body)), c.status, c.stream, agentStream)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Errorf("got %s\nwant %s", got, want) + } + }) + } + } +} + +// TestReplyReader_Incremental checks rule 10: the first chunk gives the +// message_start frame while the upstream stream is still open. +func TestReplyReader_Incremental(t *testing.T) { + pr, pw := io.Pipe() + defer pw.Close() + go func() { + _, _ = pw.Write([]byte(`data: {"id":"chatcmpl-1","model":"m","choices":[{"index":0,"delta":{"role":"assistant"}}]}` + "\n\n")) + }() + r := newReader(pr, 200, true, true) + got := make(chan string, 1) + go func() { + buf := make([]byte, 4096) + n, _ := r.Read(buf) + got <- string(buf[:n]) + }() + select { + case s := <-got: + if !strings.HasPrefix(s, "event: message_start\n") { + t.Errorf("first Read = %q, want the message_start frame", s) + } + case <-time.After(5 * time.Second): + t.Fatal("Read waited for more of the stream") + } +} + +// TestReplyReader_ReadError checks that an upstream read error comes +// after the bytes already converted, and that Close closes the upstream. +func TestReplyReader_ReadError(t *testing.T) { + boom := errors.New("boom") + up := io.MultiReader(strings.NewReader(`data: {"id":"c","model":"m","choices":[]}`+"\n\n"), iotest.ErrReader(boom)) + closed := false + r := NewReplyReader(closer{up, &closed}, + chat.Protocol{}.NewReplyDecoder(200, true), anthropic.Protocol{}.NewReplyEncoder(true)) + got, err := io.ReadAll(r) + if !errors.Is(err, boom) { + t.Errorf("err = %v, want %v", err, boom) + } + if !strings.HasPrefix(string(got), "event: message_start\n") { + t.Errorf("bytes before the error = %q", got) + } + if err := r.Close(); err != nil || !closed { + t.Errorf("Close = %v, closed = %v", err, closed) + } +} + +type closer struct { + io.Reader + closed *bool +} + +func (c closer) Close() error { *c.closed = true; return nil } diff --git a/core/translate/registry.go b/core/translate/registry.go new file mode 100644 index 000000000..72b14877a --- /dev/null +++ b/core/translate/registry.go @@ -0,0 +1,72 @@ +package translate + +import ( + "slices" + + "github.com/rossoctl/cortex/core/translate/anthropic" + "github.com/rossoctl/cortex/core/translate/chat" + "github.com/rossoctl/cortex/core/translate/pivot" +) + +// The registered protocol names. +const ( + Anthropic = "anthropic-messages" + Chat = "chat-completions" + Responses = "responses" +) + +// protocols is the compiled-in registry (rule H5). It has no +// registration API. +var protocols = map[string]pivot.Protocol{ + Anthropic: anthropic.Protocol{}, + Chat: chat.Protocol{}, + Responses: responses{}, +} + +// preference is the order in which Target picks a protocol (decision 12). +var preference = []string{Chat, Anthropic, Responses} + +// responses is registered with no directions, so that a server can say +// that it serves Responses (rule C1). +type responses struct{} + +func (responses) Name() string { return Responses } + +// EncodeError gives an error body in the OpenAI shape. +func (responses) EncodeError(status int, errType, message string) []byte { + return chat.Protocol{}.EncodeError(status, errType, message) +} + +// Lookup gives the protocol with this name. +func Lookup(name string) (pivot.Protocol, bool) { + p, ok := protocols[name] + return p, ok +} + +// Names gives the registered protocol names, sorted. +func Names() []string { + names := make([]string, 0, len(protocols)) + for n := range protocols { + names = append(names, n) + } + slices.Sort(names) + return names +} + +// Decodes tells if the protocol can decode a request. +func Decodes(name string) bool { + _, ok := protocols[name].(pivot.RequestDecoder) + return ok +} + +// Target gives the protocol to translate to, from the protocols that a +// server serves: the first in the preference order that has a request +// encoder (rule 4). false when there is none (rule 4a). +func Target(served []string) (string, bool) { + for _, name := range preference { + if _, ok := protocols[name].(pivot.RequestEncoder); ok && slices.Contains(served, name) { + return name, true + } + } + return "", false +} diff --git a/core/translate/registry_test.go b/core/translate/registry_test.go new file mode 100644 index 000000000..df05cdcba --- /dev/null +++ b/core/translate/registry_test.go @@ -0,0 +1,68 @@ +package translate + +import ( + "reflect" + "testing" +) + +func TestNames(t *testing.T) { + want := []string{"anthropic-messages", "chat-completions", "responses"} + if got := Names(); !reflect.DeepEqual(got, want) { + t.Errorf("Names() = %v, want %v", got, want) + } + for _, n := range want { + p, ok := Lookup(n) + if !ok || p.Name() != n { + t.Errorf("Lookup(%q) = %v, %v", n, p, ok) + } + } + if _, ok := Lookup("gemini"); ok { + t.Error(`Lookup("gemini") found a protocol`) + } +} + +func TestTarget(t *testing.T) { + for _, tc := range []struct { + served []string + want string + ok bool + }{ + {[]string{"responses"}, "", false}, + {[]string{"responses", "chat-completions"}, "chat-completions", true}, + {[]string{"anthropic-messages", "responses"}, "", false}, + {nil, "", false}, + {[]string{"unknown", "chat-completions"}, "chat-completions", true}, + } { + got, ok := Target(tc.served) + if got != tc.want || ok != tc.ok { + t.Errorf("Target(%v) = %q, %v; want %q, %v", tc.served, got, ok, tc.want, tc.ok) + } + } +} + +func TestDecodes(t *testing.T) { + for name, want := range map[string]bool{Anthropic: true, Chat: false, Responses: false, "unknown": false} { + if got := Decodes(name); got != want { + t.Errorf("Decodes(%q) = %v, want %v", name, got, want) + } + } +} + +func TestErrorBody(t *testing.T) { + for _, tc := range []struct{ protocol, want string }{ + {Anthropic, `{"type":"error","error":{"type":"api_error","message":"m"}}`}, + {Chat, `{"error":{"message":"m","type":"api_error","code":null}}`}, + {Responses, `{"error":{"message":"m","type":"api_error","code":null}}`}, + {"unknown", ``}, + } { + if got := string(ErrorBody(tc.protocol, 502, "api_error", "m")); got != tc.want { + t.Errorf("ErrorBody(%q) = %s, want %s", tc.protocol, got, tc.want) + } + } +} + +func TestRequest_NoDirection(t *testing.T) { + if _, err := Request(Chat, Anthropic, []byte(`{}`), nil); err == nil { + t.Error("Request(chat, anthropic) gave no error; neither direction exists") + } +} diff --git a/core/translate/testdata/reply/bob_shape1.json b/core/translate/testdata/reply/bob_shape1.json new file mode 100644 index 000000000..ec14be4ac --- /dev/null +++ b/core/translate/testdata/reply/bob_shape1.json @@ -0,0 +1,5 @@ +{"id":"chatcmpl-b1","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion", + "choices":[{"index":0,"finish_reason":"tool_calls","message":{"role":"assistant","content":"391 is not prime","reasoning":"391 Not", + "thinking_blocks":[{"type":"thinking","thinking":"391 Not","signature":"EvkBsig"}], + "tool_calls":[{"id":"tooluse_9nf","type":"function","function":{"name":"Bash","arguments":"{\"command\": \"echo done\"}"}}]}}], + "usage":{"prompt_tokens":578,"completion_tokens":55,"total_tokens":633,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"prompt_tokens_details":{"cached_tokens":0},"completion_tokens_details":{"reasoning_tokens":0}}} diff --git a/core/translate/testdata/reply/bob_shape1.json.to-json.golden b/core/translate/testdata/reply/bob_shape1.json.to-json.golden new file mode 100644 index 000000000..812a4043a --- /dev/null +++ b/core/translate/testdata/reply/bob_shape1.json.to-json.golden @@ -0,0 +1 @@ +{"id":"chatcmpl-b1","type":"message","role":"assistant","model":"claude-opus-5-5","content":[{"type":"thinking","thinking":"391 Not","signature":"EvkBsig"},{"type":"text","text":"391 is not prime"},{"type":"tool_use","id":"tooluse_9nf","name":"Bash","input":{"command":"echo done"}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":578,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":55}} \ No newline at end of file diff --git a/core/translate/testdata/reply/bob_shape1.sse b/core/translate/testdata/reply/bob_shape1.sse new file mode 100644 index 000000000..e358cd1a0 --- /dev/null +++ b/core/translate/testdata/reply/bob_shape1.sse @@ -0,0 +1,23 @@ +: ping + +data: {"id":"chatcmpl-b1","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"reasoning":"391","thinking_blocks":[{"type":"thinking","thinking":"391"}],"role":"assistant"},"finish_reason":null}]} + +data: {"id":"chatcmpl-b1","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"reasoning":" Not","thinking_blocks":[{"type":"thinking","thinking":" Not"}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-b1","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"thinking_blocks":[{"type":"thinking","thinking":"","signature":"EvkBsig"}]},"finish_reason":null}]} + +: ping + +data: {"id":"chatcmpl-b1","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"391 is not prime"},"finish_reason":null}]} + +data: {"id":"chatcmpl-b1","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"tool_calls":[{"id":"tooluse_9nf","function":{"arguments":"","name":"Bash"},"type":"function","index":0}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-b1","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"tool_calls":[{"function":{"arguments":"{\"comma"},"type":"function","index":0}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-b1","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"tool_calls":[{"function":{"arguments":"nd\": \"echo done\"}"},"type":"function","index":0}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-b1","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]} + +data: {"id":"chatcmpl-b1","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[],"usage":{"prompt_tokens":578,"completion_tokens":55,"total_tokens":633,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"prompt_tokens_details":{"cached_tokens":0},"completion_tokens_details":{"reasoning_tokens":0}}} + +data: [DONE] diff --git a/core/translate/testdata/reply/bob_shape1.sse.to-sse.golden b/core/translate/testdata/reply/bob_shape1.sse.to-sse.golden new file mode 100644 index 000000000..837f5809c --- /dev/null +++ b/core/translate/testdata/reply/bob_shape1.sse.to-sse.golden @@ -0,0 +1,45 @@ +event: message_start +data: {"type":"message_start","message":{"id":"chatcmpl-b1","type":"message","role":"assistant","model":"claude-opus-5-5","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"391"}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":" Not"}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"EvkBsig"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: content_block_start +data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"391 is not prime"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":1} + +event: content_block_start +data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"tooluse_9nf","name":"Bash","input":{}}} + +event: content_block_delta +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\"comma"}} + +event: content_block_delta +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"nd\": \"echo done\"}"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":2} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"input_tokens":578,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":55}} + +event: message_stop +data: {"type":"message_stop"} + diff --git a/core/translate/testdata/reply/bob_shape2.json b/core/translate/testdata/reply/bob_shape2.json new file mode 100644 index 000000000..367442b09 --- /dev/null +++ b/core/translate/testdata/reply/bob_shape2.json @@ -0,0 +1,5 @@ +{"id":"chatcmpl-b2","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion", + "choices":[{"index":0,"finish_reason":"tool_calls","message":{"role":"assistant","content":"I will list the files.","reasoning":"", + "thinking_blocks":[{"type":"thinking","thinking":"","signature":"CAQSsig"}], + "tool_calls":[{"id":"tooluse_Zq1","type":"function","function":{"name":"execute_command","arguments":"{\"command\":\"ls\"}"}}]}}], + "usage":{"prompt_tokens":22333,"completion_tokens":216,"total_tokens":22549,"cache_read_input_tokens":21649,"cache_creation_input_tokens":682,"prompt_tokens_details":{"cached_tokens":21649},"completion_tokens_details":{"reasoning_tokens":0}}} diff --git a/core/translate/testdata/reply/bob_shape2.json.to-json.golden b/core/translate/testdata/reply/bob_shape2.json.to-json.golden new file mode 100644 index 000000000..335dec6ce --- /dev/null +++ b/core/translate/testdata/reply/bob_shape2.json.to-json.golden @@ -0,0 +1 @@ +{"id":"chatcmpl-b2","type":"message","role":"assistant","model":"claude-opus-5-5","content":[{"type":"thinking","thinking":"","signature":"CAQSsig"},{"type":"text","text":"I will list the files."},{"type":"tool_use","id":"tooluse_Zq1","name":"execute_command","input":{"command":"ls"}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":682,"cache_read_input_tokens":21649,"output_tokens":216}} \ No newline at end of file diff --git a/core/translate/testdata/reply/bob_shape2.sse b/core/translate/testdata/reply/bob_shape2.sse new file mode 100644 index 000000000..c52d249a8 --- /dev/null +++ b/core/translate/testdata/reply/bob_shape2.sse @@ -0,0 +1,15 @@ +data: {"id":"chatcmpl-b2","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"reasoning":"","thinking_blocks":[{"type":"thinking","thinking":""}],"content":"","role":"assistant"},"finish_reason":null}]} + +data: {"id":"chatcmpl-b2","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"reasoning":"","thinking_blocks":[{"type":"thinking","thinking":"","signature":"CAQSsig"}],"content":""},"finish_reason":null}]} + +data: {"id":"chatcmpl-b2","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"I will list the files."},"finish_reason":null}]} + +data: {"id":"chatcmpl-b2","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"","tool_calls":[{"id":"tooluse_Zq1","function":{"arguments":"","name":"execute_command"},"type":"function","index":0}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-b2","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"","tool_calls":[{"function":{"arguments":"{\"command\":\"ls\"}"},"type":"function","index":0}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-b2","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]} + +data: {"id":"chatcmpl-b2","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[],"usage":{"prompt_tokens":22333,"completion_tokens":216,"total_tokens":22549,"cache_read_input_tokens":21649,"cache_creation_input_tokens":682,"prompt_tokens_details":{"cached_tokens":21649},"completion_tokens_details":{"reasoning_tokens":0}}} + +data: [DONE] diff --git a/core/translate/testdata/reply/bob_shape2.sse.to-sse.golden b/core/translate/testdata/reply/bob_shape2.sse.to-sse.golden new file mode 100644 index 000000000..ae02a512e --- /dev/null +++ b/core/translate/testdata/reply/bob_shape2.sse.to-sse.golden @@ -0,0 +1,36 @@ +event: message_start +data: {"type":"message_start","message":{"id":"chatcmpl-b2","type":"message","role":"assistant","model":"claude-opus-5-5","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"CAQSsig"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: content_block_start +data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"I will list the files."}} + +event: content_block_stop +data: {"type":"content_block_stop","index":1} + +event: content_block_start +data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"tooluse_Zq1","name":"execute_command","input":{}}} + +event: content_block_delta +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\"command\":\"ls\"}"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":2} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"input_tokens":2,"cache_creation_input_tokens":682,"cache_read_input_tokens":21649,"output_tokens":216}} + +event: message_stop +data: {"type":"message_stop"} + diff --git a/core/translate/testdata/reply/bob_shape3.json b/core/translate/testdata/reply/bob_shape3.json new file mode 100644 index 000000000..fda320e76 --- /dev/null +++ b/core/translate/testdata/reply/bob_shape3.json @@ -0,0 +1,5 @@ +{"id":"chatcmpl-b3","created":1763337600,"model":"premium-ide","object":"chat.completion", + "choices":[{"index":0,"finish_reason":"tool_calls","message":{"role":"assistant","content":null, + "provider_specific_fields":{"thought_signatures":["AY89a1+8zd/x="]}, + "tool_calls":[{"id":"call_3833__thought__AY89a1+8zd/x=","type":"function","function":{"name":"spawn_subagent","arguments":"{\"task\":\"look\"}"}}]}}], + "usage":{"prompt_tokens":13314,"completion_tokens":296,"total_tokens":13610,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"completion_tokens_details":{"reasoning_tokens":249}}} diff --git a/core/translate/testdata/reply/bob_shape3.json.to-json.golden b/core/translate/testdata/reply/bob_shape3.json.to-json.golden new file mode 100644 index 000000000..5c516518c --- /dev/null +++ b/core/translate/testdata/reply/bob_shape3.json.to-json.golden @@ -0,0 +1 @@ +{"id":"chatcmpl-b3","type":"message","role":"assistant","model":"premium-ide","content":[{"type":"tool_use","id":"call_3833__thought__AY89a1+8zd/x=","name":"spawn_subagent","input":{"task":"look"}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":13314,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":296}} \ No newline at end of file diff --git a/core/translate/testdata/reply/bob_shape3.sse b/core/translate/testdata/reply/bob_shape3.sse new file mode 100644 index 000000000..9be0802c1 --- /dev/null +++ b/core/translate/testdata/reply/bob_shape3.sse @@ -0,0 +1,7 @@ +data: {"id":"chatcmpl-b3","created":1763337600,"model":"premium-ide","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"provider_specific_fields":{"thought_signatures":["AY89a1+8zd/x="]},"role":"assistant","tool_calls":[{"id":"call_3833__thought__AY89a1+8zd/x=","function":{"arguments":"{\"task\":\"look\"}","name":"spawn_subagent"},"type":"function","index":0}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-b3","created":1763337600,"model":"premium-ide","object":"chat.completion.chunk","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]} + +data: {"id":"chatcmpl-b3","created":1763337600,"model":"premium-ide","object":"chat.completion.chunk","choices":[],"usage":{"prompt_tokens":13314,"completion_tokens":296,"total_tokens":13610,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"completion_tokens_details":{"reasoning_tokens":249}}} + +data: [DONE] diff --git a/core/translate/testdata/reply/bob_shape3.sse.to-sse.golden b/core/translate/testdata/reply/bob_shape3.sse.to-sse.golden new file mode 100644 index 000000000..9fdee2fb5 --- /dev/null +++ b/core/translate/testdata/reply/bob_shape3.sse.to-sse.golden @@ -0,0 +1,18 @@ +event: message_start +data: {"type":"message_start","message":{"id":"chatcmpl-b3","type":"message","role":"assistant","model":"premium-ide","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"call_3833__thought__AY89a1+8zd/x=","name":"spawn_subagent","input":{}}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"task\":\"look\"}"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"input_tokens":13314,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":296}} + +event: message_stop +data: {"type":"message_stop"} + diff --git a/core/translate/testdata/reply/chunk_error.json b/core/translate/testdata/reply/chunk_error.json new file mode 100644 index 000000000..924fc3c51 --- /dev/null +++ b/core/translate/testdata/reply/chunk_error.json @@ -0,0 +1 @@ +{"error":{"message":"litellm.InternalServerError: the upstream connection was reset","type":"None","code":"500"}} diff --git a/core/translate/testdata/reply/chunk_error.json.to-json.golden b/core/translate/testdata/reply/chunk_error.json.to-json.golden new file mode 100644 index 000000000..764313697 --- /dev/null +++ b/core/translate/testdata/reply/chunk_error.json.to-json.golden @@ -0,0 +1 @@ +{"type":"error","error":{"type":"api_error","message":"litellm.InternalServerError: the upstream connection was reset"}} \ No newline at end of file diff --git a/core/translate/testdata/reply/chunk_error.sse b/core/translate/testdata/reply/chunk_error.sse new file mode 100644 index 000000000..98b2657bc --- /dev/null +++ b/core/translate/testdata/reply/chunk_error.sse @@ -0,0 +1,3 @@ +data: {"id":"chatcmpl-e1","model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":"Starting"},"finish_reason":null}]} + +data: {"error":{"message":"litellm.InternalServerError: the upstream connection was reset","type":"None","code":"500"}} diff --git a/core/translate/testdata/reply/chunk_error.sse.to-sse.golden b/core/translate/testdata/reply/chunk_error.sse.to-sse.golden new file mode 100644 index 000000000..8154e4761 --- /dev/null +++ b/core/translate/testdata/reply/chunk_error.sse.to-sse.golden @@ -0,0 +1,12 @@ +event: message_start +data: {"type":"message_start","message":{"id":"chatcmpl-e1","type":"message","role":"assistant","model":"claude-opus-5-5","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Starting"}} + +event: error +data: {"type":"error","error":{"type":"api_error","message":"litellm.InternalServerError: the upstream connection was reset"}} + diff --git a/core/translate/testdata/reply/error_403_litellm.json b/core/translate/testdata/reply/error_403_litellm.json new file mode 100644 index 000000000..43092ce96 --- /dev/null +++ b/core/translate/testdata/reply/error_403_litellm.json @@ -0,0 +1 @@ +{"error":{"message":"team not allowed to access model. This team can only access models=['granite-4']. Tried to access claude-opus-5-5","type":"team_model_access_denied","param":"model","code":"403"}} diff --git a/core/translate/testdata/reply/error_403_litellm.json.to-json.golden b/core/translate/testdata/reply/error_403_litellm.json.to-json.golden new file mode 100644 index 000000000..0e686c0ab --- /dev/null +++ b/core/translate/testdata/reply/error_403_litellm.json.to-json.golden @@ -0,0 +1 @@ +{"type":"error","error":{"type":"permission_error","message":"team not allowed to access model. This team can only access models=['granite-4']. Tried to access claude-opus-5-5"}} \ No newline at end of file diff --git a/core/translate/testdata/reply/error_403_litellm.json.to-sse.golden b/core/translate/testdata/reply/error_403_litellm.json.to-sse.golden new file mode 100644 index 000000000..a987899d6 --- /dev/null +++ b/core/translate/testdata/reply/error_403_litellm.json.to-sse.golden @@ -0,0 +1,3 @@ +event: error +data: {"type":"error","error":{"type":"permission_error","message":"team not allowed to access model. This team can only access models=['granite-4']. Tried to access claude-opus-5-5"}} + diff --git a/core/translate/testdata/reply/error_404_bob.json b/core/translate/testdata/reply/error_404_bob.json new file mode 100644 index 000000000..b3190be02 --- /dev/null +++ b/core/translate/testdata/reply/error_404_bob.json @@ -0,0 +1 @@ +{"error":{"message":"model \"claude-opus-5-5\" not found or not accessible","type":"not_found_error"}} diff --git a/core/translate/testdata/reply/error_404_bob.json.to-json.golden b/core/translate/testdata/reply/error_404_bob.json.to-json.golden new file mode 100644 index 000000000..df92bd20c --- /dev/null +++ b/core/translate/testdata/reply/error_404_bob.json.to-json.golden @@ -0,0 +1 @@ +{"type":"error","error":{"type":"not_found_error","message":"model \"claude-opus-5-5\" not found or not accessible"}} \ No newline at end of file diff --git a/core/translate/testdata/reply/error_404_bob.json.to-sse.golden b/core/translate/testdata/reply/error_404_bob.json.to-sse.golden new file mode 100644 index 000000000..214548a45 --- /dev/null +++ b/core/translate/testdata/reply/error_404_bob.json.to-sse.golden @@ -0,0 +1,3 @@ +event: error +data: {"type":"error","error":{"type":"not_found_error","message":"model \"claude-opus-5-5\" not found or not accessible"}} + diff --git a/core/translate/testdata/reply/error_429_openai.json b/core/translate/testdata/reply/error_429_openai.json new file mode 100644 index 000000000..4ffa90a84 --- /dev/null +++ b/core/translate/testdata/reply/error_429_openai.json @@ -0,0 +1 @@ +{"error":{"message":"Rate limit reached for requests. Try again in 20s.","type":"rate_limit_exceeded","param":null,"code":"rate_limit_exceeded"}} diff --git a/core/translate/testdata/reply/error_429_openai.json.to-json.golden b/core/translate/testdata/reply/error_429_openai.json.to-json.golden new file mode 100644 index 000000000..3193c6b83 --- /dev/null +++ b/core/translate/testdata/reply/error_429_openai.json.to-json.golden @@ -0,0 +1 @@ +{"type":"error","error":{"type":"rate_limit_error","message":"Rate limit reached for requests. Try again in 20s."}} \ No newline at end of file diff --git a/core/translate/testdata/reply/error_429_openai.json.to-sse.golden b/core/translate/testdata/reply/error_429_openai.json.to-sse.golden new file mode 100644 index 000000000..84ab97deb --- /dev/null +++ b/core/translate/testdata/reply/error_429_openai.json.to-sse.golden @@ -0,0 +1,3 @@ +event: error +data: {"type":"error","error":{"type":"rate_limit_error","message":"Rate limit reached for requests. Try again in 20s."}} + diff --git a/core/translate/testdata/reply/stream_two_tools.sse b/core/translate/testdata/reply/stream_two_tools.sse new file mode 100644 index 000000000..242f7fc65 --- /dev/null +++ b/core/translate/testdata/reply/stream_two_tools.sse @@ -0,0 +1,15 @@ +data: {"id":"chatcmpl-t2","model":"gpt-x","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":"Two reads."},"finish_reason":null}]} + +data: {"id":"chatcmpl-t2","model":"gpt-x","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_a","type":"function","function":{"name":"Read","arguments":""}}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-t2","model":"gpt-x","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"path\":"}}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-t2","model":"gpt-x","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"a.go\"}"}}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-t2","model":"gpt-x","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"id":"call_b","type":"function","function":{"name":"Read","arguments":"{\"path\":"}}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-t2","model":"gpt-x","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"arguments":"\"b.go\"}"}}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-t2","model":"gpt-x","object":"chat.completion.chunk","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"completion_tokens":30,"total_tokens":130,"prompt_tokens_details":{"cached_tokens":64}}} + +data: [DONE] diff --git a/core/translate/testdata/reply/stream_two_tools.sse.to-sse.golden b/core/translate/testdata/reply/stream_two_tools.sse.to-sse.golden new file mode 100644 index 000000000..f609f0f04 --- /dev/null +++ b/core/translate/testdata/reply/stream_two_tools.sse.to-sse.golden @@ -0,0 +1,42 @@ +event: message_start +data: {"type":"message_start","message":{"id":"chatcmpl-t2","type":"message","role":"assistant","model":"gpt-x","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Two reads."}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: content_block_start +data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"call_a","name":"Read","input":{}}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"path\":"}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"\"a.go\"}"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":1} + +event: content_block_start +data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"call_b","name":"Read","input":{}}} + +event: content_block_delta +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\"path\":"}} + +event: content_block_delta +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"\"b.go\"}"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":2} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"input_tokens":36,"cache_creation_input_tokens":0,"cache_read_input_tokens":64,"output_tokens":30}} + +event: message_stop +data: {"type":"message_stop"} + diff --git a/core/translate/testdata/reply/two_tools.json b/core/translate/testdata/reply/two_tools.json new file mode 100644 index 000000000..775dc5416 --- /dev/null +++ b/core/translate/testdata/reply/two_tools.json @@ -0,0 +1,6 @@ +{"id":"chatcmpl-t2","model":"gpt-x","object":"chat.completion", + "choices":[{"index":0,"finish_reason":"tool_calls","message":{"role":"assistant","content":"Two reads.", + "tool_calls":[ + {"id":"call_a","type":"function","function":{"name":"Read","arguments":"{\"path\":\"a.go\"}"}}, + {"id":"call_b","type":"function","function":{"name":"Read","arguments":"{\"path\":\"b.go\"}"}}]}}], + "usage":{"prompt_tokens":100,"completion_tokens":30,"total_tokens":130,"prompt_tokens_details":{"cached_tokens":64}}} diff --git a/core/translate/testdata/reply/two_tools.json.to-json.golden b/core/translate/testdata/reply/two_tools.json.to-json.golden new file mode 100644 index 000000000..5a9897824 --- /dev/null +++ b/core/translate/testdata/reply/two_tools.json.to-json.golden @@ -0,0 +1 @@ +{"id":"chatcmpl-t2","type":"message","role":"assistant","model":"gpt-x","content":[{"type":"text","text":"Two reads."},{"type":"tool_use","id":"call_a","name":"Read","input":{"path":"a.go"}},{"type":"tool_use","id":"call_b","name":"Read","input":{"path":"b.go"}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":36,"cache_creation_input_tokens":0,"cache_read_input_tokens":64,"output_tokens":30}} \ No newline at end of file diff --git a/core/translate/testdata/request/full_multiturn.golden b/core/translate/testdata/request/full_multiturn.golden new file mode 100644 index 000000000..a164916e7 --- /dev/null +++ b/core/translate/testdata/request/full_multiturn.golden @@ -0,0 +1,2 @@ +dropped: metadata +{"model":"claude-opus-5-5","messages":[{"role":"system","content":[{"type":"text","text":"You are Claude Code."},{"type":"text","text":"Follow the project rules.","cache_control":{"type":"ephemeral"}}]},{"role":"user","content":[{"type":"text","text":"What is in this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgo="}}]},{"role":"assistant","content":"Let me check the file.","thinking_blocks":[{"type":"thinking","thinking":"Look at it first.","signature":"EvkBsig1"}],"tool_calls":[{"id":"toolu_01","type":"function","function":{"name":"Bash","arguments":"{\"command\":\"file cat.png\"}"}}]},{"role":"tool","content":[{"type":"text","text":"cat.png: PNG image data"}],"tool_call_id":"toolu_01"},{"role":"user","content":[{"type":"text","text":"The user changed cat.png."}]},{"role":"assistant","content":"It is a PNG image of a cat."},{"role":"user","content":[{"type":"text","text":"Thanks. Is 391 prime?","cache_control":{"type":"ephemeral"}}]}],"max_tokens":32000,"stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"Bash","description":"Run a shell command.","parameters":{"type":"object","properties":{"command":{"type":"string"}},"required":["command"]}}}],"tool_choice":"auto","reasoning_effort":"high"} diff --git a/core/translate/testdata/request/full_multiturn.json b/core/translate/testdata/request/full_multiturn.json new file mode 100644 index 000000000..7dcdc1603 --- /dev/null +++ b/core/translate/testdata/request/full_multiturn.json @@ -0,0 +1,14 @@ +{"model":"claude-opus-5-5","max_tokens":32000,"stream":true, + "system":[{"type":"text","text":"You are Claude Code."},{"type":"text","text":"Follow the project rules.","cache_control":{"type":"ephemeral"}}], + "tools":[{"name":"Bash","description":"Run a shell command.","input_schema":{"type":"object","properties":{"command":{"type":"string"}},"required":["command"]}}], + "tool_choice":{"type":"auto"}, + "thinking":{"type":"adaptive"}, + "output_config":{"effort":"high"}, + "metadata":{"user_id":"u1"}, + "messages":[ + {"role":"user","content":[{"type":"text","text":"What is in this image?"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"iVBORw0KGgo="}}]}, + {"role":"assistant","content":[{"type":"thinking","thinking":"Look at it first.","signature":"EvkBsig1"},{"type":"text","text":"Let me check the file."},{"type":"tool_use","id":"toolu_01","name":"Bash","input":{"command":"file cat.png"}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_01","content":"cat.png: PNG image data"}]}, + {"role":"system","content":"The user changed cat.png."}, + {"role":"assistant","content":[{"type":"text","text":"It is a PNG image of a cat."}]}, + {"role":"user","content":[{"type":"text","text":"Thanks. Is 391 prime?","cache_control":{"type":"ephemeral"}}]}]} diff --git a/core/translate/testdata/request/rule11_unknown_fields.golden b/core/translate/testdata/request/rule11_unknown_fields.golden new file mode 100644 index 000000000..961899bd6 --- /dev/null +++ b/core/translate/testdata/request/rule11_unknown_fields.golden @@ -0,0 +1,2 @@ +dropped: context_management,metadata,prompt_cache_key,service_tier +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule11_unknown_fields.json b/core/translate/testdata/request/rule11_unknown_fields.json new file mode 100644 index 000000000..4f41b1ac7 --- /dev/null +++ b/core/translate/testdata/request/rule11_unknown_fields.json @@ -0,0 +1,2 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"metadata":{"user_id":"u1"},"prompt_cache_key":"k","service_tier":"auto","context_management":{"edits":[]}, + "messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule12_stream.golden b/core/translate/testdata/request/rule12_stream.golden new file mode 100644 index 000000000..e892734b7 --- /dev/null +++ b/core/translate/testdata/request/rule12_stream.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024,"stream":true,"stream_options":{"include_usage":true}} diff --git a/core/translate/testdata/request/rule12_stream.json b/core/translate/testdata/request/rule12_stream.json new file mode 100644 index 000000000..72ab14dad --- /dev/null +++ b/core/translate/testdata/request/rule12_stream.json @@ -0,0 +1 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"stream":true,"messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule13_system_blocks.golden b/core/translate/testdata/request/rule13_system_blocks.golden new file mode 100644 index 000000000..6297b355e --- /dev/null +++ b/core/translate/testdata/request/rule13_system_blocks.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"system","content":[{"type":"text","text":"You are Claude Code."},{"type":"text","text":"Long instructions.","cache_control":{"type":"ephemeral","ttl":"1h"}}]},{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule13_system_blocks.json b/core/translate/testdata/request/rule13_system_blocks.json new file mode 100644 index 000000000..7ff0d0bb0 --- /dev/null +++ b/core/translate/testdata/request/rule13_system_blocks.json @@ -0,0 +1,3 @@ +{"model":"claude-opus-5-5","max_tokens":1024, + "system":[{"type":"text","text":"You are Claude Code."},{"type":"text","text":"Long instructions.","cache_control":{"type":"ephemeral","ttl":"1h"}}], + "messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule13_system_nontext.golden b/core/translate/testdata/request/rule13_system_nontext.golden new file mode 100644 index 000000000..522561982 --- /dev/null +++ b/core/translate/testdata/request/rule13_system_nontext.golden @@ -0,0 +1,2 @@ +dropped: content,image,search_result,source,title +{"model":"claude-opus-5-5","messages":[{"role":"system","content":[{"type":"text","text":"hi"}]},{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule13_system_nontext.json b/core/translate/testdata/request/rule13_system_nontext.json new file mode 100644 index 000000000..7ca1bca41 --- /dev/null +++ b/core/translate/testdata/request/rule13_system_nontext.json @@ -0,0 +1,3 @@ +{"model":"claude-opus-5-5","max_tokens":1024, + "system":[{"type":"text","text":"hi"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"iVBORw0KGgo="}},{"type":"search_result","source":"https://example.com","title":"t","content":[{"type":"text","text":"r"}]}], + "messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule13_system_only_nontext.golden b/core/translate/testdata/request/rule13_system_only_nontext.golden new file mode 100644 index 000000000..db122278a --- /dev/null +++ b/core/translate/testdata/request/rule13_system_only_nontext.golden @@ -0,0 +1,2 @@ +dropped: document +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule13_system_only_nontext.json b/core/translate/testdata/request/rule13_system_only_nontext.json new file mode 100644 index 000000000..c597a24ad --- /dev/null +++ b/core/translate/testdata/request/rule13_system_only_nontext.json @@ -0,0 +1,3 @@ +{"model":"claude-opus-5-5","max_tokens":1024, + "system":[{"type":"document","source":{"type":"text","media_type":"text/plain","data":"doc"}}], + "messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule13_system_string.golden b/core/translate/testdata/request/rule13_system_string.golden new file mode 100644 index 000000000..e7f7b0888 --- /dev/null +++ b/core/translate/testdata/request/rule13_system_string.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"system","content":[{"type":"text","text":"You are a helpful agent."}]},{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule13_system_string.json b/core/translate/testdata/request/rule13_system_string.json new file mode 100644 index 000000000..6e2572d45 --- /dev/null +++ b/core/translate/testdata/request/rule13_system_string.json @@ -0,0 +1 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"system":"You are a helpful agent.","messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule13a_mid_system.golden b/core/translate/testdata/request/rule13a_mid_system.golden new file mode 100644 index 000000000..ff5e26865 --- /dev/null +++ b/core/translate/testdata/request/rule13a_mid_system.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"system","content":[{"type":"text","text":"sys"}]},{"role":"user","content":[{"type":"text","text":"hi"}]},{"role":"assistant","content":"hello"},{"role":"user","content":[{"type":"text","text":"hook output"}]},{"role":"user","content":[{"type":"text","text":"go on"}]}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule13a_mid_system.json b/core/translate/testdata/request/rule13a_mid_system.json new file mode 100644 index 000000000..f15b89c66 --- /dev/null +++ b/core/translate/testdata/request/rule13a_mid_system.json @@ -0,0 +1,6 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"system":"sys", + "messages":[ + {"role":"user","content":"hi"}, + {"role":"assistant","content":[{"type":"text","text":"hello"}]}, + {"role":"system","content":"hook output"}, + {"role":"user","content":"go on"}]} diff --git a/core/translate/testdata/request/rule14_document.golden b/core/translate/testdata/request/rule14_document.golden new file mode 100644 index 000000000..14499cbfe --- /dev/null +++ b/core/translate/testdata/request/rule14_document.golden @@ -0,0 +1 @@ +error: 400 invalid_request_error: this inference server cannot read documents (PDF) diff --git a/core/translate/testdata/request/rule14_document.json b/core/translate/testdata/request/rule14_document.json new file mode 100644 index 000000000..232289dc7 --- /dev/null +++ b/core/translate/testdata/request/rule14_document.json @@ -0,0 +1,2 @@ +{"model":"claude-opus-5-5","max_tokens":1024, + "messages":[{"role":"user","content":[{"type":"document","source":{"type":"base64","media_type":"application/pdf","data":"JVBERi0="}},{"type":"text","text":"Summarize."}]}]} diff --git a/core/translate/testdata/request/rule14_image.golden b/core/translate/testdata/request/rule14_image.golden new file mode 100644 index 000000000..cf56510b0 --- /dev/null +++ b/core/translate/testdata/request/rule14_image.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgo="}},{"type":"image_url","image_url":{"url":"https://example.com/cat.jpg"}},{"type":"text","text":"What is in these images?"}]}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule14_image.json b/core/translate/testdata/request/rule14_image.json new file mode 100644 index 000000000..3010d3c84 --- /dev/null +++ b/core/translate/testdata/request/rule14_image.json @@ -0,0 +1,5 @@ +{"model":"claude-opus-5-5","max_tokens":1024, + "messages":[{"role":"user","content":[ + {"type":"image","source":{"type":"base64","media_type":"image/png","data":"iVBORw0KGgo="}}, + {"type":"image","source":{"type":"url","url":"https://example.com/cat.jpg"}}, + {"type":"text","text":"What is in these images?"}]}]} diff --git a/core/translate/testdata/request/rule14_text.golden b/core/translate/testdata/request/rule14_text.golden new file mode 100644 index 000000000..6b719ffbb --- /dev/null +++ b/core/translate/testdata/request/rule14_text.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"one"},{"type":"text","text":"two ","cache_control":{"type":"ephemeral"}}]}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule14_text.json b/core/translate/testdata/request/rule14_text.json new file mode 100644 index 000000000..77f325ab1 --- /dev/null +++ b/core/translate/testdata/request/rule14_text.json @@ -0,0 +1,2 @@ +{"model":"claude-opus-5-5","max_tokens":1024, + "messages":[{"role":"user","content":[{"type":"text","text":"one"},{"type":"text","text":"two ","cache_control":{"type":"ephemeral"}}]}]} diff --git a/core/translate/testdata/request/rule14_tool_result.golden b/core/translate/testdata/request/rule14_tool_result.golden new file mode 100644 index 000000000..c3a78d129 --- /dev/null +++ b/core/translate/testdata/request/rule14_tool_result.golden @@ -0,0 +1,2 @@ +dropped: is_error +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"list and screenshot"}]},{"role":"assistant","tool_calls":[{"id":"toolu_01","type":"function","function":{"name":"Bash","arguments":"{\"command\":\"ls\"}"}},{"id":"toolu_02","type":"function","function":{"name":"Screenshot","arguments":"{}"}},{"id":"toolu_03","type":"function","function":{"name":"Bash","arguments":"{\"command\":\"false\"}"}}]},{"role":"tool","content":[{"type":"text","text":"a.txt\nb.txt"}],"tool_call_id":"toolu_01"},{"role":"tool","content":[{"type":"text","text":"the screen"}],"tool_call_id":"toolu_02"},{"role":"tool","content":[{"type":"text","text":"exit 1","cache_control":{"type":"ephemeral"}}],"tool_call_id":"toolu_03"},{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,AAAA"}}]},{"role":"user","content":[{"type":"text","text":"before the results"}]}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule14_tool_result.json b/core/translate/testdata/request/rule14_tool_result.json new file mode 100644 index 000000000..a4b8fd65a --- /dev/null +++ b/core/translate/testdata/request/rule14_tool_result.json @@ -0,0 +1,12 @@ +{"model":"claude-opus-5-5","max_tokens":1024, + "messages":[ + {"role":"user","content":"list and screenshot"}, + {"role":"assistant","content":[ + {"type":"tool_use","id":"toolu_01","name":"Bash","input":{"command":"ls"}}, + {"type":"tool_use","id":"toolu_02","name":"Screenshot","input":{}}, + {"type":"tool_use","id":"toolu_03","name":"Bash","input":{"command":"false"}}]}, + {"role":"user","content":[ + {"type":"text","text":"before the results"}, + {"type":"tool_result","tool_use_id":"toolu_01","content":"a.txt\nb.txt"}, + {"type":"tool_result","tool_use_id":"toolu_02","content":[{"type":"text","text":"the screen"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"AAAA"}}]}, + {"type":"tool_result","tool_use_id":"toolu_03","is_error":true,"content":[{"type":"text","text":"exit 1"}],"cache_control":{"type":"ephemeral"}}]}]} diff --git a/core/translate/testdata/request/rule15_assistant.golden b/core/translate/testdata/request/rule15_assistant.golden new file mode 100644 index 000000000..7a2052723 --- /dev/null +++ b/core/translate/testdata/request/rule15_assistant.golden @@ -0,0 +1,2 @@ +dropped: thinking +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"Is 391 prime?"}]},{"role":"assistant","content":"391 is not prime.","thinking_blocks":[{"type":"thinking","thinking":"391 = 17 x 23.","signature":"EvkBsig"},{"type":"redacted_thinking","data":"RDdata"}],"tool_calls":[{"id":"tooluse_9nf","type":"function","function":{"name":"Bash","arguments":"{\"command\":\"echo done\"}"}}]},{"role":"assistant","tool_calls":[{"id":"tooluse_x","type":"function","function":{"name":"Read","arguments":"{}"}}]},{"role":"tool","content":[{"type":"text","text":"done"}],"tool_call_id":"tooluse_9nf"},{"role":"tool","content":[{"type":"text","text":"ok"}],"tool_call_id":"tooluse_x"}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule15_assistant.json b/core/translate/testdata/request/rule15_assistant.json new file mode 100644 index 000000000..b96ab620f --- /dev/null +++ b/core/translate/testdata/request/rule15_assistant.json @@ -0,0 +1,12 @@ +{"model":"claude-opus-5-5","max_tokens":1024, + "messages":[ + {"role":"user","content":"Is 391 prime?"}, + {"role":"assistant","content":[ + {"type":"thinking","thinking":"391 = 17 x 23.","signature":"EvkBsig"}, + {"type":"redacted_thinking","data":"RDdata"}, + {"type":"thinking","thinking":"no signature"}, + {"type":"text","text":"391 is "}, + {"type":"text","text":"not prime."}, + {"type":"tool_use","id":"tooluse_9nf","name":"Bash","input":{"command":"echo done"}}]}, + {"role":"assistant","content":[{"type":"tool_use","id":"tooluse_x","name":"Read","input":{}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"tooluse_9nf","content":"done"},{"type":"tool_result","tool_use_id":"tooluse_x","content":"ok"}]}]} diff --git a/core/translate/testdata/request/rule16_server_tool.golden b/core/translate/testdata/request/rule16_server_tool.golden new file mode 100644 index 000000000..191b8744c --- /dev/null +++ b/core/translate/testdata/request/rule16_server_tool.golden @@ -0,0 +1 @@ +error: 400 invalid_request_error: this inference server cannot run the server tool "web_search" (type web_search_20250305) diff --git a/core/translate/testdata/request/rule16_server_tool.json b/core/translate/testdata/request/rule16_server_tool.json new file mode 100644 index 000000000..3f8ea2fac --- /dev/null +++ b/core/translate/testdata/request/rule16_server_tool.json @@ -0,0 +1,3 @@ +{"model":"claude-opus-5-5","max_tokens":1024, + "tools":[{"name":"Bash","description":"Run.","input_schema":{"type":"object"}},{"type":"web_search_20250305","name":"web_search","max_uses":5}], + "messages":[{"role":"user","content":"search"}]} diff --git a/core/translate/testdata/request/rule16_tools.golden b/core/translate/testdata/request/rule16_tools.golden new file mode 100644 index 000000000..66f7d6dc5 --- /dev/null +++ b/core/translate/testdata/request/rule16_tools.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024,"tools":[{"type":"function","function":{"name":"Bash","description":"Run a shell command.","parameters":{"type":"object","properties":{"command":{"type":"string"}},"required":["command"]}}},{"type":"function","function":{"name":"Read","description":"Read a file.","parameters":{"type":"object","properties":{"path":{"type":"string"}}}}}]} diff --git a/core/translate/testdata/request/rule16_tools.json b/core/translate/testdata/request/rule16_tools.json new file mode 100644 index 000000000..88e26e8e5 --- /dev/null +++ b/core/translate/testdata/request/rule16_tools.json @@ -0,0 +1,5 @@ +{"model":"claude-opus-5-5","max_tokens":1024, + "tools":[ + {"name":"Bash","description":"Run a shell command.","input_schema":{"type":"object","properties":{"command":{"type":"string"}},"required":["command"]}}, + {"name":"Read","description":"Read a file.","input_schema":{"type":"object","properties":{"path":{"type":"string"}}},"cache_control":{"type":"ephemeral"}}], + "messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule17_any_no_parallel.golden b/core/translate/testdata/request/rule17_any_no_parallel.golden new file mode 100644 index 000000000..84821de7e --- /dev/null +++ b/core/translate/testdata/request/rule17_any_no_parallel.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024,"tools":[{"type":"function","function":{"name":"Bash","parameters":{"type":"object"}}}],"tool_choice":"required","parallel_tool_calls":false} diff --git a/core/translate/testdata/request/rule17_any_no_parallel.json b/core/translate/testdata/request/rule17_any_no_parallel.json new file mode 100644 index 000000000..25fbf6f2c --- /dev/null +++ b/core/translate/testdata/request/rule17_any_no_parallel.json @@ -0,0 +1 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"tools":[{"name":"Bash","input_schema":{"type":"object"}}],"tool_choice":{"type":"any","disable_parallel_tool_use":true},"messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule17_auto.golden b/core/translate/testdata/request/rule17_auto.golden new file mode 100644 index 000000000..bce41128a --- /dev/null +++ b/core/translate/testdata/request/rule17_auto.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024,"tools":[{"type":"function","function":{"name":"Bash","parameters":{"type":"object"}}}],"tool_choice":"auto"} diff --git a/core/translate/testdata/request/rule17_auto.json b/core/translate/testdata/request/rule17_auto.json new file mode 100644 index 000000000..8bdc11554 --- /dev/null +++ b/core/translate/testdata/request/rule17_auto.json @@ -0,0 +1 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"tools":[{"name":"Bash","input_schema":{"type":"object"}}],"tool_choice":{"type":"auto"},"messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule17_none.golden b/core/translate/testdata/request/rule17_none.golden new file mode 100644 index 000000000..00b1d239a --- /dev/null +++ b/core/translate/testdata/request/rule17_none.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024,"tools":[{"type":"function","function":{"name":"Bash","parameters":{"type":"object"}}}],"tool_choice":"none"} diff --git a/core/translate/testdata/request/rule17_none.json b/core/translate/testdata/request/rule17_none.json new file mode 100644 index 000000000..b2094fc7a --- /dev/null +++ b/core/translate/testdata/request/rule17_none.json @@ -0,0 +1 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"tools":[{"name":"Bash","input_schema":{"type":"object"}}],"tool_choice":{"type":"none"},"messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule17_tool.golden b/core/translate/testdata/request/rule17_tool.golden new file mode 100644 index 000000000..c7240eca5 --- /dev/null +++ b/core/translate/testdata/request/rule17_tool.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024,"tools":[{"type":"function","function":{"name":"Bash","parameters":{"type":"object"}}}],"tool_choice":{"type":"function","function":{"name":"Bash"}}} diff --git a/core/translate/testdata/request/rule17_tool.json b/core/translate/testdata/request/rule17_tool.json new file mode 100644 index 000000000..45f18672a --- /dev/null +++ b/core/translate/testdata/request/rule17_tool.json @@ -0,0 +1 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"tools":[{"name":"Bash","input_schema":{"type":"object"}}],"tool_choice":{"type":"tool","name":"Bash"},"messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule18_sampling.golden b/core/translate/testdata/request/rule18_sampling.golden new file mode 100644 index 000000000..d6eb3a384 --- /dev/null +++ b/core/translate/testdata/request/rule18_sampling.golden @@ -0,0 +1,2 @@ +dropped: metadata,top_k +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024,"stop":["END","\n\nHuman:"],"temperature":0.7,"top_p":0.95} diff --git a/core/translate/testdata/request/rule18_sampling.json b/core/translate/testdata/request/rule18_sampling.json new file mode 100644 index 000000000..00398bee4 --- /dev/null +++ b/core/translate/testdata/request/rule18_sampling.json @@ -0,0 +1,2 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"stop_sequences":["END","\n\nHuman:"],"temperature":0.7,"top_p":0.95,"top_k":40,"metadata":{"user_id":"u1"}, + "messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule19_effort_low.golden b/core/translate/testdata/request/rule19_effort_low.golden new file mode 100644 index 000000000..0f630f4f1 --- /dev/null +++ b/core/translate/testdata/request/rule19_effort_low.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024,"reasoning_effort":"low"} diff --git a/core/translate/testdata/request/rule19_effort_low.json b/core/translate/testdata/request/rule19_effort_low.json new file mode 100644 index 000000000..017653a72 --- /dev/null +++ b/core/translate/testdata/request/rule19_effort_low.json @@ -0,0 +1 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"output_config":{"effort":"low"},"messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule19_effort_xhigh.golden b/core/translate/testdata/request/rule19_effort_xhigh.golden new file mode 100644 index 000000000..c5f5b4815 --- /dev/null +++ b/core/translate/testdata/request/rule19_effort_xhigh.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024,"reasoning_effort":"high"} diff --git a/core/translate/testdata/request/rule19_effort_xhigh.json b/core/translate/testdata/request/rule19_effort_xhigh.json new file mode 100644 index 000000000..bf3e5817c --- /dev/null +++ b/core/translate/testdata/request/rule19_effort_xhigh.json @@ -0,0 +1 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"output_config":{"effort":"xhigh"},"thinking":{"type":"adaptive"},"messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule19_thinking_adaptive.golden b/core/translate/testdata/request/rule19_thinking_adaptive.golden new file mode 100644 index 000000000..d83c7d97c --- /dev/null +++ b/core/translate/testdata/request/rule19_thinking_adaptive.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":16000,"reasoning_effort":"medium"} diff --git a/core/translate/testdata/request/rule19_thinking_adaptive.json b/core/translate/testdata/request/rule19_thinking_adaptive.json new file mode 100644 index 000000000..625e717a1 --- /dev/null +++ b/core/translate/testdata/request/rule19_thinking_adaptive.json @@ -0,0 +1 @@ +{"model":"claude-opus-5-5","max_tokens":16000,"thinking":{"type":"adaptive"},"messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule19_thinking_disabled.golden b/core/translate/testdata/request/rule19_thinking_disabled.golden new file mode 100644 index 000000000..ef3ad2952 --- /dev/null +++ b/core/translate/testdata/request/rule19_thinking_disabled.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule19_thinking_disabled.json b/core/translate/testdata/request/rule19_thinking_disabled.json new file mode 100644 index 000000000..40d144e1e --- /dev/null +++ b/core/translate/testdata/request/rule19_thinking_disabled.json @@ -0,0 +1 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"thinking":{"type":"disabled"},"messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule19_thinking_enabled.golden b/core/translate/testdata/request/rule19_thinking_enabled.golden new file mode 100644 index 000000000..d83c7d97c --- /dev/null +++ b/core/translate/testdata/request/rule19_thinking_enabled.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":16000,"reasoning_effort":"medium"} diff --git a/core/translate/testdata/request/rule19_thinking_enabled.json b/core/translate/testdata/request/rule19_thinking_enabled.json new file mode 100644 index 000000000..ba67e5857 --- /dev/null +++ b/core/translate/testdata/request/rule19_thinking_enabled.json @@ -0,0 +1 @@ +{"model":"claude-opus-5-5","max_tokens":16000,"thinking":{"type":"enabled","budget_tokens":8000},"messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule20a_thought_id.golden b/core/translate/testdata/request/rule20a_thought_id.golden new file mode 100644 index 000000000..e3de589ca --- /dev/null +++ b/core/translate/testdata/request/rule20a_thought_id.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"premium-ide","messages":[{"role":"user","content":[{"type":"text","text":"spawn a helper"}]},{"role":"assistant","tool_calls":[{"id":"call_3833__thought__AY89a1+8zd/x=","type":"function","function":{"name":"spawn_subagent","arguments":"{\"task\":\"look\"}"}}]},{"role":"tool","content":[{"type":"text","text":"done"}],"tool_call_id":"call_3833__thought__AY89a1+8zd/x="}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule20a_thought_id.json b/core/translate/testdata/request/rule20a_thought_id.json new file mode 100644 index 000000000..6fe2608e1 --- /dev/null +++ b/core/translate/testdata/request/rule20a_thought_id.json @@ -0,0 +1,5 @@ +{"model":"premium-ide","max_tokens":1024, + "messages":[ + {"role":"user","content":"spawn a helper"}, + {"role":"assistant","content":[{"type":"tool_use","id":"call_3833__thought__AY89a1+8zd/x=","name":"spawn_subagent","input":{"task":"look"}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"call_3833__thought__AY89a1+8zd/x=","content":"done"}]}]} diff --git a/docs/README.md b/docs/README.md index 81ff42f4e..c01baa8d1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -15,6 +15,7 @@ holds the repo-level pieces; this file is the index. | Enforce policy on agent traffic in praxis mode | [`praxis-policy.md`](praxis-policy.md) | | Customize Cortex with a small Python or shell script | [`script-plugin.md`](script-plugin.md) | | Run an agent on a local model, or switch inference servers | [`laptop-local-model.md`](laptop-local-model.md) | +| Use a server whose protocol Cortex cannot translate to | [`translation-server.md`](translation-server.md) | | Use the `agentop` TUI | [`cmd/agentop/README.md`](../cmd/agentop/README.md) | | Add your agent's sessions from before Cortex | [`agentop import`](../cmd/agentop/README.md#importing-agent-history-agentop-import) | | Export sessions to files | [`session-dump.md`](session-dump.md) | diff --git a/docs/inference-router-plugin.md b/docs/inference-router-plugin.md index 89a8fc1c0..bea3e5e62 100644 --- a/docs/inference-router-plugin.md +++ b/docs/inference-router-plugin.md @@ -14,9 +14,12 @@ at startup, and in-cluster use has not been examined. - `servers` (map, required) — inference servers by name: lowercase letters, digits, `.`, `_` and `-`. Each has: - `url` (string, required) — `scheme://host[:port]`, `http` or `https`, no path, query or fragment. A port must be 1–65535; the scheme's default is dropped, so `https://x:443` is `x`. No two servers may share a host, compared without case and, except on a loopback host such as `localhost`, without port, since one machine serves many things there: a session's server is named from the host its requests went to. A plain-`http` server on another machine logs a WARN at load, since routed requests would cross the network decrypted. - - `key` (string, required) — the API key sent to this server in place of the client's, in the header the client used: `X-Api-Key` when it sent one, `Authorization: Bearer` otherwise, both when it sent both. Printable ASCII, no spaces. `/config` and `/v1/pipeline` redact it. + - `key` (string, required) — the API key sent to this server in place of the client's, in the header the client used: `X-Api-Key` when it sent one, `Authorization: Bearer` otherwise, both when it sent both. `auth_header` and a translated request change this (see **Servers that serve another protocol**). Printable ASCII, no spaces. `/config` and `/v1/pipeline` redact it. - `main` (string, required) — a word naming this server's model for an agent's main work, a request that carries tools, sent in place of a model the server refuses (see **Models**). It resolves to the newest model on the server's list whose name, without its provider prefix, contains the word, case ignored: `opus` against `aws/claude-opus-5-5` and `claude-opus-4-8` is the first. No `/` and no whitespace. The fields `opus`, `sonnet` and `haiku` it replaced are refused by name. - `helper` (string, required) — the same for a request without tools, such as a title. + - `api` (map) — protocols the server serves, by protocol name, to the path it serves each at. Empty: requests go in the agent's protocol, to the agent's path. Valid keys: `anthropic-messages`, `chat-completions` and `responses`. Each path must be absolute, with no query, fragment or `..` segment. See **Servers that serve another protocol**. + - `auth_header` (string) — header that carries `key`. `authorization` sends `Bearer `; any other name sends the raw key. Empty: the agent's header, or the target protocol's usual one for a translated request. + - `headers` (map) — fixed headers set on every request to this server after the plugins run. Values may use `${ENV_VAR}`. - `agents` (map) — agent name, as agentop shows it (`claude-code`, `opencode`), to a server name. Each value must name a listed server. `unknown`, the name for requests with no User-Agent, cannot be routed. - `sessions` (map) — session id to a server name, moving one running session to that server. Each value must name a listed server, or be `""` to stop routing that session. An entry beats the session's pin and every rule below, and the pin follows it, so removing the entry does not move the session back; `agentop server move` and the TUI's `S` on a session row write these; a `server move` from the shell also prunes an entry whose session is neither live nor archived. A synthetic id (`default`, `pending:`) is refused: it holds many conversations, not one to move. @@ -254,3 +257,85 @@ server's host, and each event's `requestedHost` keeps the one asked for when it differs. The router also rewrites the request body to send a substitute, so the framework holds it after every body reader: a chain with a parser after the router is refused, at startup and on reload. + +## Servers that serve another protocol + +A server's `api` map says which protocols it serves, and at which path. DeepSeek +serves two, at different paths: + +```yaml +servers: + deepseek: + url: https://api.deepseek.com + key: sk-… + main: chat + helper: chat + api: + chat-completions: /chat/completions + anthropic-messages: /anthropic/v1/messages +``` + +The router uses the agent's protocol, from the request path, and these rules in +this order: + +1. The map is empty: the request goes in the agent's protocol, to the agent's path, + as before. +2. The request is `/v1/messages/count_tokens`, and the map has no + `anthropic-messages`: the proxy answers a 404 Anthropic `not_found_error` and + sends nothing upstream (`deny/count_tokens_local`). This applies only to a + `count_tokens` request that is routed to this server. The router does not + capture `count_tokens`, so when the agent sends it to another host, it goes there. +3. The agent's protocol is a key of the map: the request goes to that key's path, + with no translation. Claude Code's `/v1/messages` goes to DeepSeek's + `/anthropic/v1/messages`. +4. Cortex can read the agent's protocol (only `anthropic-messages` at this time): + Cortex translates the request. The target is the first of `chat-completions`, + `anthropic-messages`, `responses` that is in the map and that Cortex can write + (only `chat-completions` at this time). The request goes to the target's path, + with no query string. If no protocol in the map is a target, Cortex sends + nothing and answers a 502 Anthropic `api_error` + (`deny/cannot_translate`): "Cortex cannot translate anthropic-messages to + \ yet. Use a translation server for this pair". The message names each + map key, sorted, joined with " or ". For that pair, see + [`translation-server.md`](translation-server.md). +5. Otherwise (a chat-completions or Responses request): the request goes as it is, + with no translation and no 502. + +**A translated request.** Every plugin, the session store, the timeline and the +cost ledger see only the Anthropic side: the request as the agent sent it, and the +reply as the agent gets it. The proxy converts the request after the last request +plugin, and it converts the reply before a reply plugin reads it. Streams stay +incremental. The request row carries the framework's `modify/translated` record, +with `from`, `to`, `path` (the path sent upstream), and `dropped` when the +conversion removed fields (for example `metadata,top_k`). A request that the target +cannot carry, such as one with a PDF document, gets a 400 `invalid_request_error` +(`deny/translation_refused`), and nothing goes upstream. The raw capture keeps the +translated request and the server's reply before translation. + +**The auth header.** The sent request has only the header that this table picks. +The router removes the agent's `Authorization`, `X-Api-Key` and `Api-Key` first, +except in the first row: + +| `auth_header` | Request | Header sent | +|---|---|---| +| empty | not translated | as before: the header the agent used (see `key`) | +| empty | translated to `chat-completions` | `Authorization: Bearer ` | +| empty | translated to `anthropic-messages` | `X-Api-Key: ` | +| `authorization` | each request | `Authorization: Bearer ` | +| any other name | each request | that header, with the raw key | + +For example, Azure uses `api-key`. Bob accepts its key as `x-api-key`, and refuses +it as Bearer. + +**Fixed headers.** `headers` values replace the agent's header of the same name, +after the plugins run, on translated and not translated requests. The config load +refuses these names: `Host`, `Content-Length`, the hop-by-hop headers +(`Connection`, `Keep-Alive`, `Proxy-Connection`, `Transfer-Encoding`, `Upgrade`, +`TE`, `Trailer`, `Proxy-Authenticate`, `Proxy-Authorization`), the auth headers +(`Authorization`, `X-Api-Key`, `Api-Key`) and the server's own `auth_header`. It +also refuses a value with CR, LF or NUL. `/config` and `/v1/pipeline` show the +header names and hide the values, as they hide `key`. Bob needs +`User-Agent: bob-shell`. + +`auth_header` and `headers` apply only to the requests that the router routes. The +model-list fetch (`GET /v1/models`) still sends `Authorization: Bearer ` only. diff --git a/docs/session-api.md b/docs/session-api.md index b4b873726..e803e3f45 100644 --- a/docs/session-api.md +++ b/docs/session-api.md @@ -165,15 +165,19 @@ second or so before the archive flushes a new event. A request event's `raw`: |---|---| | `request` | the body as the agent sent it | | `forwarded` | the body as Cortex sent it upstream; absent when no plugin rewrote it | +| `translated` | the body after Cortex converted it to the server's protocol, as sent upstream; absent when Cortex did not convert it. `forwarded`, or `request` when `forwarded` is absent, is the input of the conversion | | `requestHeaders` | the request headers, without credential headers | | `query` | the URL's raw query | A response event's `raw` has `response`: `{status, headers, body, decoded, -truncated, incomplete}`. `body` is up to 10 MiB of the body as it came off the wire, +truncated, incomplete, upstream, upstreamTruncated}`. `body` is up to 10 MiB of the body as it came off the wire, in its `Content-Encoding`, unless `decoded` says the proxy decompressed it to read it for a plugin. `truncated` means it was longer, and `incomplete` means it was not read -to its end: the stream broke off, or the proxy, a plugin or the client stopped it. A body that is not valid UTF-8 — every compressed body — is in -`requestBase64`, `forwardedBase64` or `bodyBase64` instead. An unknown `raw` +to its end: the stream broke off, or the proxy, a plugin or the client stopped it. +When Cortex converted the reply to the agent's protocol, `body` is the converted reply +as the agent got it, and `upstream` is up to 10 MiB of the server's reply before the +conversion (`upstreamTruncated` when it was longer). A body that is not valid UTF-8 — every compressed body — is in +`requestBase64`, `forwardedBase64`, `translatedBase64`, `bodyBase64` or `upstreamBase64` instead. An unknown `raw` value is ignored, so an older proxy answers with the event alone. ## `GET /v1/sessions/{id}/timeline` — agents, tools and hosts on one time axis @@ -637,9 +641,10 @@ there. ```json {"v":1, "schemaVersion":1, "op":"append", "requestId":"…", "startedAt":"", "session":"", "event":{ "...SessionEvent wire JSON..." }, - "capture":{"original":"", "forwarded":null, "header":{"Name":["v"]}, "query":"", + "capture":{"original":"", "forwarded":null, "translated":null, "header":{"Name":["v"]}, "query":"", "response":{"status":200, "header":{}, "body":"", "decoded":true, - "truncated":false, "ended":true, "incomplete":false}}, + "truncated":false, "ended":true, "incomplete":false, + "upstream":null, "upstreamTruncated":false}}, "cost":{"headers":{"X-Litellm-Response-Cost":"0.01", "Content-Type":"application/json"}}} ``` diff --git a/docs/translation-server.md b/docs/translation-server.md new file mode 100644 index 000000000..2f5266693 --- /dev/null +++ b/docs/translation-server.md @@ -0,0 +1,60 @@ +# Use a translation server + +The [inference router](inference-router-plugin.md#servers-that-serve-another-protocol) +can translate one protocol pair only: Anthropic Messages (the agent's request) to +chat completions (the server). For each other pair, Cortex does not send the +request. It answers with a 502 in the agent's protocol: + +> Cortex cannot translate \ to \ yet. Use a translation server +> for this pair (see this page). + +For such a pair, run a translation server on loopback, and add it to the router as +the server for the agent. + +## Run LiteLLM + +This LiteLLM config worked on 2026-11-16. It accepts Anthropic Messages and sends +chat completions to the server: + +```yaml +model_list: + - model_name: + litellm_params: + model: hosted_vllm/ + api_base: + api_key: os.environ/UPSTREAM_API_KEY + additional_drop_params: ["prompt_cache_key"] +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY +``` + +- Use the `hosted_vllm/` provider, not `openai/`. LiteLLM's `openai/` provider + sent the requests to `/v1/responses`, and the server answered 403. +- `additional_drop_params` removes `prompt_cache_key`, which the server does not + accept. + +Start it: + +```sh +litellm --config litellm.yaml --host 127.0.0.1 --port 4000 +``` + +Bind it to loopback only, and always set a master key. Without them, any client that can +get to port 4000 can use your server key. + +## Add it to the router + +```sh +agentop server add --main --helper \ + --api anthropic-messages=/v1/messages http://127.0.0.1:4000 +``` + +Give the master key as the server key: at the prompt, or on stdin with +`--key-stdin`. Because `anthropic-messages` is in the server's `api` map, Cortex +sends Claude Code's requests to LiteLLM with no translation. + +## What Cortex records + +Cortex records only the agent's side of the hop: the request to LiteLLM and the +reply from LiteLLM. The cost and the errors of the hop from LiteLLM to the server +are LiteLLM's. Cortex does not see them.