From 3d7305360d0d47db5cbaeeb1ea870312a23024b6 Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Tue, 17 Nov 2026 15:33:54 -0500 Subject: [PATCH 01/15] feat: Add pipeline calls for translation and fixed upstream headers Add Translate and TranslateTarget. A routing plugin uses them to ask the listener to convert a redirected request to another protocol and to send it to a new path. Translate does not change Path and records nothing. Add SetUpstreamHeaders and UpstreamHeaders. The listener sets these headers after all plugins run. The call keeps a clone of the headers. All the new calls use the gates of SetRedirectPath. SetRedirectPath now uses the same gate and path check helpers. Part of #1911. Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- core/pipeline/context.go | 5 + core/pipeline/destination.go | 96 +++++++++++++-- core/pipeline/translate_test.go | 201 ++++++++++++++++++++++++++++++++ 3 files changed, 291 insertions(+), 11 deletions(-) create mode 100644 core/pipeline/translate_test.go diff --git a/core/pipeline/context.go b/core/pipeline/context.go index c37f87d02..35f8a2ddc 100644 --- a/core/pipeline/context.go +++ b/core/pipeline/context.go @@ -359,6 +359,11 @@ type Context struct { redirectHost string // redirectPath is the path SetRedirectPath set, "" to keep the client's. redirectPath string + // translateSource, translateTarget and translatePath are what Translate set; + // translateTarget is "" when no Translate was accepted. + translateSource, translateTarget, translatePath string + // upstreamHeaders is the clone SetUpstreamHeaders kept. See UpstreamHeaders. + upstreamHeaders http.Header // bodyMutated / responseBodyMutated flag that some plugin's // SetBody / SetResponseBody took effect on this context. Listeners read diff --git a/core/pipeline/destination.go b/core/pipeline/destination.go index d54bf448e..15fbc4b97 100644 --- a/core/pipeline/destination.go +++ b/core/pipeline/destination.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "log/slog" + "net/http" "net/url" "slices" "strings" @@ -139,17 +140,11 @@ func (c *Context) RedirectTarget() (scheme, host string, ok bool) { // went to, and the path the client named is kept in the modify/path_rewritten record. // The listener applies the copy RedirectPath reports. func (c *Context) SetRedirectPath(path string) error { - switch { - case c.inFinish: - return errors.New("pipeline: SetRedirectPath refused in OnFinish: the request has already been sent") - case c.currentPhase != InvocationPhaseRequest: - return fmt.Errorf("pipeline: SetRedirectPath refused in phase %q: it is accepted only from OnRequest", c.currentPhase) - case !c.currentMayRedirect: - return fmt.Errorf("pipeline: plugin %q called SetRedirectPath without declaring WritesDestination", c.currentPlugin) - case !c.redirected: - return errors.New("pipeline: SetRedirectPath refused: no Redirect has taken effect on this request") - case !strings.HasPrefix(path, "/") || strings.ContainsAny(path, "?#") || slices.Contains(strings.Split(path, "/"), ".."): - return errors.New("pipeline: SetRedirectPath path must be absolute, with no query, fragment or \"..\" segment") + if err := c.redirectedGate("SetRedirectPath"); err != nil { + return err + } + if err := checkRedirectPath("SetRedirectPath", path); err != nil { + return err } from := c.Path c.redirectPath, c.Path = path, path @@ -164,6 +159,85 @@ func (c *Context) RedirectPath() (string, bool) { return c.redirectPath, c.redirected && c.redirectPath != "" } +// Translate asks the listener to convert this redirected request from the source +// protocol to the target protocol and send it to path, with no query string. It does +// not change Path: the request row keeps the path the client named. +// +// Gates as SetRedirectPath: OnRequest only, a plugin that declares WritesDestination, +// a Redirect took effect, and path absolute with no query, fragment or ".." segment. +// Also source and target must be non-empty and different. It records nothing: only the +// listener knows what the conversion dropped, so the listener records +// modify/translated. A second call replaces the first. +func (c *Context) Translate(source, target, path string) error { + if err := c.redirectedGate("Translate"); err != nil { + return err + } + if err := checkRedirectPath("Translate", path); err != nil { + return err + } + if source == "" || target == "" || source == target { + return errors.New("pipeline: Translate needs a source and a target protocol that are not empty and not the same") + } + c.translateSource, c.translateTarget, c.translatePath = source, target, path + return nil +} + +// TranslateTarget is what the last accepted Translate set, with ok only when one was +// accepted on a request a Redirect moved. The listener applies it. +func (c *Context) TranslateTarget() (source, target, path string, ok bool) { + if !c.redirected || c.translateTarget == "" { + return "", "", "", false + } + return c.translateSource, c.translateTarget, c.translatePath, true +} + +// SetUpstreamHeaders gives headers that the listener sets on the upstream request +// after every plugin has run, each one replacing a header of the same name. A plugin +// cannot do this through Headers, because a later plugin can change those. Gates as +// SetRedirectPath, without the path. It keeps a clone of h, so a later change to h +// does not show. A second call replaces the first. +func (c *Context) SetUpstreamHeaders(h http.Header) error { + if err := c.redirectedGate("SetUpstreamHeaders"); err != nil { + return err + } + c.upstreamHeaders = h.Clone() + return nil +} + +// UpstreamHeaders is what the last accepted SetUpstreamHeaders set, and nil unless a +// Redirect took effect. +func (c *Context) UpstreamHeaders() http.Header { + if !c.redirected { + return nil + } + return c.upstreamHeaders +} + +// redirectedGate refuses a call that only a WritesDestination plugin may make from +// OnRequest, after a Redirect took effect. call names the method in the error. +func (c *Context) redirectedGate(call string) error { + switch { + case c.inFinish: + return fmt.Errorf("pipeline: %s refused in OnFinish: the request has already been sent", call) + case c.currentPhase != InvocationPhaseRequest: + return fmt.Errorf("pipeline: %s refused in phase %q: it is accepted only from OnRequest", call, c.currentPhase) + case !c.currentMayRedirect: + return fmt.Errorf("pipeline: plugin %q called %s without declaring WritesDestination", c.currentPlugin, call) + case !c.redirected: + return fmt.Errorf("pipeline: %s refused: no Redirect has taken effect on this request", call) + } + return nil +} + +// checkRedirectPath refuses a path that is not absolute, or that has a query, a +// fragment or a ".." segment. The error quotes no part of path. +func checkRedirectPath(call, path string) error { + if !strings.HasPrefix(path, "/") || strings.ContainsAny(path, "?#") || slices.Contains(strings.Split(path, "/"), "..") { + return fmt.Errorf("pipeline: %s path must be absolute, with no query, fragment or \"..\" segment", call) + } + return nil +} + // RequestedHost is the host the client named when a redirect sent the request // somewhere else, and "" otherwise — including when the redirects ended back at // that host. It is what a session event records beside Host. It compares against diff --git a/core/pipeline/translate_test.go b/core/pipeline/translate_test.go new file mode 100644 index 000000000..0125d4366 --- /dev/null +++ b/core/pipeline/translate_test.go @@ -0,0 +1,201 @@ +package pipeline + +import ( + "context" + "net/http" + "testing" +) + +const ( + srcProto = "anthropic-messages" + tgtProto = "chat-completions" +) + +// redirectThen gives a plugin that redirects to b.example and then runs do. +func redirectThen(do func(*Context) error, err *error) func(*Context) { + return func(c *Context) { + if *err = c.Redirect(mustURL("https://b.example")); *err == nil { + *err = do(c) + } + } +} + +func TestTranslate_AcceptedAfterARedirect(t *testing.T) { + var err error + _, pctx := runRequest(t, redirector(true, func(c *Context) { + c.Path = "/v1/messages" + redirectThen(func(c *Context) error { return c.Translate(srcProto, tgtProto, "/v1/chat/completions") }, &err)(c) + }), true) + if err != nil { + t.Fatalf("Translate: %v", err) + } + src, tgt, path, ok := pctx.TranslateTarget() + if !ok || src != srcProto || tgt != tgtProto || path != "/v1/chat/completions" { + t.Errorf("TranslateTarget() = %q, %q, %q, %v", src, tgt, path, ok) + } + if pctx.Path != "/v1/messages" { + t.Errorf("Path = %q, want the client's path: Translate does not change Path", pctx.Path) + } + for _, inv := range outbound(pctx) { + if inv.Reason != "redirected" { + t.Errorf("invocation %+v: Translate must record nothing", inv) + } + } +} + +func TestTranslate_SecondCallReplacesTheFirst(t *testing.T) { + var err error + _, pctx := runRequest(t, redirector(true, redirectThen(func(c *Context) error { + if err := c.Translate(srcProto, tgtProto, "/v1/a"); err != nil { + return err + } + return c.Translate(srcProto, "responses", "/v1/responses") + }, &err)), true) + if err != nil { + t.Fatalf("Translate: %v", err) + } + if _, tgt, path, ok := pctx.TranslateTarget(); !ok || tgt != "responses" || path != "/v1/responses" { + t.Errorf("TranslateTarget() = %q, %q, %v; want the second call", tgt, path, ok) + } +} + +func TestTranslate_RefusedWithoutARedirect(t *testing.T) { + var err error + _, pctx := runRequest(t, redirector(true, func(c *Context) { + err = c.Translate(srcProto, tgtProto, "/v1/chat/completions") + }), true) + if err == nil { + t.Fatal("Translate succeeded with no Redirect in effect") + } + if _, _, _, ok := pctx.TranslateTarget(); ok { + t.Error("TranslateTarget() ok for a request that was not redirected") + } +} + +func TestTranslate_RefusedWithoutTheCapability(t *testing.T) { + var err error + runRequest(t, redirector(false, func(c *Context) { + err = c.Translate(srcProto, tgtProto, "/v1/chat/completions") + }), true) + if err == nil { + t.Fatal("Translate succeeded for a plugin that does not declare WritesDestination") + } +} + +func TestTranslate_RefusedInTheResponsePass(t *testing.T) { + var err, rerr error + plug := redirector(true, func(c *Context) { rerr = c.Redirect(mustURL("https://b.example")) }) + plug.onResponse = func(c *Context) { err = c.Translate(srcProto, tgtProto, "/v1/chat/completions") } + p, pctx := runRequest(t, plug, true) + if rerr != nil { + t.Fatalf("Redirect: %v", rerr) + } + p.RunResponse(context.Background(), pctx) + if err == nil { + t.Fatal("Translate succeeded from OnResponse") + } + if _, _, _, ok := pctx.TranslateTarget(); ok { + t.Error("TranslateTarget() ok after a refusal in OnResponse") + } +} + +func TestTranslate_NothingUnderObserve(t *testing.T) { + var err error + _, pctx := runRequest(t, redirector(true, func(c *Context) { + _ = c.Redirect(mustURL("https://b.example")) + err = c.Translate(srcProto, tgtProto, "/v1/chat/completions") + }), true, WithPolicies(ErrorPolicyObserve)) + if err == nil { + t.Error("Translate succeeded under observe, where the Redirect moved nothing") + } + if _, _, _, ok := pctx.TranslateTarget(); ok { + t.Error("TranslateTarget() ok under observe") + } +} + +func TestTranslate_RefusesBadArguments(t *testing.T) { + for _, tc := range []struct{ src, tgt, path string }{ + {srcProto, tgtProto, "v1/x"}, + {srcProto, tgtProto, "/a?b"}, + {srcProto, tgtProto, "/a#b"}, + {srcProto, tgtProto, "/a/../b"}, + {srcProto, tgtProto, ""}, + {srcProto, srcProto, "/v1/messages"}, + {"", tgtProto, "/v1/chat/completions"}, + {srcProto, "", "/v1/chat/completions"}, + } { + var err error + _, pctx := runRequest(t, redirector(true, redirectThen(func(c *Context) error { + return c.Translate(tc.src, tc.tgt, tc.path) + }, &err)), true) + if err == nil { + t.Errorf("Translate(%q, %q, %q) succeeded", tc.src, tc.tgt, tc.path) + } + if _, _, _, ok := pctx.TranslateTarget(); ok { + t.Errorf("Translate(%q, %q, %q) left a target to apply", tc.src, tc.tgt, tc.path) + } + } +} + +func TestSetUpstreamHeaders_KeepsAClone(t *testing.T) { + var err error + h := http.Header{"User-Agent": {"bob-shell"}} + _, pctx := runRequest(t, redirector(true, redirectThen(func(c *Context) error { + return c.SetUpstreamHeaders(h) + }, &err)), true) + if err != nil { + t.Fatalf("SetUpstreamHeaders: %v", err) + } + h.Set("User-Agent", "changed") + h.Set("X-Later", "1") + got := pctx.UpstreamHeaders() + if got.Get("User-Agent") != "bob-shell" || got.Get("X-Later") != "" { + t.Errorf("UpstreamHeaders() = %v; a later change to the caller's map must not show", got) + } + if len(outbound(pctx)) != 1 { + t.Errorf("invocations = %+v, want only modify/redirected", outbound(pctx)) + } +} + +func TestSetUpstreamHeaders_Refusals(t *testing.T) { + h := http.Header{"User-Agent": {"bob-shell"}} + cases := map[string]func(t *testing.T) (error, *Context){ + "no redirect": func(t *testing.T) (error, *Context) { + var err error + _, pctx := runRequest(t, redirector(true, func(c *Context) { err = c.SetUpstreamHeaders(h) }), true) + return err, pctx + }, + "no capability": func(t *testing.T) (error, *Context) { + var err error + _, pctx := runRequest(t, redirector(false, func(c *Context) { err = c.SetUpstreamHeaders(h) }), true) + return err, pctx + }, + "observe": func(t *testing.T) (error, *Context) { + var err error + _, pctx := runRequest(t, redirector(true, func(c *Context) { + _ = c.Redirect(mustURL("https://b.example")) + err = c.SetUpstreamHeaders(h) + }), true, WithPolicies(ErrorPolicyObserve)) + return err, pctx + }, + "response pass": func(t *testing.T) (error, *Context) { + var err error + plug := redirector(true, func(c *Context) { _ = c.Redirect(mustURL("https://b.example")) }) + plug.onResponse = func(c *Context) { err = c.SetUpstreamHeaders(h) } + p, pctx := runRequest(t, plug, true) + p.RunResponse(context.Background(), pctx) + return err, pctx + }, + } + for name, run := range cases { + t.Run(name, func(t *testing.T) { + err, pctx := run(t) + if err == nil { + t.Error("SetUpstreamHeaders succeeded") + } + if pctx.UpstreamHeaders() != nil { + t.Errorf("UpstreamHeaders() = %v, want nil", pctx.UpstreamHeaders()) + } + }) + } +} From 83aeac1158272917d30d5a99be96293b4c41777f Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Tue, 17 Nov 2026 15:37:49 -0500 Subject: [PATCH 02/15] feat: Add the translation registry and golden tests Add the compiled-in protocol registry (rule H5) in core/translate: Lookup, Names, Decodes and Target (decision 12). The responses protocol is registered with only Name and EncodeError. Add the conversion helpers: Request, Reply and ErrorBody. Add NewReplyReader, which converts a reply as it reads it and does not wait for the whole stream (rule 10). Add golden tests from hand-written inputs: one request for each rule from 11 to 19 and for rule 20a, a full multi-turn request, the three Bob reply shapes, two tool calls, and the error replies, streamed and not streamed. A test splits each reply at every byte. TestDeterministic checks rule 9a. Exclude golden files from the whitespace fixers of pre-commit. They hold exact output bytes, and an SSE reply ends in a blank line. Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- .pre-commit-config.yaml | 4 +- core/translate/convert.go | 86 +++++++++ core/translate/golden_test.go | 179 ++++++++++++++++++ core/translate/reader.go | 55 ++++++ core/translate/reader_test.go | 102 ++++++++++ core/translate/registry.go | 72 +++++++ core/translate/registry_test.go | 68 +++++++ core/translate/testdata/reply/bob_shape1.json | 5 + .../reply/bob_shape1.json.to-json.golden | 1 + core/translate/testdata/reply/bob_shape1.sse | 23 +++ .../reply/bob_shape1.sse.to-sse.golden | 45 +++++ core/translate/testdata/reply/bob_shape2.json | 5 + .../reply/bob_shape2.json.to-json.golden | 1 + core/translate/testdata/reply/bob_shape2.sse | 15 ++ .../reply/bob_shape2.sse.to-sse.golden | 36 ++++ core/translate/testdata/reply/bob_shape3.json | 5 + .../reply/bob_shape3.json.to-json.golden | 1 + core/translate/testdata/reply/bob_shape3.sse | 7 + .../reply/bob_shape3.sse.to-sse.golden | 18 ++ .../translate/testdata/reply/chunk_error.json | 1 + .../reply/chunk_error.json.to-json.golden | 1 + core/translate/testdata/reply/chunk_error.sse | 3 + .../reply/chunk_error.sse.to-sse.golden | 12 ++ .../testdata/reply/error_403_litellm.json | 1 + .../error_403_litellm.json.to-json.golden | 1 + .../error_403_litellm.json.to-sse.golden | 3 + .../testdata/reply/error_404_bob.json | 1 + .../reply/error_404_bob.json.to-json.golden | 1 + .../reply/error_404_bob.json.to-sse.golden | 3 + .../testdata/reply/error_429_openai.json | 1 + .../error_429_openai.json.to-json.golden | 1 + .../reply/error_429_openai.json.to-sse.golden | 3 + .../testdata/reply/stream_two_tools.sse | 15 ++ .../reply/stream_two_tools.sse.to-sse.golden | 42 ++++ core/translate/testdata/reply/two_tools.json | 6 + .../reply/two_tools.json.to-json.golden | 1 + .../testdata/request/full_multiturn.golden | 2 + .../testdata/request/full_multiturn.json | 14 ++ .../request/rule11_unknown_fields.golden | 2 + .../request/rule11_unknown_fields.json | 2 + .../testdata/request/rule12_stream.golden | 2 + .../testdata/request/rule12_stream.json | 1 + .../request/rule13_system_blocks.golden | 2 + .../request/rule13_system_blocks.json | 3 + .../request/rule13_system_string.golden | 2 + .../request/rule13_system_string.json | 1 + .../request/rule13a_mid_system.golden | 2 + .../testdata/request/rule13a_mid_system.json | 6 + .../testdata/request/rule14_document.golden | 1 + .../testdata/request/rule14_document.json | 2 + .../testdata/request/rule14_image.golden | 2 + .../testdata/request/rule14_image.json | 5 + .../testdata/request/rule14_text.golden | 2 + .../testdata/request/rule14_text.json | 2 + .../request/rule14_tool_result.golden | 2 + .../testdata/request/rule14_tool_result.json | 12 ++ .../testdata/request/rule15_assistant.golden | 2 + .../testdata/request/rule15_assistant.json | 12 ++ .../request/rule16_server_tool.golden | 1 + .../testdata/request/rule16_server_tool.json | 3 + .../testdata/request/rule16_tools.golden | 2 + .../testdata/request/rule16_tools.json | 5 + .../request/rule17_any_no_parallel.golden | 2 + .../request/rule17_any_no_parallel.json | 1 + .../testdata/request/rule17_auto.golden | 2 + .../testdata/request/rule17_auto.json | 1 + .../testdata/request/rule17_none.golden | 2 + .../testdata/request/rule17_none.json | 1 + .../testdata/request/rule17_tool.golden | 2 + .../testdata/request/rule17_tool.json | 1 + .../testdata/request/rule18_sampling.golden | 2 + .../testdata/request/rule18_sampling.json | 2 + .../testdata/request/rule19_effort_low.golden | 2 + .../testdata/request/rule19_effort_low.json | 1 + .../request/rule19_effort_xhigh.golden | 2 + .../testdata/request/rule19_effort_xhigh.json | 1 + .../request/rule19_thinking_adaptive.golden | 2 + .../request/rule19_thinking_adaptive.json | 1 + .../request/rule19_thinking_disabled.golden | 2 + .../request/rule19_thinking_disabled.json | 1 + .../request/rule19_thinking_enabled.golden | 2 + .../request/rule19_thinking_enabled.json | 1 + .../request/rule20a_thought_id.golden | 2 + .../testdata/request/rule20a_thought_id.json | 5 + 84 files changed, 953 insertions(+), 1 deletion(-) create mode 100644 core/translate/convert.go create mode 100644 core/translate/golden_test.go create mode 100644 core/translate/reader.go create mode 100644 core/translate/reader_test.go create mode 100644 core/translate/registry.go create mode 100644 core/translate/registry_test.go create mode 100644 core/translate/testdata/reply/bob_shape1.json create mode 100644 core/translate/testdata/reply/bob_shape1.json.to-json.golden create mode 100644 core/translate/testdata/reply/bob_shape1.sse create mode 100644 core/translate/testdata/reply/bob_shape1.sse.to-sse.golden create mode 100644 core/translate/testdata/reply/bob_shape2.json create mode 100644 core/translate/testdata/reply/bob_shape2.json.to-json.golden create mode 100644 core/translate/testdata/reply/bob_shape2.sse create mode 100644 core/translate/testdata/reply/bob_shape2.sse.to-sse.golden create mode 100644 core/translate/testdata/reply/bob_shape3.json create mode 100644 core/translate/testdata/reply/bob_shape3.json.to-json.golden create mode 100644 core/translate/testdata/reply/bob_shape3.sse create mode 100644 core/translate/testdata/reply/bob_shape3.sse.to-sse.golden create mode 100644 core/translate/testdata/reply/chunk_error.json create mode 100644 core/translate/testdata/reply/chunk_error.json.to-json.golden create mode 100644 core/translate/testdata/reply/chunk_error.sse create mode 100644 core/translate/testdata/reply/chunk_error.sse.to-sse.golden create mode 100644 core/translate/testdata/reply/error_403_litellm.json create mode 100644 core/translate/testdata/reply/error_403_litellm.json.to-json.golden create mode 100644 core/translate/testdata/reply/error_403_litellm.json.to-sse.golden create mode 100644 core/translate/testdata/reply/error_404_bob.json create mode 100644 core/translate/testdata/reply/error_404_bob.json.to-json.golden create mode 100644 core/translate/testdata/reply/error_404_bob.json.to-sse.golden create mode 100644 core/translate/testdata/reply/error_429_openai.json create mode 100644 core/translate/testdata/reply/error_429_openai.json.to-json.golden create mode 100644 core/translate/testdata/reply/error_429_openai.json.to-sse.golden create mode 100644 core/translate/testdata/reply/stream_two_tools.sse create mode 100644 core/translate/testdata/reply/stream_two_tools.sse.to-sse.golden create mode 100644 core/translate/testdata/reply/two_tools.json create mode 100644 core/translate/testdata/reply/two_tools.json.to-json.golden create mode 100644 core/translate/testdata/request/full_multiturn.golden create mode 100644 core/translate/testdata/request/full_multiturn.json create mode 100644 core/translate/testdata/request/rule11_unknown_fields.golden create mode 100644 core/translate/testdata/request/rule11_unknown_fields.json create mode 100644 core/translate/testdata/request/rule12_stream.golden create mode 100644 core/translate/testdata/request/rule12_stream.json create mode 100644 core/translate/testdata/request/rule13_system_blocks.golden create mode 100644 core/translate/testdata/request/rule13_system_blocks.json create mode 100644 core/translate/testdata/request/rule13_system_string.golden create mode 100644 core/translate/testdata/request/rule13_system_string.json create mode 100644 core/translate/testdata/request/rule13a_mid_system.golden create mode 100644 core/translate/testdata/request/rule13a_mid_system.json create mode 100644 core/translate/testdata/request/rule14_document.golden create mode 100644 core/translate/testdata/request/rule14_document.json create mode 100644 core/translate/testdata/request/rule14_image.golden create mode 100644 core/translate/testdata/request/rule14_image.json create mode 100644 core/translate/testdata/request/rule14_text.golden create mode 100644 core/translate/testdata/request/rule14_text.json create mode 100644 core/translate/testdata/request/rule14_tool_result.golden create mode 100644 core/translate/testdata/request/rule14_tool_result.json create mode 100644 core/translate/testdata/request/rule15_assistant.golden create mode 100644 core/translate/testdata/request/rule15_assistant.json create mode 100644 core/translate/testdata/request/rule16_server_tool.golden create mode 100644 core/translate/testdata/request/rule16_server_tool.json create mode 100644 core/translate/testdata/request/rule16_tools.golden create mode 100644 core/translate/testdata/request/rule16_tools.json create mode 100644 core/translate/testdata/request/rule17_any_no_parallel.golden create mode 100644 core/translate/testdata/request/rule17_any_no_parallel.json create mode 100644 core/translate/testdata/request/rule17_auto.golden create mode 100644 core/translate/testdata/request/rule17_auto.json create mode 100644 core/translate/testdata/request/rule17_none.golden create mode 100644 core/translate/testdata/request/rule17_none.json create mode 100644 core/translate/testdata/request/rule17_tool.golden create mode 100644 core/translate/testdata/request/rule17_tool.json create mode 100644 core/translate/testdata/request/rule18_sampling.golden create mode 100644 core/translate/testdata/request/rule18_sampling.json create mode 100644 core/translate/testdata/request/rule19_effort_low.golden create mode 100644 core/translate/testdata/request/rule19_effort_low.json create mode 100644 core/translate/testdata/request/rule19_effort_xhigh.golden create mode 100644 core/translate/testdata/request/rule19_effort_xhigh.json create mode 100644 core/translate/testdata/request/rule19_thinking_adaptive.golden create mode 100644 core/translate/testdata/request/rule19_thinking_adaptive.json create mode 100644 core/translate/testdata/request/rule19_thinking_disabled.golden create mode 100644 core/translate/testdata/request/rule19_thinking_disabled.json create mode 100644 core/translate/testdata/request/rule19_thinking_enabled.golden create mode 100644 core/translate/testdata/request/rule19_thinking_enabled.json create mode 100644 core/translate/testdata/request/rule20a_thought_id.golden create mode 100644 core/translate/testdata/request/rule20a_thought_id.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 51d820efa..3318e0146 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,8 +4,10 @@ repos: hooks: - id: trailing-whitespace # The plans embed unified diffs, whose blank context lines are a lone space. - exclude: ^docs/superpowers/plans/ + # Golden files hold exact output bytes. A fixer must not change them. + exclude: ^docs/superpowers/plans/|\.golden$ - id: end-of-file-fixer + exclude: \.golden$ - id: check-added-large-files args: ['--maxkb=1024'] - id: check-yaml diff --git a/core/translate/convert.go b/core/translate/convert.go new file mode 100644 index 000000000..47b4436df --- /dev/null +++ b/core/translate/convert.go @@ -0,0 +1,86 @@ +package translate + +import ( + "fmt" + "net/http" + + "github.com/rossoctl/cortex/core/translate/pivot" +) + +// Converted is a request converted to the target protocol. +type Converted struct { + Body []byte + Dropped []string // the fields that the target could not carry (rule H4) + Stream bool // the agent asked for a stream (decision 4) +} + +// Request converts a request body from the source protocol to the target +// protocol, and removes the source protocol's own headers from h. A +// refusal to send to the agent is a *pivot.RequestError. +func Request(source, target string, body []byte, h http.Header) (Converted, error) { + dec, ok := protocols[source].(pivot.RequestDecoder) + if !ok { + return Converted{}, fmt.Errorf("translate: protocol %q cannot decode a request", source) + } + enc, ok := protocols[target].(pivot.RequestEncoder) + if !ok { + return Converted{}, fmt.Errorf("translate: protocol %q cannot encode a request", target) + } + req, err := dec.DecodeRequest(body) + if err != nil { + return Converted{}, err + } + out, dropped, err := enc.EncodeRequest(req, h) + if err != nil { + return Converted{}, err + } + return Converted{Body: out, Dropped: dropped, Stream: req.Stream}, nil +} + +// replyCodecs gives the decoder of the target's reply and the encoder of +// the source's reply. false when a direction is missing. +func replyCodecs(source, target string, status int, upstreamStream, agentStream bool) (pivot.ReplyDecoder, pivot.ReplyEncoder, bool) { + dec, ok := protocols[target].(pivot.ReplyDecoding) + if !ok { + return nil, nil, false + } + enc, ok := protocols[source].(pivot.ReplyEncoding) + if !ok { + return nil, nil, false + } + return dec.NewReplyDecoder(status, upstreamStream), enc.NewReplyEncoder(agentStream), true +} + +// Reply converts a whole reply body of the target protocol to the reply +// that the source protocol's agent expects. upstreamStream tells if the +// body is SSE; agentStream tells if the agent asked for a stream. When a +// direction is missing, it gives a 502 error body in the source protocol +// (nil for an unknown source). +func Reply(source, target string, status int, upstreamStream, agentStream bool, body []byte) []byte { + dec, enc, ok := replyCodecs(source, target, status, upstreamStream, agentStream) + if !ok { + return ErrorBody(source, http.StatusBadGateway, "api_error", + "Cortex cannot convert a "+target+" reply to "+source) + } + out := encodeAll(enc, nil, dec.Feed(body)) + out = encodeAll(enc, out, dec.End()) + return append(out, enc.End()...) +} + +// encodeAll appends the encoded events to out. +func encodeAll(enc pivot.ReplyEncoder, out []byte, evs []pivot.Event) []byte { + for _, ev := range evs { + out = append(out, enc.Encode(ev)...) + } + return out +} + +// ErrorBody gives an error body in the protocol's shape, or nil for an +// unknown protocol. +func ErrorBody(protocol string, status int, errType, message string) []byte { + p, ok := protocols[protocol] + if !ok { + return nil + } + return p.EncodeError(status, errType, message) +} diff --git a/core/translate/golden_test.go b/core/translate/golden_test.go new file mode 100644 index 000000000..f65272c0e --- /dev/null +++ b/core/translate/golden_test.go @@ -0,0 +1,179 @@ +package translate + +import ( + "bytes" + "flag" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +var update = flag.Bool("update", false, "rewrite golden files") + +// checkGolden compares got to the golden file, or writes it with -update. +func checkGolden(t *testing.T, path string, got []byte) { + t.Helper() + if *update { + if err := os.WriteFile(path, got, 0o644); err != nil { + t.Fatal(err) + } + return + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatalf("%v (run with -update to write it)", err) + } + if !bytes.Equal(got, want) { + t.Errorf("%s:\n got %s\nwant %s", path, got, want) + } +} + +// requestGolden gives the golden form of one request conversion: the +// first line names the dropped fields, the second line is the body. A +// refusal gives one "error:" line. +func requestGolden(in []byte) []byte { + c, err := Request(Anthropic, Chat, in, nil) + if err != nil { + return []byte("error: " + err.Error() + "\n") + } + return []byte("dropped: " + strings.Join(c.Dropped, ",") + "\n" + string(c.Body) + "\n") +} + +// TestRequestGolden converts each hand-written Anthropic request in +// testdata/request to chat completions (rules 11-19, 20a). +func TestRequestGolden(t *testing.T) { + inputs, _ := filepath.Glob("testdata/request/*.json") + if len(inputs) == 0 { + t.Fatal("no request inputs") + } + for _, in := range inputs { + t.Run(filepath.Base(in), func(t *testing.T) { + body, err := os.ReadFile(in) + if err != nil { + t.Fatal(err) + } + checkGolden(t, strings.TrimSuffix(in, ".json")+".golden", requestGolden(body)) + }) + } +} + +func encodeChat(t *testing.T, in []byte) string { + t.Helper() + c, err := Request(Anthropic, Chat, in, nil) + if err != nil { + t.Fatal(err) + } + return string(c.Body) +} + +// TestDeterministic checks rule 9a on the full multi-turn request. +func TestDeterministic(t *testing.T) { + in, err := os.ReadFile("testdata/request/full_multiturn.json") + if err != nil { + t.Fatal(err) + } + first := encodeChat(t, in) + if again := encodeChat(t, in); again != first { + t.Fatalf("two encodings differ:\n%s\n%s", first, again) + } + + // One more turn: the bytes up to the end of the old last message stay. + const marker = `,"cache_control":{"type":"ephemeral"}` + const oldLast = `"text":"Thanks. Is 391 prime?"` + lastOut := `{"role":"user","content":[{"type":"text",` + oldLast + marker + `}]}` + end := strings.Index(first, lastOut) + if end < 0 { + t.Fatalf("the last old message is not in the output:\n%s", first) + } + end += len(lastOut) + lastIn := oldLast + marker + `}]}` + longer := bytes.Replace(in, []byte(lastIn), []byte(lastIn+ + `,{"role":"assistant","content":[{"type":"text","text":"No: 17 x 23."}]}`+ + `,{"role":"user","content":[{"type":"text","text":"And 397?"}]}`), 1) + if bytes.Equal(longer, in) { + t.Fatal("the fixture does not hold the last user turn") + } + grown := encodeChat(t, longer) + if !strings.HasPrefix(grown, first[:end]) { + t.Errorf("one more turn changed the old bytes:\n old %s\n new %s", first[:end], grown) + } + + // Move the cache marker to the new last turn: only the marker bytes + // change. + moved := bytes.Replace(longer, []byte(oldLast+marker), []byte(oldLast), 1) + moved = bytes.Replace(moved, []byte(`"text":"And 397?"`), []byte(`"text":"And 397?"`+marker), 1) + movedOut := encodeChat(t, moved) + if movedOut == grown { + t.Fatal("moving the marker did not change the output") + } + if strings.ReplaceAll(movedOut, marker, "") != strings.ReplaceAll(grown, marker, "") { + t.Errorf("moving the marker changed more than the marker:\n%s\n%s", grown, movedOut) + } +} + +// replyCase is one hand-written chat-completions reply in testdata/reply. +// A .sse input is a stream. The status is 200, or N for a name that +// starts with "error_N_". +type replyCase struct { + path string + status int + stream bool +} + +func replyCases(t *testing.T) []replyCase { + t.Helper() + inputs, _ := filepath.Glob("testdata/reply/*.sse") + more, _ := filepath.Glob("testdata/reply/*.json") + inputs = append(inputs, more...) + if len(inputs) == 0 { + t.Fatal("no reply inputs") + } + var out []replyCase + for _, in := range inputs { + c := replyCase{path: in, status: 200, stream: strings.HasSuffix(in, ".sse")} + if rest, ok := strings.CutPrefix(filepath.Base(in), "error_"); ok { + n, err := strconv.Atoi(strings.SplitN(rest, "_", 2)[0]) + if err != nil { + t.Fatalf("%s: no status in the name", in) + } + c.status = n + } + out = append(out, c) + } + return out +} + +// agentForms gives the reply shapes that the agent asks for: the same +// shape as the upstream for a 2xx reply, both shapes for an error. +func (c replyCase) agentForms() []bool { + if c.status >= 200 && c.status < 300 { + return []bool{c.stream} + } + return []bool{true, false} +} + +func goldenName(c replyCase, agentStream bool) string { + if agentStream { + return c.path + ".to-sse.golden" + } + return c.path + ".to-json.golden" +} + +// TestReplyGolden converts each chat-completions reply to the Anthropic +// reply (rules 20-24). +func TestReplyGolden(t *testing.T) { + for _, c := range replyCases(t) { + body, err := os.ReadFile(c.path) + if err != nil { + t.Fatal(err) + } + for _, agentStream := range c.agentForms() { + name := goldenName(c, agentStream) + t.Run(filepath.Base(name), func(t *testing.T) { + checkGolden(t, name, Reply(Anthropic, Chat, c.status, c.stream, agentStream, body)) + }) + } + } +} diff --git a/core/translate/reader.go b/core/translate/reader.go new file mode 100644 index 000000000..da860fa7c --- /dev/null +++ b/core/translate/reader.go @@ -0,0 +1,55 @@ +package translate + +import ( + "io" + + "github.com/rossoctl/cortex/core/translate/pivot" +) + +// readSize is the most that one Read takes from the upstream body. +const readSize = 32 << 10 + +// replyReader converts an upstream reply body as it is read (rule 10). +type replyReader struct { + body io.ReadCloser + dec pivot.ReplyDecoder + enc pivot.ReplyEncoder + buf []byte // upstream bytes of one Read + pending []byte // converted bytes not read yet + err error // the upstream error, after pending is read +} + +// NewReplyReader gives the converted reply as a stream. Each Read takes +// at most 32 KiB from body and returns as soon as it has converted bytes: +// it never waits for the whole stream. At the upstream EOF it ends the +// decoder and the encoder. An upstream read error other than EOF comes +// after the bytes already converted. Close closes body. +func NewReplyReader(body io.ReadCloser, dec pivot.ReplyDecoder, enc pivot.ReplyEncoder) io.ReadCloser { + return &replyReader{body: body, dec: dec, enc: enc} +} + +func (r *replyReader) Read(p []byte) (int, error) { + for len(r.pending) == 0 { + if r.err != nil { + return 0, r.err + } + if r.buf == nil { + r.buf = make([]byte, readSize) + } + n, err := r.body.Read(r.buf) + r.pending = encodeAll(r.enc, r.pending, r.dec.Feed(r.buf[:n])) + switch { + case err == io.EOF: + r.pending = encodeAll(r.enc, r.pending, r.dec.End()) + r.pending = append(r.pending, r.enc.End()...) + r.err = io.EOF + case err != nil: + r.err = err + } + } + n := copy(p, r.pending) + r.pending = r.pending[n:] + return n, nil +} + +func (r *replyReader) Close() error { return r.body.Close() } diff --git a/core/translate/reader_test.go b/core/translate/reader_test.go new file mode 100644 index 000000000..55eeea677 --- /dev/null +++ b/core/translate/reader_test.go @@ -0,0 +1,102 @@ +package translate + +import ( + "bytes" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + "testing/iotest" + "time" + + "github.com/rossoctl/cortex/core/translate/anthropic" + "github.com/rossoctl/cortex/core/translate/chat" +) + +func newReader(r io.Reader, status int, upstreamStream, agentStream bool) io.ReadCloser { + return NewReplyReader(io.NopCloser(r), + chat.Protocol{}.NewReplyDecoder(status, upstreamStream), + anthropic.Protocol{}.NewReplyEncoder(agentStream)) +} + +// TestReplyReader_EveryByteSplit reads each reply input one byte per +// Read, so the input splits at every byte boundary. The output must equal +// the golden. +func TestReplyReader_EveryByteSplit(t *testing.T) { + for _, c := range replyCases(t) { + body, err := os.ReadFile(c.path) + if err != nil { + t.Fatal(err) + } + for _, agentStream := range c.agentForms() { + name := goldenName(c, agentStream) + t.Run(filepath.Base(name), func(t *testing.T) { + want, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + got, err := io.ReadAll(newReader(iotest.OneByteReader(bytes.NewReader(body)), c.status, c.stream, agentStream)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Errorf("got %s\nwant %s", got, want) + } + }) + } + } +} + +// TestReplyReader_Incremental checks rule 10: the first chunk gives the +// message_start frame while the upstream stream is still open. +func TestReplyReader_Incremental(t *testing.T) { + pr, pw := io.Pipe() + defer pw.Close() + go func() { + _, _ = pw.Write([]byte(`data: {"id":"chatcmpl-1","model":"m","choices":[{"index":0,"delta":{"role":"assistant"}}]}` + "\n\n")) + }() + r := newReader(pr, 200, true, true) + got := make(chan string, 1) + go func() { + buf := make([]byte, 4096) + n, _ := r.Read(buf) + got <- string(buf[:n]) + }() + select { + case s := <-got: + if !strings.HasPrefix(s, "event: message_start\n") { + t.Errorf("first Read = %q, want the message_start frame", s) + } + case <-time.After(5 * time.Second): + t.Fatal("Read waited for more of the stream") + } +} + +// TestReplyReader_ReadError checks that an upstream read error comes +// after the bytes already converted, and that Close closes the upstream. +func TestReplyReader_ReadError(t *testing.T) { + boom := errors.New("boom") + up := io.MultiReader(strings.NewReader(`data: {"id":"c","model":"m","choices":[]}`+"\n\n"), iotest.ErrReader(boom)) + closed := false + r := NewReplyReader(closer{up, &closed}, + chat.Protocol{}.NewReplyDecoder(200, true), anthropic.Protocol{}.NewReplyEncoder(true)) + got, err := io.ReadAll(r) + if !errors.Is(err, boom) { + t.Errorf("err = %v, want %v", err, boom) + } + if !strings.HasPrefix(string(got), "event: message_start\n") { + t.Errorf("bytes before the error = %q", got) + } + if err := r.Close(); err != nil || !closed { + t.Errorf("Close = %v, closed = %v", err, closed) + } +} + +type closer struct { + io.Reader + closed *bool +} + +func (c closer) Close() error { *c.closed = true; return nil } diff --git a/core/translate/registry.go b/core/translate/registry.go new file mode 100644 index 000000000..72b14877a --- /dev/null +++ b/core/translate/registry.go @@ -0,0 +1,72 @@ +package translate + +import ( + "slices" + + "github.com/rossoctl/cortex/core/translate/anthropic" + "github.com/rossoctl/cortex/core/translate/chat" + "github.com/rossoctl/cortex/core/translate/pivot" +) + +// The registered protocol names. +const ( + Anthropic = "anthropic-messages" + Chat = "chat-completions" + Responses = "responses" +) + +// protocols is the compiled-in registry (rule H5). It has no +// registration API. +var protocols = map[string]pivot.Protocol{ + Anthropic: anthropic.Protocol{}, + Chat: chat.Protocol{}, + Responses: responses{}, +} + +// preference is the order in which Target picks a protocol (decision 12). +var preference = []string{Chat, Anthropic, Responses} + +// responses is registered with no directions, so that a server can say +// that it serves Responses (rule C1). +type responses struct{} + +func (responses) Name() string { return Responses } + +// EncodeError gives an error body in the OpenAI shape. +func (responses) EncodeError(status int, errType, message string) []byte { + return chat.Protocol{}.EncodeError(status, errType, message) +} + +// Lookup gives the protocol with this name. +func Lookup(name string) (pivot.Protocol, bool) { + p, ok := protocols[name] + return p, ok +} + +// Names gives the registered protocol names, sorted. +func Names() []string { + names := make([]string, 0, len(protocols)) + for n := range protocols { + names = append(names, n) + } + slices.Sort(names) + return names +} + +// Decodes tells if the protocol can decode a request. +func Decodes(name string) bool { + _, ok := protocols[name].(pivot.RequestDecoder) + return ok +} + +// Target gives the protocol to translate to, from the protocols that a +// server serves: the first in the preference order that has a request +// encoder (rule 4). false when there is none (rule 4a). +func Target(served []string) (string, bool) { + for _, name := range preference { + if _, ok := protocols[name].(pivot.RequestEncoder); ok && slices.Contains(served, name) { + return name, true + } + } + return "", false +} diff --git a/core/translate/registry_test.go b/core/translate/registry_test.go new file mode 100644 index 000000000..df05cdcba --- /dev/null +++ b/core/translate/registry_test.go @@ -0,0 +1,68 @@ +package translate + +import ( + "reflect" + "testing" +) + +func TestNames(t *testing.T) { + want := []string{"anthropic-messages", "chat-completions", "responses"} + if got := Names(); !reflect.DeepEqual(got, want) { + t.Errorf("Names() = %v, want %v", got, want) + } + for _, n := range want { + p, ok := Lookup(n) + if !ok || p.Name() != n { + t.Errorf("Lookup(%q) = %v, %v", n, p, ok) + } + } + if _, ok := Lookup("gemini"); ok { + t.Error(`Lookup("gemini") found a protocol`) + } +} + +func TestTarget(t *testing.T) { + for _, tc := range []struct { + served []string + want string + ok bool + }{ + {[]string{"responses"}, "", false}, + {[]string{"responses", "chat-completions"}, "chat-completions", true}, + {[]string{"anthropic-messages", "responses"}, "", false}, + {nil, "", false}, + {[]string{"unknown", "chat-completions"}, "chat-completions", true}, + } { + got, ok := Target(tc.served) + if got != tc.want || ok != tc.ok { + t.Errorf("Target(%v) = %q, %v; want %q, %v", tc.served, got, ok, tc.want, tc.ok) + } + } +} + +func TestDecodes(t *testing.T) { + for name, want := range map[string]bool{Anthropic: true, Chat: false, Responses: false, "unknown": false} { + if got := Decodes(name); got != want { + t.Errorf("Decodes(%q) = %v, want %v", name, got, want) + } + } +} + +func TestErrorBody(t *testing.T) { + for _, tc := range []struct{ protocol, want string }{ + {Anthropic, `{"type":"error","error":{"type":"api_error","message":"m"}}`}, + {Chat, `{"error":{"message":"m","type":"api_error","code":null}}`}, + {Responses, `{"error":{"message":"m","type":"api_error","code":null}}`}, + {"unknown", ``}, + } { + if got := string(ErrorBody(tc.protocol, 502, "api_error", "m")); got != tc.want { + t.Errorf("ErrorBody(%q) = %s, want %s", tc.protocol, got, tc.want) + } + } +} + +func TestRequest_NoDirection(t *testing.T) { + if _, err := Request(Chat, Anthropic, []byte(`{}`), nil); err == nil { + t.Error("Request(chat, anthropic) gave no error; neither direction exists") + } +} diff --git a/core/translate/testdata/reply/bob_shape1.json b/core/translate/testdata/reply/bob_shape1.json new file mode 100644 index 000000000..ec14be4ac --- /dev/null +++ b/core/translate/testdata/reply/bob_shape1.json @@ -0,0 +1,5 @@ +{"id":"chatcmpl-b1","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion", + "choices":[{"index":0,"finish_reason":"tool_calls","message":{"role":"assistant","content":"391 is not prime","reasoning":"391 Not", + "thinking_blocks":[{"type":"thinking","thinking":"391 Not","signature":"EvkBsig"}], + "tool_calls":[{"id":"tooluse_9nf","type":"function","function":{"name":"Bash","arguments":"{\"command\": \"echo done\"}"}}]}}], + "usage":{"prompt_tokens":578,"completion_tokens":55,"total_tokens":633,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"prompt_tokens_details":{"cached_tokens":0},"completion_tokens_details":{"reasoning_tokens":0}}} diff --git a/core/translate/testdata/reply/bob_shape1.json.to-json.golden b/core/translate/testdata/reply/bob_shape1.json.to-json.golden new file mode 100644 index 000000000..812a4043a --- /dev/null +++ b/core/translate/testdata/reply/bob_shape1.json.to-json.golden @@ -0,0 +1 @@ +{"id":"chatcmpl-b1","type":"message","role":"assistant","model":"claude-opus-5-5","content":[{"type":"thinking","thinking":"391 Not","signature":"EvkBsig"},{"type":"text","text":"391 is not prime"},{"type":"tool_use","id":"tooluse_9nf","name":"Bash","input":{"command":"echo done"}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":578,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":55}} \ No newline at end of file diff --git a/core/translate/testdata/reply/bob_shape1.sse b/core/translate/testdata/reply/bob_shape1.sse new file mode 100644 index 000000000..e358cd1a0 --- /dev/null +++ b/core/translate/testdata/reply/bob_shape1.sse @@ -0,0 +1,23 @@ +: ping + +data: {"id":"chatcmpl-b1","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"reasoning":"391","thinking_blocks":[{"type":"thinking","thinking":"391"}],"role":"assistant"},"finish_reason":null}]} + +data: {"id":"chatcmpl-b1","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"reasoning":" Not","thinking_blocks":[{"type":"thinking","thinking":" Not"}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-b1","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"thinking_blocks":[{"type":"thinking","thinking":"","signature":"EvkBsig"}]},"finish_reason":null}]} + +: ping + +data: {"id":"chatcmpl-b1","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"391 is not prime"},"finish_reason":null}]} + +data: {"id":"chatcmpl-b1","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"tool_calls":[{"id":"tooluse_9nf","function":{"arguments":"","name":"Bash"},"type":"function","index":0}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-b1","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"tool_calls":[{"function":{"arguments":"{\"comma"},"type":"function","index":0}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-b1","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"tool_calls":[{"function":{"arguments":"nd\": \"echo done\"}"},"type":"function","index":0}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-b1","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]} + +data: {"id":"chatcmpl-b1","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[],"usage":{"prompt_tokens":578,"completion_tokens":55,"total_tokens":633,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"prompt_tokens_details":{"cached_tokens":0},"completion_tokens_details":{"reasoning_tokens":0}}} + +data: [DONE] diff --git a/core/translate/testdata/reply/bob_shape1.sse.to-sse.golden b/core/translate/testdata/reply/bob_shape1.sse.to-sse.golden new file mode 100644 index 000000000..837f5809c --- /dev/null +++ b/core/translate/testdata/reply/bob_shape1.sse.to-sse.golden @@ -0,0 +1,45 @@ +event: message_start +data: {"type":"message_start","message":{"id":"chatcmpl-b1","type":"message","role":"assistant","model":"claude-opus-5-5","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"391"}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":" Not"}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"EvkBsig"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: content_block_start +data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"391 is not prime"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":1} + +event: content_block_start +data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"tooluse_9nf","name":"Bash","input":{}}} + +event: content_block_delta +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\"comma"}} + +event: content_block_delta +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"nd\": \"echo done\"}"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":2} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"input_tokens":578,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":55}} + +event: message_stop +data: {"type":"message_stop"} + diff --git a/core/translate/testdata/reply/bob_shape2.json b/core/translate/testdata/reply/bob_shape2.json new file mode 100644 index 000000000..367442b09 --- /dev/null +++ b/core/translate/testdata/reply/bob_shape2.json @@ -0,0 +1,5 @@ +{"id":"chatcmpl-b2","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion", + "choices":[{"index":0,"finish_reason":"tool_calls","message":{"role":"assistant","content":"I will list the files.","reasoning":"", + "thinking_blocks":[{"type":"thinking","thinking":"","signature":"CAQSsig"}], + "tool_calls":[{"id":"tooluse_Zq1","type":"function","function":{"name":"execute_command","arguments":"{\"command\":\"ls\"}"}}]}}], + "usage":{"prompt_tokens":22333,"completion_tokens":216,"total_tokens":22549,"cache_read_input_tokens":21649,"cache_creation_input_tokens":682,"prompt_tokens_details":{"cached_tokens":21649},"completion_tokens_details":{"reasoning_tokens":0}}} diff --git a/core/translate/testdata/reply/bob_shape2.json.to-json.golden b/core/translate/testdata/reply/bob_shape2.json.to-json.golden new file mode 100644 index 000000000..335dec6ce --- /dev/null +++ b/core/translate/testdata/reply/bob_shape2.json.to-json.golden @@ -0,0 +1 @@ +{"id":"chatcmpl-b2","type":"message","role":"assistant","model":"claude-opus-5-5","content":[{"type":"thinking","thinking":"","signature":"CAQSsig"},{"type":"text","text":"I will list the files."},{"type":"tool_use","id":"tooluse_Zq1","name":"execute_command","input":{"command":"ls"}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":682,"cache_read_input_tokens":21649,"output_tokens":216}} \ No newline at end of file diff --git a/core/translate/testdata/reply/bob_shape2.sse b/core/translate/testdata/reply/bob_shape2.sse new file mode 100644 index 000000000..c52d249a8 --- /dev/null +++ b/core/translate/testdata/reply/bob_shape2.sse @@ -0,0 +1,15 @@ +data: {"id":"chatcmpl-b2","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"reasoning":"","thinking_blocks":[{"type":"thinking","thinking":""}],"content":"","role":"assistant"},"finish_reason":null}]} + +data: {"id":"chatcmpl-b2","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"reasoning":"","thinking_blocks":[{"type":"thinking","thinking":"","signature":"CAQSsig"}],"content":""},"finish_reason":null}]} + +data: {"id":"chatcmpl-b2","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"I will list the files."},"finish_reason":null}]} + +data: {"id":"chatcmpl-b2","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"","tool_calls":[{"id":"tooluse_Zq1","function":{"arguments":"","name":"execute_command"},"type":"function","index":0}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-b2","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"","tool_calls":[{"function":{"arguments":"{\"command\":\"ls\"}"},"type":"function","index":0}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-b2","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]} + +data: {"id":"chatcmpl-b2","created":1763337600,"model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[],"usage":{"prompt_tokens":22333,"completion_tokens":216,"total_tokens":22549,"cache_read_input_tokens":21649,"cache_creation_input_tokens":682,"prompt_tokens_details":{"cached_tokens":21649},"completion_tokens_details":{"reasoning_tokens":0}}} + +data: [DONE] diff --git a/core/translate/testdata/reply/bob_shape2.sse.to-sse.golden b/core/translate/testdata/reply/bob_shape2.sse.to-sse.golden new file mode 100644 index 000000000..ae02a512e --- /dev/null +++ b/core/translate/testdata/reply/bob_shape2.sse.to-sse.golden @@ -0,0 +1,36 @@ +event: message_start +data: {"type":"message_start","message":{"id":"chatcmpl-b2","type":"message","role":"assistant","model":"claude-opus-5-5","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"CAQSsig"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: content_block_start +data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"I will list the files."}} + +event: content_block_stop +data: {"type":"content_block_stop","index":1} + +event: content_block_start +data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"tooluse_Zq1","name":"execute_command","input":{}}} + +event: content_block_delta +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\"command\":\"ls\"}"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":2} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"input_tokens":2,"cache_creation_input_tokens":682,"cache_read_input_tokens":21649,"output_tokens":216}} + +event: message_stop +data: {"type":"message_stop"} + diff --git a/core/translate/testdata/reply/bob_shape3.json b/core/translate/testdata/reply/bob_shape3.json new file mode 100644 index 000000000..fda320e76 --- /dev/null +++ b/core/translate/testdata/reply/bob_shape3.json @@ -0,0 +1,5 @@ +{"id":"chatcmpl-b3","created":1763337600,"model":"premium-ide","object":"chat.completion", + "choices":[{"index":0,"finish_reason":"tool_calls","message":{"role":"assistant","content":null, + "provider_specific_fields":{"thought_signatures":["AY89a1+8zd/x="]}, + "tool_calls":[{"id":"call_3833__thought__AY89a1+8zd/x=","type":"function","function":{"name":"spawn_subagent","arguments":"{\"task\":\"look\"}"}}]}}], + "usage":{"prompt_tokens":13314,"completion_tokens":296,"total_tokens":13610,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"completion_tokens_details":{"reasoning_tokens":249}}} diff --git a/core/translate/testdata/reply/bob_shape3.json.to-json.golden b/core/translate/testdata/reply/bob_shape3.json.to-json.golden new file mode 100644 index 000000000..5c516518c --- /dev/null +++ b/core/translate/testdata/reply/bob_shape3.json.to-json.golden @@ -0,0 +1 @@ +{"id":"chatcmpl-b3","type":"message","role":"assistant","model":"premium-ide","content":[{"type":"tool_use","id":"call_3833__thought__AY89a1+8zd/x=","name":"spawn_subagent","input":{"task":"look"}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":13314,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":296}} \ No newline at end of file diff --git a/core/translate/testdata/reply/bob_shape3.sse b/core/translate/testdata/reply/bob_shape3.sse new file mode 100644 index 000000000..9be0802c1 --- /dev/null +++ b/core/translate/testdata/reply/bob_shape3.sse @@ -0,0 +1,7 @@ +data: {"id":"chatcmpl-b3","created":1763337600,"model":"premium-ide","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"provider_specific_fields":{"thought_signatures":["AY89a1+8zd/x="]},"role":"assistant","tool_calls":[{"id":"call_3833__thought__AY89a1+8zd/x=","function":{"arguments":"{\"task\":\"look\"}","name":"spawn_subagent"},"type":"function","index":0}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-b3","created":1763337600,"model":"premium-ide","object":"chat.completion.chunk","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]} + +data: {"id":"chatcmpl-b3","created":1763337600,"model":"premium-ide","object":"chat.completion.chunk","choices":[],"usage":{"prompt_tokens":13314,"completion_tokens":296,"total_tokens":13610,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"completion_tokens_details":{"reasoning_tokens":249}}} + +data: [DONE] diff --git a/core/translate/testdata/reply/bob_shape3.sse.to-sse.golden b/core/translate/testdata/reply/bob_shape3.sse.to-sse.golden new file mode 100644 index 000000000..9fdee2fb5 --- /dev/null +++ b/core/translate/testdata/reply/bob_shape3.sse.to-sse.golden @@ -0,0 +1,18 @@ +event: message_start +data: {"type":"message_start","message":{"id":"chatcmpl-b3","type":"message","role":"assistant","model":"premium-ide","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"call_3833__thought__AY89a1+8zd/x=","name":"spawn_subagent","input":{}}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"task\":\"look\"}"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"input_tokens":13314,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":296}} + +event: message_stop +data: {"type":"message_stop"} + diff --git a/core/translate/testdata/reply/chunk_error.json b/core/translate/testdata/reply/chunk_error.json new file mode 100644 index 000000000..924fc3c51 --- /dev/null +++ b/core/translate/testdata/reply/chunk_error.json @@ -0,0 +1 @@ +{"error":{"message":"litellm.InternalServerError: the upstream connection was reset","type":"None","code":"500"}} diff --git a/core/translate/testdata/reply/chunk_error.json.to-json.golden b/core/translate/testdata/reply/chunk_error.json.to-json.golden new file mode 100644 index 000000000..764313697 --- /dev/null +++ b/core/translate/testdata/reply/chunk_error.json.to-json.golden @@ -0,0 +1 @@ +{"type":"error","error":{"type":"api_error","message":"litellm.InternalServerError: the upstream connection was reset"}} \ No newline at end of file diff --git a/core/translate/testdata/reply/chunk_error.sse b/core/translate/testdata/reply/chunk_error.sse new file mode 100644 index 000000000..98b2657bc --- /dev/null +++ b/core/translate/testdata/reply/chunk_error.sse @@ -0,0 +1,3 @@ +data: {"id":"chatcmpl-e1","model":"claude-opus-5-5","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":"Starting"},"finish_reason":null}]} + +data: {"error":{"message":"litellm.InternalServerError: the upstream connection was reset","type":"None","code":"500"}} diff --git a/core/translate/testdata/reply/chunk_error.sse.to-sse.golden b/core/translate/testdata/reply/chunk_error.sse.to-sse.golden new file mode 100644 index 000000000..8154e4761 --- /dev/null +++ b/core/translate/testdata/reply/chunk_error.sse.to-sse.golden @@ -0,0 +1,12 @@ +event: message_start +data: {"type":"message_start","message":{"id":"chatcmpl-e1","type":"message","role":"assistant","model":"claude-opus-5-5","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Starting"}} + +event: error +data: {"type":"error","error":{"type":"api_error","message":"litellm.InternalServerError: the upstream connection was reset"}} + diff --git a/core/translate/testdata/reply/error_403_litellm.json b/core/translate/testdata/reply/error_403_litellm.json new file mode 100644 index 000000000..43092ce96 --- /dev/null +++ b/core/translate/testdata/reply/error_403_litellm.json @@ -0,0 +1 @@ +{"error":{"message":"team not allowed to access model. This team can only access models=['granite-4']. Tried to access claude-opus-5-5","type":"team_model_access_denied","param":"model","code":"403"}} diff --git a/core/translate/testdata/reply/error_403_litellm.json.to-json.golden b/core/translate/testdata/reply/error_403_litellm.json.to-json.golden new file mode 100644 index 000000000..0e686c0ab --- /dev/null +++ b/core/translate/testdata/reply/error_403_litellm.json.to-json.golden @@ -0,0 +1 @@ +{"type":"error","error":{"type":"permission_error","message":"team not allowed to access model. This team can only access models=['granite-4']. Tried to access claude-opus-5-5"}} \ No newline at end of file diff --git a/core/translate/testdata/reply/error_403_litellm.json.to-sse.golden b/core/translate/testdata/reply/error_403_litellm.json.to-sse.golden new file mode 100644 index 000000000..a987899d6 --- /dev/null +++ b/core/translate/testdata/reply/error_403_litellm.json.to-sse.golden @@ -0,0 +1,3 @@ +event: error +data: {"type":"error","error":{"type":"permission_error","message":"team not allowed to access model. This team can only access models=['granite-4']. Tried to access claude-opus-5-5"}} + diff --git a/core/translate/testdata/reply/error_404_bob.json b/core/translate/testdata/reply/error_404_bob.json new file mode 100644 index 000000000..b3190be02 --- /dev/null +++ b/core/translate/testdata/reply/error_404_bob.json @@ -0,0 +1 @@ +{"error":{"message":"model \"claude-opus-5-5\" not found or not accessible","type":"not_found_error"}} diff --git a/core/translate/testdata/reply/error_404_bob.json.to-json.golden b/core/translate/testdata/reply/error_404_bob.json.to-json.golden new file mode 100644 index 000000000..df92bd20c --- /dev/null +++ b/core/translate/testdata/reply/error_404_bob.json.to-json.golden @@ -0,0 +1 @@ +{"type":"error","error":{"type":"not_found_error","message":"model \"claude-opus-5-5\" not found or not accessible"}} \ No newline at end of file diff --git a/core/translate/testdata/reply/error_404_bob.json.to-sse.golden b/core/translate/testdata/reply/error_404_bob.json.to-sse.golden new file mode 100644 index 000000000..214548a45 --- /dev/null +++ b/core/translate/testdata/reply/error_404_bob.json.to-sse.golden @@ -0,0 +1,3 @@ +event: error +data: {"type":"error","error":{"type":"not_found_error","message":"model \"claude-opus-5-5\" not found or not accessible"}} + diff --git a/core/translate/testdata/reply/error_429_openai.json b/core/translate/testdata/reply/error_429_openai.json new file mode 100644 index 000000000..4ffa90a84 --- /dev/null +++ b/core/translate/testdata/reply/error_429_openai.json @@ -0,0 +1 @@ +{"error":{"message":"Rate limit reached for requests. Try again in 20s.","type":"rate_limit_exceeded","param":null,"code":"rate_limit_exceeded"}} diff --git a/core/translate/testdata/reply/error_429_openai.json.to-json.golden b/core/translate/testdata/reply/error_429_openai.json.to-json.golden new file mode 100644 index 000000000..3193c6b83 --- /dev/null +++ b/core/translate/testdata/reply/error_429_openai.json.to-json.golden @@ -0,0 +1 @@ +{"type":"error","error":{"type":"rate_limit_error","message":"Rate limit reached for requests. Try again in 20s."}} \ No newline at end of file diff --git a/core/translate/testdata/reply/error_429_openai.json.to-sse.golden b/core/translate/testdata/reply/error_429_openai.json.to-sse.golden new file mode 100644 index 000000000..84ab97deb --- /dev/null +++ b/core/translate/testdata/reply/error_429_openai.json.to-sse.golden @@ -0,0 +1,3 @@ +event: error +data: {"type":"error","error":{"type":"rate_limit_error","message":"Rate limit reached for requests. Try again in 20s."}} + diff --git a/core/translate/testdata/reply/stream_two_tools.sse b/core/translate/testdata/reply/stream_two_tools.sse new file mode 100644 index 000000000..242f7fc65 --- /dev/null +++ b/core/translate/testdata/reply/stream_two_tools.sse @@ -0,0 +1,15 @@ +data: {"id":"chatcmpl-t2","model":"gpt-x","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":"Two reads."},"finish_reason":null}]} + +data: {"id":"chatcmpl-t2","model":"gpt-x","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_a","type":"function","function":{"name":"Read","arguments":""}}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-t2","model":"gpt-x","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"path\":"}}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-t2","model":"gpt-x","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"a.go\"}"}}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-t2","model":"gpt-x","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"id":"call_b","type":"function","function":{"name":"Read","arguments":"{\"path\":"}}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-t2","model":"gpt-x","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"arguments":"\"b.go\"}"}}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-t2","model":"gpt-x","object":"chat.completion.chunk","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"completion_tokens":30,"total_tokens":130,"prompt_tokens_details":{"cached_tokens":64}}} + +data: [DONE] diff --git a/core/translate/testdata/reply/stream_two_tools.sse.to-sse.golden b/core/translate/testdata/reply/stream_two_tools.sse.to-sse.golden new file mode 100644 index 000000000..f609f0f04 --- /dev/null +++ b/core/translate/testdata/reply/stream_two_tools.sse.to-sse.golden @@ -0,0 +1,42 @@ +event: message_start +data: {"type":"message_start","message":{"id":"chatcmpl-t2","type":"message","role":"assistant","model":"gpt-x","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Two reads."}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: content_block_start +data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"call_a","name":"Read","input":{}}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"path\":"}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"\"a.go\"}"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":1} + +event: content_block_start +data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"call_b","name":"Read","input":{}}} + +event: content_block_delta +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\"path\":"}} + +event: content_block_delta +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"\"b.go\"}"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":2} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"input_tokens":36,"cache_creation_input_tokens":0,"cache_read_input_tokens":64,"output_tokens":30}} + +event: message_stop +data: {"type":"message_stop"} + diff --git a/core/translate/testdata/reply/two_tools.json b/core/translate/testdata/reply/two_tools.json new file mode 100644 index 000000000..775dc5416 --- /dev/null +++ b/core/translate/testdata/reply/two_tools.json @@ -0,0 +1,6 @@ +{"id":"chatcmpl-t2","model":"gpt-x","object":"chat.completion", + "choices":[{"index":0,"finish_reason":"tool_calls","message":{"role":"assistant","content":"Two reads.", + "tool_calls":[ + {"id":"call_a","type":"function","function":{"name":"Read","arguments":"{\"path\":\"a.go\"}"}}, + {"id":"call_b","type":"function","function":{"name":"Read","arguments":"{\"path\":\"b.go\"}"}}]}}], + "usage":{"prompt_tokens":100,"completion_tokens":30,"total_tokens":130,"prompt_tokens_details":{"cached_tokens":64}}} diff --git a/core/translate/testdata/reply/two_tools.json.to-json.golden b/core/translate/testdata/reply/two_tools.json.to-json.golden new file mode 100644 index 000000000..5a9897824 --- /dev/null +++ b/core/translate/testdata/reply/two_tools.json.to-json.golden @@ -0,0 +1 @@ +{"id":"chatcmpl-t2","type":"message","role":"assistant","model":"gpt-x","content":[{"type":"text","text":"Two reads."},{"type":"tool_use","id":"call_a","name":"Read","input":{"path":"a.go"}},{"type":"tool_use","id":"call_b","name":"Read","input":{"path":"b.go"}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":36,"cache_creation_input_tokens":0,"cache_read_input_tokens":64,"output_tokens":30}} \ No newline at end of file diff --git a/core/translate/testdata/request/full_multiturn.golden b/core/translate/testdata/request/full_multiturn.golden new file mode 100644 index 000000000..a164916e7 --- /dev/null +++ b/core/translate/testdata/request/full_multiturn.golden @@ -0,0 +1,2 @@ +dropped: metadata +{"model":"claude-opus-5-5","messages":[{"role":"system","content":[{"type":"text","text":"You are Claude Code."},{"type":"text","text":"Follow the project rules.","cache_control":{"type":"ephemeral"}}]},{"role":"user","content":[{"type":"text","text":"What is in this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgo="}}]},{"role":"assistant","content":"Let me check the file.","thinking_blocks":[{"type":"thinking","thinking":"Look at it first.","signature":"EvkBsig1"}],"tool_calls":[{"id":"toolu_01","type":"function","function":{"name":"Bash","arguments":"{\"command\":\"file cat.png\"}"}}]},{"role":"tool","content":[{"type":"text","text":"cat.png: PNG image data"}],"tool_call_id":"toolu_01"},{"role":"user","content":[{"type":"text","text":"The user changed cat.png."}]},{"role":"assistant","content":"It is a PNG image of a cat."},{"role":"user","content":[{"type":"text","text":"Thanks. Is 391 prime?","cache_control":{"type":"ephemeral"}}]}],"max_tokens":32000,"stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"Bash","description":"Run a shell command.","parameters":{"type":"object","properties":{"command":{"type":"string"}},"required":["command"]}}}],"tool_choice":"auto","reasoning_effort":"high"} diff --git a/core/translate/testdata/request/full_multiturn.json b/core/translate/testdata/request/full_multiturn.json new file mode 100644 index 000000000..7dcdc1603 --- /dev/null +++ b/core/translate/testdata/request/full_multiturn.json @@ -0,0 +1,14 @@ +{"model":"claude-opus-5-5","max_tokens":32000,"stream":true, + "system":[{"type":"text","text":"You are Claude Code."},{"type":"text","text":"Follow the project rules.","cache_control":{"type":"ephemeral"}}], + "tools":[{"name":"Bash","description":"Run a shell command.","input_schema":{"type":"object","properties":{"command":{"type":"string"}},"required":["command"]}}], + "tool_choice":{"type":"auto"}, + "thinking":{"type":"adaptive"}, + "output_config":{"effort":"high"}, + "metadata":{"user_id":"u1"}, + "messages":[ + {"role":"user","content":[{"type":"text","text":"What is in this image?"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"iVBORw0KGgo="}}]}, + {"role":"assistant","content":[{"type":"thinking","thinking":"Look at it first.","signature":"EvkBsig1"},{"type":"text","text":"Let me check the file."},{"type":"tool_use","id":"toolu_01","name":"Bash","input":{"command":"file cat.png"}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_01","content":"cat.png: PNG image data"}]}, + {"role":"system","content":"The user changed cat.png."}, + {"role":"assistant","content":[{"type":"text","text":"It is a PNG image of a cat."}]}, + {"role":"user","content":[{"type":"text","text":"Thanks. Is 391 prime?","cache_control":{"type":"ephemeral"}}]}]} diff --git a/core/translate/testdata/request/rule11_unknown_fields.golden b/core/translate/testdata/request/rule11_unknown_fields.golden new file mode 100644 index 000000000..961899bd6 --- /dev/null +++ b/core/translate/testdata/request/rule11_unknown_fields.golden @@ -0,0 +1,2 @@ +dropped: context_management,metadata,prompt_cache_key,service_tier +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule11_unknown_fields.json b/core/translate/testdata/request/rule11_unknown_fields.json new file mode 100644 index 000000000..4f41b1ac7 --- /dev/null +++ b/core/translate/testdata/request/rule11_unknown_fields.json @@ -0,0 +1,2 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"metadata":{"user_id":"u1"},"prompt_cache_key":"k","service_tier":"auto","context_management":{"edits":[]}, + "messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule12_stream.golden b/core/translate/testdata/request/rule12_stream.golden new file mode 100644 index 000000000..e892734b7 --- /dev/null +++ b/core/translate/testdata/request/rule12_stream.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024,"stream":true,"stream_options":{"include_usage":true}} diff --git a/core/translate/testdata/request/rule12_stream.json b/core/translate/testdata/request/rule12_stream.json new file mode 100644 index 000000000..72ab14dad --- /dev/null +++ b/core/translate/testdata/request/rule12_stream.json @@ -0,0 +1 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"stream":true,"messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule13_system_blocks.golden b/core/translate/testdata/request/rule13_system_blocks.golden new file mode 100644 index 000000000..6297b355e --- /dev/null +++ b/core/translate/testdata/request/rule13_system_blocks.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"system","content":[{"type":"text","text":"You are Claude Code."},{"type":"text","text":"Long instructions.","cache_control":{"type":"ephemeral","ttl":"1h"}}]},{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule13_system_blocks.json b/core/translate/testdata/request/rule13_system_blocks.json new file mode 100644 index 000000000..7ff0d0bb0 --- /dev/null +++ b/core/translate/testdata/request/rule13_system_blocks.json @@ -0,0 +1,3 @@ +{"model":"claude-opus-5-5","max_tokens":1024, + "system":[{"type":"text","text":"You are Claude Code."},{"type":"text","text":"Long instructions.","cache_control":{"type":"ephemeral","ttl":"1h"}}], + "messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule13_system_string.golden b/core/translate/testdata/request/rule13_system_string.golden new file mode 100644 index 000000000..e7f7b0888 --- /dev/null +++ b/core/translate/testdata/request/rule13_system_string.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"system","content":[{"type":"text","text":"You are a helpful agent."}]},{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule13_system_string.json b/core/translate/testdata/request/rule13_system_string.json new file mode 100644 index 000000000..6e2572d45 --- /dev/null +++ b/core/translate/testdata/request/rule13_system_string.json @@ -0,0 +1 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"system":"You are a helpful agent.","messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule13a_mid_system.golden b/core/translate/testdata/request/rule13a_mid_system.golden new file mode 100644 index 000000000..ff5e26865 --- /dev/null +++ b/core/translate/testdata/request/rule13a_mid_system.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"system","content":[{"type":"text","text":"sys"}]},{"role":"user","content":[{"type":"text","text":"hi"}]},{"role":"assistant","content":"hello"},{"role":"user","content":[{"type":"text","text":"hook output"}]},{"role":"user","content":[{"type":"text","text":"go on"}]}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule13a_mid_system.json b/core/translate/testdata/request/rule13a_mid_system.json new file mode 100644 index 000000000..f15b89c66 --- /dev/null +++ b/core/translate/testdata/request/rule13a_mid_system.json @@ -0,0 +1,6 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"system":"sys", + "messages":[ + {"role":"user","content":"hi"}, + {"role":"assistant","content":[{"type":"text","text":"hello"}]}, + {"role":"system","content":"hook output"}, + {"role":"user","content":"go on"}]} diff --git a/core/translate/testdata/request/rule14_document.golden b/core/translate/testdata/request/rule14_document.golden new file mode 100644 index 000000000..14499cbfe --- /dev/null +++ b/core/translate/testdata/request/rule14_document.golden @@ -0,0 +1 @@ +error: 400 invalid_request_error: this inference server cannot read documents (PDF) diff --git a/core/translate/testdata/request/rule14_document.json b/core/translate/testdata/request/rule14_document.json new file mode 100644 index 000000000..232289dc7 --- /dev/null +++ b/core/translate/testdata/request/rule14_document.json @@ -0,0 +1,2 @@ +{"model":"claude-opus-5-5","max_tokens":1024, + "messages":[{"role":"user","content":[{"type":"document","source":{"type":"base64","media_type":"application/pdf","data":"JVBERi0="}},{"type":"text","text":"Summarize."}]}]} diff --git a/core/translate/testdata/request/rule14_image.golden b/core/translate/testdata/request/rule14_image.golden new file mode 100644 index 000000000..cf56510b0 --- /dev/null +++ b/core/translate/testdata/request/rule14_image.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgo="}},{"type":"image_url","image_url":{"url":"https://example.com/cat.jpg"}},{"type":"text","text":"What is in these images?"}]}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule14_image.json b/core/translate/testdata/request/rule14_image.json new file mode 100644 index 000000000..3010d3c84 --- /dev/null +++ b/core/translate/testdata/request/rule14_image.json @@ -0,0 +1,5 @@ +{"model":"claude-opus-5-5","max_tokens":1024, + "messages":[{"role":"user","content":[ + {"type":"image","source":{"type":"base64","media_type":"image/png","data":"iVBORw0KGgo="}}, + {"type":"image","source":{"type":"url","url":"https://example.com/cat.jpg"}}, + {"type":"text","text":"What is in these images?"}]}]} diff --git a/core/translate/testdata/request/rule14_text.golden b/core/translate/testdata/request/rule14_text.golden new file mode 100644 index 000000000..6b719ffbb --- /dev/null +++ b/core/translate/testdata/request/rule14_text.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"one"},{"type":"text","text":"two ","cache_control":{"type":"ephemeral"}}]}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule14_text.json b/core/translate/testdata/request/rule14_text.json new file mode 100644 index 000000000..77f325ab1 --- /dev/null +++ b/core/translate/testdata/request/rule14_text.json @@ -0,0 +1,2 @@ +{"model":"claude-opus-5-5","max_tokens":1024, + "messages":[{"role":"user","content":[{"type":"text","text":"one"},{"type":"text","text":"two ","cache_control":{"type":"ephemeral"}}]}]} diff --git a/core/translate/testdata/request/rule14_tool_result.golden b/core/translate/testdata/request/rule14_tool_result.golden new file mode 100644 index 000000000..c3a78d129 --- /dev/null +++ b/core/translate/testdata/request/rule14_tool_result.golden @@ -0,0 +1,2 @@ +dropped: is_error +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"list and screenshot"}]},{"role":"assistant","tool_calls":[{"id":"toolu_01","type":"function","function":{"name":"Bash","arguments":"{\"command\":\"ls\"}"}},{"id":"toolu_02","type":"function","function":{"name":"Screenshot","arguments":"{}"}},{"id":"toolu_03","type":"function","function":{"name":"Bash","arguments":"{\"command\":\"false\"}"}}]},{"role":"tool","content":[{"type":"text","text":"a.txt\nb.txt"}],"tool_call_id":"toolu_01"},{"role":"tool","content":[{"type":"text","text":"the screen"}],"tool_call_id":"toolu_02"},{"role":"tool","content":[{"type":"text","text":"exit 1","cache_control":{"type":"ephemeral"}}],"tool_call_id":"toolu_03"},{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,AAAA"}}]},{"role":"user","content":[{"type":"text","text":"before the results"}]}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule14_tool_result.json b/core/translate/testdata/request/rule14_tool_result.json new file mode 100644 index 000000000..a4b8fd65a --- /dev/null +++ b/core/translate/testdata/request/rule14_tool_result.json @@ -0,0 +1,12 @@ +{"model":"claude-opus-5-5","max_tokens":1024, + "messages":[ + {"role":"user","content":"list and screenshot"}, + {"role":"assistant","content":[ + {"type":"tool_use","id":"toolu_01","name":"Bash","input":{"command":"ls"}}, + {"type":"tool_use","id":"toolu_02","name":"Screenshot","input":{}}, + {"type":"tool_use","id":"toolu_03","name":"Bash","input":{"command":"false"}}]}, + {"role":"user","content":[ + {"type":"text","text":"before the results"}, + {"type":"tool_result","tool_use_id":"toolu_01","content":"a.txt\nb.txt"}, + {"type":"tool_result","tool_use_id":"toolu_02","content":[{"type":"text","text":"the screen"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"AAAA"}}]}, + {"type":"tool_result","tool_use_id":"toolu_03","is_error":true,"content":[{"type":"text","text":"exit 1"}],"cache_control":{"type":"ephemeral"}}]}]} diff --git a/core/translate/testdata/request/rule15_assistant.golden b/core/translate/testdata/request/rule15_assistant.golden new file mode 100644 index 000000000..7a2052723 --- /dev/null +++ b/core/translate/testdata/request/rule15_assistant.golden @@ -0,0 +1,2 @@ +dropped: thinking +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"Is 391 prime?"}]},{"role":"assistant","content":"391 is not prime.","thinking_blocks":[{"type":"thinking","thinking":"391 = 17 x 23.","signature":"EvkBsig"},{"type":"redacted_thinking","data":"RDdata"}],"tool_calls":[{"id":"tooluse_9nf","type":"function","function":{"name":"Bash","arguments":"{\"command\":\"echo done\"}"}}]},{"role":"assistant","tool_calls":[{"id":"tooluse_x","type":"function","function":{"name":"Read","arguments":"{}"}}]},{"role":"tool","content":[{"type":"text","text":"done"}],"tool_call_id":"tooluse_9nf"},{"role":"tool","content":[{"type":"text","text":"ok"}],"tool_call_id":"tooluse_x"}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule15_assistant.json b/core/translate/testdata/request/rule15_assistant.json new file mode 100644 index 000000000..b96ab620f --- /dev/null +++ b/core/translate/testdata/request/rule15_assistant.json @@ -0,0 +1,12 @@ +{"model":"claude-opus-5-5","max_tokens":1024, + "messages":[ + {"role":"user","content":"Is 391 prime?"}, + {"role":"assistant","content":[ + {"type":"thinking","thinking":"391 = 17 x 23.","signature":"EvkBsig"}, + {"type":"redacted_thinking","data":"RDdata"}, + {"type":"thinking","thinking":"no signature"}, + {"type":"text","text":"391 is "}, + {"type":"text","text":"not prime."}, + {"type":"tool_use","id":"tooluse_9nf","name":"Bash","input":{"command":"echo done"}}]}, + {"role":"assistant","content":[{"type":"tool_use","id":"tooluse_x","name":"Read","input":{}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"tooluse_9nf","content":"done"},{"type":"tool_result","tool_use_id":"tooluse_x","content":"ok"}]}]} diff --git a/core/translate/testdata/request/rule16_server_tool.golden b/core/translate/testdata/request/rule16_server_tool.golden new file mode 100644 index 000000000..191b8744c --- /dev/null +++ b/core/translate/testdata/request/rule16_server_tool.golden @@ -0,0 +1 @@ +error: 400 invalid_request_error: this inference server cannot run the server tool "web_search" (type web_search_20250305) diff --git a/core/translate/testdata/request/rule16_server_tool.json b/core/translate/testdata/request/rule16_server_tool.json new file mode 100644 index 000000000..3f8ea2fac --- /dev/null +++ b/core/translate/testdata/request/rule16_server_tool.json @@ -0,0 +1,3 @@ +{"model":"claude-opus-5-5","max_tokens":1024, + "tools":[{"name":"Bash","description":"Run.","input_schema":{"type":"object"}},{"type":"web_search_20250305","name":"web_search","max_uses":5}], + "messages":[{"role":"user","content":"search"}]} diff --git a/core/translate/testdata/request/rule16_tools.golden b/core/translate/testdata/request/rule16_tools.golden new file mode 100644 index 000000000..66f7d6dc5 --- /dev/null +++ b/core/translate/testdata/request/rule16_tools.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024,"tools":[{"type":"function","function":{"name":"Bash","description":"Run a shell command.","parameters":{"type":"object","properties":{"command":{"type":"string"}},"required":["command"]}}},{"type":"function","function":{"name":"Read","description":"Read a file.","parameters":{"type":"object","properties":{"path":{"type":"string"}}}}}]} diff --git a/core/translate/testdata/request/rule16_tools.json b/core/translate/testdata/request/rule16_tools.json new file mode 100644 index 000000000..88e26e8e5 --- /dev/null +++ b/core/translate/testdata/request/rule16_tools.json @@ -0,0 +1,5 @@ +{"model":"claude-opus-5-5","max_tokens":1024, + "tools":[ + {"name":"Bash","description":"Run a shell command.","input_schema":{"type":"object","properties":{"command":{"type":"string"}},"required":["command"]}}, + {"name":"Read","description":"Read a file.","input_schema":{"type":"object","properties":{"path":{"type":"string"}}},"cache_control":{"type":"ephemeral"}}], + "messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule17_any_no_parallel.golden b/core/translate/testdata/request/rule17_any_no_parallel.golden new file mode 100644 index 000000000..84821de7e --- /dev/null +++ b/core/translate/testdata/request/rule17_any_no_parallel.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024,"tools":[{"type":"function","function":{"name":"Bash","parameters":{"type":"object"}}}],"tool_choice":"required","parallel_tool_calls":false} diff --git a/core/translate/testdata/request/rule17_any_no_parallel.json b/core/translate/testdata/request/rule17_any_no_parallel.json new file mode 100644 index 000000000..25fbf6f2c --- /dev/null +++ b/core/translate/testdata/request/rule17_any_no_parallel.json @@ -0,0 +1 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"tools":[{"name":"Bash","input_schema":{"type":"object"}}],"tool_choice":{"type":"any","disable_parallel_tool_use":true},"messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule17_auto.golden b/core/translate/testdata/request/rule17_auto.golden new file mode 100644 index 000000000..bce41128a --- /dev/null +++ b/core/translate/testdata/request/rule17_auto.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024,"tools":[{"type":"function","function":{"name":"Bash","parameters":{"type":"object"}}}],"tool_choice":"auto"} diff --git a/core/translate/testdata/request/rule17_auto.json b/core/translate/testdata/request/rule17_auto.json new file mode 100644 index 000000000..8bdc11554 --- /dev/null +++ b/core/translate/testdata/request/rule17_auto.json @@ -0,0 +1 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"tools":[{"name":"Bash","input_schema":{"type":"object"}}],"tool_choice":{"type":"auto"},"messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule17_none.golden b/core/translate/testdata/request/rule17_none.golden new file mode 100644 index 000000000..00b1d239a --- /dev/null +++ b/core/translate/testdata/request/rule17_none.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024,"tools":[{"type":"function","function":{"name":"Bash","parameters":{"type":"object"}}}],"tool_choice":"none"} diff --git a/core/translate/testdata/request/rule17_none.json b/core/translate/testdata/request/rule17_none.json new file mode 100644 index 000000000..b2094fc7a --- /dev/null +++ b/core/translate/testdata/request/rule17_none.json @@ -0,0 +1 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"tools":[{"name":"Bash","input_schema":{"type":"object"}}],"tool_choice":{"type":"none"},"messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule17_tool.golden b/core/translate/testdata/request/rule17_tool.golden new file mode 100644 index 000000000..c7240eca5 --- /dev/null +++ b/core/translate/testdata/request/rule17_tool.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024,"tools":[{"type":"function","function":{"name":"Bash","parameters":{"type":"object"}}}],"tool_choice":{"type":"function","function":{"name":"Bash"}}} diff --git a/core/translate/testdata/request/rule17_tool.json b/core/translate/testdata/request/rule17_tool.json new file mode 100644 index 000000000..45f18672a --- /dev/null +++ b/core/translate/testdata/request/rule17_tool.json @@ -0,0 +1 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"tools":[{"name":"Bash","input_schema":{"type":"object"}}],"tool_choice":{"type":"tool","name":"Bash"},"messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule18_sampling.golden b/core/translate/testdata/request/rule18_sampling.golden new file mode 100644 index 000000000..d6eb3a384 --- /dev/null +++ b/core/translate/testdata/request/rule18_sampling.golden @@ -0,0 +1,2 @@ +dropped: metadata,top_k +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024,"stop":["END","\n\nHuman:"],"temperature":0.7,"top_p":0.95} diff --git a/core/translate/testdata/request/rule18_sampling.json b/core/translate/testdata/request/rule18_sampling.json new file mode 100644 index 000000000..00398bee4 --- /dev/null +++ b/core/translate/testdata/request/rule18_sampling.json @@ -0,0 +1,2 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"stop_sequences":["END","\n\nHuman:"],"temperature":0.7,"top_p":0.95,"top_k":40,"metadata":{"user_id":"u1"}, + "messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule19_effort_low.golden b/core/translate/testdata/request/rule19_effort_low.golden new file mode 100644 index 000000000..0f630f4f1 --- /dev/null +++ b/core/translate/testdata/request/rule19_effort_low.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024,"reasoning_effort":"low"} diff --git a/core/translate/testdata/request/rule19_effort_low.json b/core/translate/testdata/request/rule19_effort_low.json new file mode 100644 index 000000000..017653a72 --- /dev/null +++ b/core/translate/testdata/request/rule19_effort_low.json @@ -0,0 +1 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"output_config":{"effort":"low"},"messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule19_effort_xhigh.golden b/core/translate/testdata/request/rule19_effort_xhigh.golden new file mode 100644 index 000000000..c5f5b4815 --- /dev/null +++ b/core/translate/testdata/request/rule19_effort_xhigh.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024,"reasoning_effort":"high"} diff --git a/core/translate/testdata/request/rule19_effort_xhigh.json b/core/translate/testdata/request/rule19_effort_xhigh.json new file mode 100644 index 000000000..bf3e5817c --- /dev/null +++ b/core/translate/testdata/request/rule19_effort_xhigh.json @@ -0,0 +1 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"output_config":{"effort":"xhigh"},"thinking":{"type":"adaptive"},"messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule19_thinking_adaptive.golden b/core/translate/testdata/request/rule19_thinking_adaptive.golden new file mode 100644 index 000000000..d83c7d97c --- /dev/null +++ b/core/translate/testdata/request/rule19_thinking_adaptive.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":16000,"reasoning_effort":"medium"} diff --git a/core/translate/testdata/request/rule19_thinking_adaptive.json b/core/translate/testdata/request/rule19_thinking_adaptive.json new file mode 100644 index 000000000..625e717a1 --- /dev/null +++ b/core/translate/testdata/request/rule19_thinking_adaptive.json @@ -0,0 +1 @@ +{"model":"claude-opus-5-5","max_tokens":16000,"thinking":{"type":"adaptive"},"messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule19_thinking_disabled.golden b/core/translate/testdata/request/rule19_thinking_disabled.golden new file mode 100644 index 000000000..ef3ad2952 --- /dev/null +++ b/core/translate/testdata/request/rule19_thinking_disabled.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule19_thinking_disabled.json b/core/translate/testdata/request/rule19_thinking_disabled.json new file mode 100644 index 000000000..40d144e1e --- /dev/null +++ b/core/translate/testdata/request/rule19_thinking_disabled.json @@ -0,0 +1 @@ +{"model":"claude-opus-5-5","max_tokens":1024,"thinking":{"type":"disabled"},"messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule19_thinking_enabled.golden b/core/translate/testdata/request/rule19_thinking_enabled.golden new file mode 100644 index 000000000..d83c7d97c --- /dev/null +++ b/core/translate/testdata/request/rule19_thinking_enabled.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":16000,"reasoning_effort":"medium"} diff --git a/core/translate/testdata/request/rule19_thinking_enabled.json b/core/translate/testdata/request/rule19_thinking_enabled.json new file mode 100644 index 000000000..ba67e5857 --- /dev/null +++ b/core/translate/testdata/request/rule19_thinking_enabled.json @@ -0,0 +1 @@ +{"model":"claude-opus-5-5","max_tokens":16000,"thinking":{"type":"enabled","budget_tokens":8000},"messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule20a_thought_id.golden b/core/translate/testdata/request/rule20a_thought_id.golden new file mode 100644 index 000000000..e3de589ca --- /dev/null +++ b/core/translate/testdata/request/rule20a_thought_id.golden @@ -0,0 +1,2 @@ +dropped: +{"model":"premium-ide","messages":[{"role":"user","content":[{"type":"text","text":"spawn a helper"}]},{"role":"assistant","tool_calls":[{"id":"call_3833__thought__AY89a1+8zd/x=","type":"function","function":{"name":"spawn_subagent","arguments":"{\"task\":\"look\"}"}}]},{"role":"tool","content":[{"type":"text","text":"done"}],"tool_call_id":"call_3833__thought__AY89a1+8zd/x="}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule20a_thought_id.json b/core/translate/testdata/request/rule20a_thought_id.json new file mode 100644 index 000000000..6fe2608e1 --- /dev/null +++ b/core/translate/testdata/request/rule20a_thought_id.json @@ -0,0 +1,5 @@ +{"model":"premium-ide","max_tokens":1024, + "messages":[ + {"role":"user","content":"spawn a helper"}, + {"role":"assistant","content":[{"type":"tool_use","id":"call_3833__thought__AY89a1+8zd/x=","name":"spawn_subagent","input":{"task":"look"}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"call_3833__thought__AY89a1+8zd/x=","content":"done"}]}]} From 7309c6451f4a052033e6663a73e6a77e8569f01e Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Tue, 17 Nov 2026 15:48:37 -0500 Subject: [PATCH 03/15] feat: Translate requests and replies in the forward proxy When a plugin calls pctx.Translate, the forward proxy now converts the request after the last request plugin and converts the reply as it reads it, before a reply plugin or a recorder reads it. - The request goes to the target path with no query string. The request row keeps the path of the client. - The proxy records modify/translated, with "dropped" when the target cannot carry some fields. - A refused conversion gives an error in the protocol of the agent. Nothing goes upstream. - The headers from SetUpstreamHeaders replace the headers of the same name, also on a resend. - The transport decodes gzip for a translated request. - A Resender reads the raw answer. The proxy converts the changed request again before it sends it. Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- core/listener/forwardproxy/server.go | 57 ++- core/listener/forwardproxy/translate.go | 139 ++++++ core/listener/forwardproxy/translate_test.go | 479 +++++++++++++++++++ 3 files changed, 669 insertions(+), 6 deletions(-) create mode 100644 core/listener/forwardproxy/translate.go create mode 100644 core/listener/forwardproxy/translate_test.go diff --git a/core/listener/forwardproxy/server.go b/core/listener/forwardproxy/server.go index b43ea0250..3f95c4252 100644 --- a/core/listener/forwardproxy/server.go +++ b/core/listener/forwardproxy/server.go @@ -35,6 +35,7 @@ import ( "github.com/rossoctl/cortex/core/spiffe" "github.com/rossoctl/cortex/core/tlsbridge" authtls "github.com/rossoctl/cortex/core/tlsconfig" + "github.com/rossoctl/cortex/core/translate" ) // maxRequestBodySize caps a buffered request body. It is separate from @@ -630,6 +631,8 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne sessionID = s.hydrateSession(pctx, trusted, r.Header, chain, connProcOf(r.Context())) } + // tr is the conversion a plugin asked for with pctx.Translate, nil for none. + var tr *translation if !skipped { action := s.OutboundPipeline.Run(r.Context(), pctx) @@ -666,6 +669,15 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne httpx.WriteRejectionForRequest(w, action, pctx) return } + + // After every request plugin, and before the request row is recorded, so the + // modify/translated record is on that row. A refusal goes nowhere upstream. + var refusal *pipeline.Action + if tr, refusal = startTranslation(pctx); refusal != nil { + s.recordOutboundRejectIn(tl, pctx, *refusal, s.recordingSessionID(sessionID, r.Header, chain)) + httpx.WriteRejectionForRequest(w, *refusal, pctx) + return + } } // A WebSocket upgrade is the one hop-by-hop request this proxy forwards @@ -749,6 +761,7 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne // body-rewrite block below and the transport, so leave them untouched. // Mirrors reverseproxy's forwarded-request header sync. httpx.SyncRequestHeaders(r, pctx.Headers) + applyUpstreamHeaders(r, pctx.UpstreamHeaders()) // If a WritesRequestBody plugin rewrote pctx.Body, ship the new bytes // upstream and clear Content-Encoding (see forwardproxy response @@ -756,6 +769,9 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne if pctx.BodyMutated() { httpx.SetRequestBody(r, pctx.Body) } + if tr != nil { + tr.apply(r) + } // The pipeline this request runs on for the rest of the handler: every // capability probe below (encoding, streaming, buffering) must answer @@ -772,7 +788,8 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne bufferResponse := !skipped && pinned.NeedsResponseBody() && (parsed || pinned.ReadsEveryResponseBody()) // Hop branch point 3: the hop always owns the encoding. - ownEncoding := hopped || bufferResponse || (!skipped && pinned.HasStreamingResponders()) + // A translated reply is converted, so the transport must decode it first. + ownEncoding := hopped || bufferResponse || tr != nil || (!skipped && pinned.HasStreamingResponders()) stripForUpstream(r, ownEncoding, forwardUpgrade) // Clear RequestURI — set by the server but must be empty for client requests @@ -840,7 +857,7 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne // itself, with whatever credential it has now. Left as a request-keyed rule // rather than made cleverer (#1549 review). if err == nil && !skipped && !forwardUpgrade { - resp, err = s.resendIfAsked(r, pctx, client, resp, tl, ownEncoding, capture) + resp, err = s.resendIfAsked(r, pctx, client, resp, tl, ownEncoding, capture, tr) } if err != nil { // The egress guard's refusal, BEFORE the generic 502: a destination that @@ -908,6 +925,12 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne } defer resp.Body.Close() + // Before any reply plugin or recorder reads the body: they all see the reply in + // the agent's protocol. A hop's own 503 returned above and is not converted. + if tr != nil { + tr.wrapReply(resp) + } + // Response phase: populate pctx and run plugins in reverse order. pctx.StatusCode = resp.StatusCode pctx.ResponseHeaders = resp.Header.Clone() @@ -1260,7 +1283,11 @@ func stripForUpstream(r *http.Request, ownEncoding, keepUpgrade bool) { // id, so the timeline shows both attempts and each pairs with its own answer. A // refused answer that is not sent again goes to the client byte for byte, the bytes // read for the Resender included. -func (s *Server) resendIfAsked(r *http.Request, pctx *pipeline.Context, client *http.Client, resp *http.Response, tl *tunnelLog, ownEncoding bool, capture *pipeline.RequestCapture) (*http.Response, error) { +// +// tr is the translation of the request, nil for none. The Resender reads the raw +// answer, the request is converted again after the Resender changed it, and the +// refused answer's row has the reply in the agent's protocol. +func (s *Server) resendIfAsked(r *http.Request, pctx *pipeline.Context, client *http.Client, resp *http.Response, tl *tunnelLog, ownEncoding bool, capture *pipeline.RequestCapture, tr *translation) (*http.Response, error) { // Both probes and Resend itself answer for the pipeline this request // ran on (Holder.ForPctx): the reload-drain window must not flip the // resend decision onto plugins that never saw the request. @@ -1272,7 +1299,15 @@ func (s *Server) resendIfAsked(r *http.Request, pctx *pipeline.Context, client * // The parse as the refused attempt was sent: a Resender may rewrite the model, // and the refused attempt's row must name the one the upstream refused. sent := pipeline.SnapshotInference(pctx.Extensions.Inference) - if !s.OutboundPipeline.Resend(r.Context(), pctx, resp.StatusCode, head) { + asked := s.OutboundPipeline.Resend(r.Context(), pctx, resp.StatusCode, head) + var conv translate.Converted + if asked && tr != nil { + var convErr error + // A request that cannot be converted after the change is not sent again. + conv, convErr = tr.convert(pctx) + asked = convErr == nil + } + if !asked { resp.Body = struct { io.Reader io.Closer @@ -1285,7 +1320,11 @@ func (s *Server) resendIfAsked(r *http.Request, pctx *pipeline.Context, client * // back at once, because the status of the answer the client gets is the // one OutcomeFromContext must read. resent := pctx.Extensions.Inference - pctx.StatusCode, pctx.ResponseBody, pctx.Extensions.Inference = resp.StatusCode, head, sent + refused := head + if tr != nil { + refused = tr.replyBody(resp, head) + } + pctx.StatusCode, pctx.ResponseBody, pctx.Extensions.Inference = resp.StatusCode, refused, sent if capture != nil && pctx.Extensions.Inference != nil { // The refusal as far as it was read: the resend peeks at most ResendPeekLimit bytes. pctx.ResponseCapture = &pipeline.ResponseCapture{Status: resp.StatusCode, Header: redact.Headers(resp.Header), @@ -1298,7 +1337,13 @@ func (s *Server) resendIfAsked(r *http.Request, pctx *pipeline.Context, client * } again := r.Clone(r.Context()) httpx.SyncRequestHeaders(again, pctx.Headers) - httpx.SetRequestBody(again, pctx.Body) + applyUpstreamHeaders(again, pctx.UpstreamHeaders()) + if tr != nil { + tr.conv = conv + tr.apply(again) + } else { + httpx.SetRequestBody(again, pctx.Body) + } // No keepUpgrade: a resent request is never a forwarded upgrade — the // 101 branch in serveOutbound skips resendIfAsked entirely, and only that // branch leaves the headers unstripped. diff --git a/core/listener/forwardproxy/translate.go b/core/listener/forwardproxy/translate.go new file mode 100644 index 000000000..8550cf5d4 --- /dev/null +++ b/core/listener/forwardproxy/translate.go @@ -0,0 +1,139 @@ +package forwardproxy + +import ( + "errors" + "net/http" + "strings" + + "github.com/rossoctl/cortex/core/listener/httpx" + "github.com/rossoctl/cortex/core/pipeline" + "github.com/rossoctl/cortex/core/translate" + "github.com/rossoctl/cortex/core/translate/pivot" +) + +// translation is a request that the listener converts to the protocol of the +// server, as a plugin asked with pctx.Translate. It goes through the handler as a +// local, not through pctx, so no plugin can change it. +type translation struct { + source, target, path string + conv translate.Converted + dec pivot.ReplyDecoding // the target's reply decoder + enc pivot.ReplyEncoding // the source's reply encoder +} + +// startTranslation converts pctx.Body when a plugin asked for it. It is called after +// every request plugin has run (rule 8). It gives nil, nil when no plugin asked. On a +// refusal it gives the Reject action, with the error body in the agent's protocol; +// nothing must then go upstream. On success it records modify/translated, with the +// fields that the target could not carry (decision 17). +func startTranslation(pctx *pipeline.Context) (*translation, *pipeline.Action) { + src, tgt, path, ok := pctx.TranslateTarget() + if !ok { + return nil, nil + } + t := &translation{source: src, target: tgt, path: path} + conv, err := t.convert(pctx) + if err != nil { + status, errType := http.StatusBadGateway, "api_error" + var re *pivot.RequestError + if errors.As(err, &re) { + status, errType = re.Status, re.Type + } + msg := err.Error() + if re != nil { + msg = re.Message + } + pctx.Record(pipeline.Invocation{ + Plugin: "forward-proxy", + Action: pipeline.ActionDeny, + Phase: pipeline.InvocationPhaseRequest, + Reason: "translation_refused", + Details: map[string]string{"from": src, "to": tgt, "message": msg}, + }) + return nil, &pipeline.Action{Type: pipeline.Reject, Violation: &pipeline.Violation{ + Code: "inference.translation_refused", + Reason: msg, + Status: status, + Body: translate.ErrorBody(src, status, errType, msg), + BodyType: "application/json", + }} + } + details := map[string]string{"from": src, "to": tgt, "path": path} + if len(conv.Dropped) > 0 { + details["dropped"] = strings.Join(conv.Dropped, ",") + } + pctx.Record(pipeline.Invocation{ + Plugin: "forward-proxy", + Action: pipeline.ActionModify, + Phase: pipeline.InvocationPhaseRequest, + Reason: "translated", + Details: details, + }) + t.conv = conv + return t, nil +} + +// convert converts pctx.Body. It removes the source protocol's own headers from +// pctx.Headers (decision 18), so the header sync removes them from the wire. It +// refuses a pair whose reply Cortex cannot convert back, before anything is sent. +func (t *translation) convert(pctx *pipeline.Context) (translate.Converted, error) { + if pctx.Body == nil { + return translate.Converted{}, errors.New("the request body was not buffered, so Cortex cannot translate it") + } + tp, _ := translate.Lookup(t.target) + sp, _ := translate.Lookup(t.source) + var okDec, okEnc bool + t.dec, okDec = tp.(pivot.ReplyDecoding) + t.enc, okEnc = sp.(pivot.ReplyEncoding) + if !okDec || !okEnc { + return translate.Converted{}, errors.New("Cortex cannot convert a " + t.target + " reply to " + t.source) + } + return translate.Request(t.source, t.target, pctx.Body, pctx.Headers) +} + +// applyUpstreamHeaders sets the headers that a plugin gave with +// pctx.SetUpstreamHeaders on the upstream request. Each one replaces a header of +// the same name (rule C3). It runs after the sync of pctx.Headers. +func applyUpstreamHeaders(r *http.Request, h http.Header) { + for k, v := range h { + r.Header[http.CanonicalHeaderKey(k)] = append([]string(nil), v...) + } +} + +// apply puts the converted body and the target path on the upstream request, with +// no query string. pctx.Path does not change: the request row keeps the client's path. +func (t *translation) apply(r *http.Request) { + httpx.SetRequestBody(r, t.conv.Body) + r.URL.Path = t.path + r.URL.RawPath, r.URL.RawQuery, r.URL.ForceQuery = "", "", false +} + +// agentStream tells if the agent gets a stream for an upstream answer with this +// status: only a 2xx answer to a stream request (decision 4). +func (t *translation) agentStream(status int) bool { + return t.conv.Stream && status/100 == 2 +} + +// wrapReply makes resp carry the reply in the agent's protocol, converted as it is +// read (rule 10). Every later arm then reads the converted bytes. +func (t *translation) wrapReply(resp *http.Response) { + stream := t.agentStream(resp.StatusCode) + dec := t.dec.NewReplyDecoder(resp.StatusCode, httpx.IsEventStream(resp.Header.Get("Content-Type"))) + enc := t.enc.NewReplyEncoder(stream) + resp.Body = translate.NewReplyReader(resp.Body, dec, enc) + if stream { + resp.Header.Set("Content-Type", "text/event-stream") + } else { + resp.Header.Set("Content-Type", "application/json") + } + resp.Header.Del("Content-Length") + resp.Header.Del("Content-Encoding") + resp.ContentLength = -1 +} + +// replyBody converts a whole upstream body, for the row of a refused attempt +// (decision 19). +func (t *translation) replyBody(resp *http.Response, body []byte) []byte { + return translate.Reply(t.source, t.target, resp.StatusCode, + httpx.IsEventStream(resp.Header.Get("Content-Type")), t.agentStream(resp.StatusCode), body) +} diff --git a/core/listener/forwardproxy/translate_test.go b/core/listener/forwardproxy/translate_test.go new file mode 100644 index 000000000..486920e82 --- /dev/null +++ b/core/listener/forwardproxy/translate_test.go @@ -0,0 +1,479 @@ +package forwardproxy + +import ( + "bufio" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/rossoctl/cortex/core/pipeline" + "github.com/rossoctl/cortex/core/plugins/inferenceparser" + "github.com/rossoctl/cortex/core/session" +) + +// translator redirects every request to target and asks the listener to translate +// it from Anthropic Messages to chat completions, as the inference router does. When +// translate is false it only redirects. headers, when set, go to SetUpstreamHeaders. +// resend, when set, is the answer to each Resend, and heads keeps what it was shown. +type translator struct { + target string + translate bool + headers http.Header + resend bool + + mu sync.Mutex + heads []string +} + +func (p *translator) Name() string { return "translator" } +func (p *translator) Capabilities() pipeline.PluginCapabilities { + return pipeline.PluginCapabilities{WritesDestination: true, ReadsBody: true, Description: "test"} +} +func (p *translator) OnRequest(_ context.Context, pctx *pipeline.Context) pipeline.Action { + u, _ := url.Parse(p.target) + if err := pctx.Redirect(u); err != nil { + panic(err) + } + if p.translate { + if err := pctx.Translate("anthropic-messages", "chat-completions", "/v1/chat/completions"); err != nil { + panic(err) + } + } + if p.headers != nil { + if err := pctx.SetUpstreamHeaders(p.headers); err != nil { + panic(err) + } + } + return pipeline.Action{Type: pipeline.Continue} +} +func (p *translator) OnResponse(context.Context, *pipeline.Context) pipeline.Action { + return pipeline.Action{Type: pipeline.Continue} +} + +// resendingTranslator is a translator that is also a Resender. +type resendingTranslator struct{ *translator } + +func (p resendingTranslator) Resend(_ context.Context, _ *pipeline.Context, _ int, body []byte) bool { + p.mu.Lock() + defer p.mu.Unlock() + p.heads = append(p.heads, string(body)) + return p.resend +} + +// upstreamRequest is one request the chat server got. +type upstreamRequest struct { + path, rawQuery string + header http.Header + body []byte +} + +// chatServer serves answer for every request and keeps what it got. +type chatServer struct { + *httptest.Server + mu sync.Mutex + reqs []upstreamRequest +} + +func newChatServer(t *testing.T, answer http.HandlerFunc) *chatServer { + t.Helper() + c := &chatServer{} + c.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + c.mu.Lock() + c.reqs = append(c.reqs, upstreamRequest{r.URL.Path, r.URL.RawQuery, r.Header.Clone(), body}) + c.mu.Unlock() + answer(w, r) + })) + t.Cleanup(c.Close) + return c +} + +func (c *chatServer) got() []upstreamRequest { + c.mu.Lock() + defer c.mu.Unlock() + return append([]upstreamRequest(nil), c.reqs...) +} + +// replyFixture is a hand-written chat reply from the translate package's testdata. +func replyFixture(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile("../../translate/testdata/reply/" + name) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +const anthropicRequest = `{"model":"claude-opus-5-5","max_tokens":1024,"messages":[{"role":"user","content":"is 391 prime?"}]}` + +const anthropicStreamRequest = `{"model":"claude-opus-5-5","max_tokens":1024,"stream":true,"messages":[{"role":"user","content":"is 391 prime?"}]}` + +// postAnthropic sends an Anthropic Messages request through proxy to the agent's +// provider. edit, when set, changes the request before it is sent. +func postAnthropic(t *testing.T, proxy *httptest.Server, body string, edit func(*http.Request)) *http.Response { + t.Helper() + req, _ := http.NewRequest(http.MethodPost, "http://api.anthropic.example/v1/messages", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Anthropic-Version", "2023-06-01") + if edit != nil { + edit(req) + } + resp, err := proxyClient(proxy, nil).Do(req) + if err != nil { + t.Fatalf("request: %v", err) + } + t.Cleanup(func() { _ = resp.Body.Close() }) + return resp +} + +func replyText(t *testing.T, resp *http.Response) string { + t.Helper() + b, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read reply: %v", err) + } + return string(b) +} + +// translatedInvocation is the forward proxy's modify/translated record on a request +// row of store, or nil. +func translatedInvocation(store *session.Store) (*pipeline.SessionEvent, *pipeline.Invocation) { + for _, s := range store.ListSessions() { + v := store.View(s.ID) + if v == nil { + continue + } + for i := range v.Events { + e := &v.Events[i] + if e.Phase != pipeline.SessionRequest || e.Invocations == nil { + continue + } + for j := range e.Invocations.Outbound { + inv := &e.Invocations.Outbound[j] + if inv.Plugin == "forward-proxy" && inv.Action == pipeline.ActionModify && inv.Reason == "translated" { + return e, inv + } + } + } + } + return nil, nil +} + +func newStore(t *testing.T) *session.Store { + store := session.New(5*time.Minute, 100, 0) + t.Cleanup(store.Close) + return store +} + +func TestTranslate_StreamedReplyRecordedAndIncremental(t *testing.T) { + chunks := strings.SplitAfter(replyFixture(t, "bob_shape1.sse"), "\n\n") + release := make(chan struct{}) + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + // The ping comment and the first two chunks, then wait. + _, _ = io.WriteString(w, strings.Join(chunks[:3], "")) + w.(http.Flusher).Flush() + <-release + _, _ = io.WriteString(w, strings.Join(chunks[3:], "")) + }) + store := newStore(t) + proxy := newRedirectProxy(t, store, nil, inferenceparser.NewInferenceParser(), &translator{target: up.URL, translate: true}) + + resp := postAnthropic(t, proxy, anthropicStreamRequest, nil) + if ct := resp.Header.Get("Content-Type"); !strings.HasPrefix(ct, "text/event-stream") { + t.Fatalf("Content-Type = %q, want text/event-stream", ct) + } + br := bufio.NewReader(resp.Body) + done := make(chan string, 1) + go func() { + for { + line, err := br.ReadString('\n') + if line == "event: message_start\n" { + done <- line + return + } + if err != nil { + done <- "" + return + } + } + }() + select { + case got := <-done: + if got == "" { + t.Fatal("the reply ended with no message_start") + } + case <-time.After(5 * time.Second): + close(release) + t.Fatal("message_start did not reach the client before the upstream finished") + } + close(release) + rest, _ := io.ReadAll(br) + if !strings.Contains(string(rest), "event: message_stop") { + t.Errorf("the reply has no message_stop: %q", rest) + } + + // The request on the wire is chat-shaped and has no Anthropic headers. + reqs := up.got() + if len(reqs) != 1 || reqs[0].path != "/v1/chat/completions" { + t.Fatalf("upstream requests = %+v, want one to /v1/chat/completions", reqs) + } + if reqs[0].header.Get("Anthropic-Version") != "" { + t.Errorf("upstream saw Anthropic-Version %q", reqs[0].header.Get("Anthropic-Version")) + } + var chatBody struct { + Messages []json.RawMessage `json:"messages"` + } + if err := json.Unmarshal(reqs[0].body, &chatBody); err != nil || len(chatBody.Messages) == 0 { + t.Errorf("upstream body is not a chat request: %s", reqs[0].body) + } + + // The response row has the Anthropic inference. + deadline := time.Now().Add(5 * time.Second) + var inf *pipeline.InferenceExtension + for time.Now().Before(deadline) { + if inf = lastInferenceResponse(t, store); inf != nil && inf.Completion != "" { + break + } + time.Sleep(10 * time.Millisecond) + } + if inf == nil || inf.Completion != "391 is not prime" { + t.Fatalf("response inference = %+v, want completion %q", inf, "391 is not prime") + } + if len(inf.ToolCalls) != 1 || inf.ToolCalls[0].Name != "Bash" { + t.Errorf("tool calls = %+v, want one Bash", inf.ToolCalls) + } + + _, inv := translatedInvocation(store) + if inv == nil { + t.Fatal("no modify/translated invocation on the request row") + } + want := map[string]string{"from": "anthropic-messages", "to": "chat-completions", "path": "/v1/chat/completions"} + for k, v := range want { + if inv.Details[k] != v { + t.Errorf("Details[%q] = %q, want %q", k, inv.Details[k], v) + } + } + if _, ok := inv.Details["dropped"]; ok { + t.Errorf("Details has dropped %q for a request that dropped nothing", inv.Details["dropped"]) + } +} + +func TestTranslate_NoQueryString(t *testing.T) { + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, replyFixture(t, "bob_shape1.json")) + }) + store := newStore(t) + proxy := newRedirectProxy(t, store, nil, &translator{target: up.URL, translate: true}) + resp := postAnthropic(t, proxy, anthropicRequest, func(r *http.Request) { r.URL.RawQuery = "beta=true" }) + _ = replyText(t, resp) + + reqs := up.got() + if len(reqs) != 1 || reqs[0].path != "/v1/chat/completions" || reqs[0].rawQuery != "" { + t.Fatalf("upstream requests = %+v, want /v1/chat/completions with no query", reqs) + } + ev, _ := translatedInvocation(store) + if ev == nil || ev.HTTPPath != "/v1/messages" { + t.Fatalf("request row = %+v, want path /v1/messages", ev) + } +} + +func TestTranslate_UpstreamHeadersReplace(t *testing.T) { + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, replyFixture(t, "bob_shape1.json")) + }) + proxy := newRedirectProxy(t, nil, nil, &translator{target: up.URL, translate: true, + headers: http.Header{"User-Agent": {"bob-shell"}}}) + resp := postAnthropic(t, proxy, anthropicRequest, func(r *http.Request) { r.Header.Set("User-Agent", "claude-cli/2") }) + _ = replyText(t, resp) + + reqs := up.got() + if len(reqs) != 1 { + t.Fatalf("upstream got %d requests, want 1", len(reqs)) + } + if got := reqs[0].header.Values("User-Agent"); len(got) != 1 || got[0] != "bob-shell" { + t.Errorf("upstream User-Agent = %q, want only bob-shell", got) + } +} + +func TestTranslate_GzipUpstream(t *testing.T) { + up := newChatServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + _, _ = io.WriteString(w, replyFixture(t, "bob_shape1.json")) + return + } + w.Header().Set("Content-Encoding", "gzip") + zw := gzip.NewWriter(w) + _, _ = io.WriteString(zw, replyFixture(t, "bob_shape1.json")) + _ = zw.Close() + }) + proxy := newRedirectProxy(t, nil, nil, &translator{target: up.URL, translate: true}) + resp := postAnthropic(t, proxy, anthropicRequest, func(r *http.Request) { r.Header.Set("Accept-Encoding", "gzip") }) + body := replyText(t, resp) + + if ce := resp.Header.Get("Content-Encoding"); ce != "" { + t.Errorf("Content-Encoding = %q, want none", ce) + } + var msg struct { + Type string `json:"type"` + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + } + if err := json.Unmarshal([]byte(body), &msg); err != nil || msg.Type != "message" { + t.Fatalf("reply is not an Anthropic message (%v): %q", err, body) + } + if !strings.Contains(body, "391 is not prime") { + t.Errorf("reply has no text: %s", body) + } +} + +func TestTranslate_AgentStreamsServerAnswersJSON(t *testing.T) { + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, replyFixture(t, "bob_shape1.json")) + }) + proxy := newRedirectProxy(t, nil, nil, &translator{target: up.URL, translate: true}) + resp := postAnthropic(t, proxy, anthropicStreamRequest, nil) + body := replyText(t, resp) + + if ct := resp.Header.Get("Content-Type"); !strings.HasPrefix(ct, "text/event-stream") { + t.Errorf("Content-Type = %q, want text/event-stream", ct) + } + if !strings.HasPrefix(body, "event: message_start\n") || !strings.Contains(body, "event: message_stop") { + t.Errorf("reply is not an Anthropic stream: %q", body) + } +} + +func TestTranslate_ErrorStatus(t *testing.T) { + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = io.WriteString(w, replyFixture(t, "error_429_openai.json")) + }) + proxy := newRedirectProxy(t, nil, nil, &translator{target: up.URL, translate: true}) + resp := postAnthropic(t, proxy, anthropicStreamRequest, nil) + body := replyText(t, resp) + + if resp.StatusCode != http.StatusTooManyRequests { + t.Errorf("status = %d, want 429", resp.StatusCode) + } + var e struct { + Type string `json:"type"` + Error struct { + Type string `json:"type"` + } `json:"error"` + } + if err := json.Unmarshal([]byte(body), &e); err != nil || e.Type != "error" || e.Error.Type != "rate_limit_error" { + t.Errorf("reply = %q, want an Anthropic rate_limit_error body", body) + } +} + +func TestTranslate_DocumentRefused(t *testing.T) { + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) {}) + store := newStore(t) + proxy := newRedirectProxy(t, store, nil, &translator{target: up.URL, translate: true}) + doc := `{"model":"claude-opus-5-5","max_tokens":1024,"messages":[{"role":"user","content":[` + + `{"type":"document","source":{"type":"base64","media_type":"application/pdf","data":"JVBERi0="}}]}]}` + resp := postAnthropic(t, proxy, doc, nil) + body := replyText(t, resp) + + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("status = %d, want 400", resp.StatusCode) + } + if !strings.Contains(body, `"invalid_request_error"`) || !strings.Contains(body, "documents") { + t.Errorf("reply = %q, want an invalid_request_error that names documents", body) + } + if n := len(up.got()); n != 0 { + t.Errorf("upstream got %d requests, want 0", n) + } +} + +func TestTranslate_DroppedRecorded(t *testing.T) { + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, replyFixture(t, "bob_shape1.json")) + }) + store := newStore(t) + proxy := newRedirectProxy(t, store, nil, &translator{target: up.URL, translate: true}) + req := `{"model":"claude-opus-5-5","max_tokens":1024,"top_k":5,"metadata":{"user_id":"u"},` + + `"messages":[{"role":"user","content":"hi"}]}` + _ = replyText(t, postAnthropic(t, proxy, req, nil)) + + _, inv := translatedInvocation(store) + if inv == nil { + t.Fatal("no modify/translated invocation") + } + if got := inv.Details["dropped"]; got != "metadata,top_k" { + t.Errorf("dropped = %q, want metadata,top_k", got) + } +} + +func TestTranslate_ResendSeesRawReply(t *testing.T) { + var calls int + var mu sync.Mutex + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + calls++ + n := calls + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + if n == 1 { + w.WriteHeader(http.StatusForbidden) + _, _ = io.WriteString(w, replyFixture(t, "error_403_litellm.json")) + return + } + _, _ = io.WriteString(w, replyFixture(t, "bob_shape1.json")) + }) + tr := &translator{target: up.URL, translate: true, resend: true} + store := newStore(t) + proxy := newRedirectProxy(t, store, nil, resendingTranslator{tr}) + resp := postAnthropic(t, proxy, anthropicRequest, nil) + body := replyText(t, resp) + + if resp.StatusCode != http.StatusOK || !strings.Contains(body, "391 is not prime") { + t.Errorf("reply = %d %q, want the second answer, translated", resp.StatusCode, body) + } + tr.mu.Lock() + heads := append([]string(nil), tr.heads...) + tr.mu.Unlock() + if len(heads) != 1 || !strings.Contains(heads[0], "team_model_access_denied") { + t.Errorf("Resender saw %q, want the raw 403 body", heads) + } + reqs := up.got() + if len(reqs) != 2 { + t.Fatalf("upstream got %d requests, want 2", len(reqs)) + } + for i, r := range reqs { + if r.path != "/v1/chat/completions" || !bytes.Contains(r.body, []byte(`"messages":[{"role":"user","content":[`)) { + t.Errorf("upstream request %d = %s %s, want a chat body", i, r.path, r.body) + } + } +} + +func TestTranslate_NotTranslatedPassesBytes(t *testing.T) { + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "ok") + }) + proxy := newRedirectProxy(t, nil, nil, &translator{target: up.URL}) + _ = replyText(t, postAnthropic(t, proxy, anthropicRequest, nil)) + + reqs := up.got() + if len(reqs) != 1 || string(reqs[0].body) != anthropicRequest || reqs[0].path != "/v1/messages" { + t.Fatalf("upstream requests = %+v, want the client body unchanged", reqs) + } +} From 8e2d91e740a9d886dff48beee30b618904b7f910 Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Tue, 17 Nov 2026 15:59:09 -0500 Subject: [PATCH 04/15] feat: Capture the translated request and the raw server reply For a translated request, the lossless capture now keeps four bodies (rule 8a of #1911), so you can find a conversion bug from the archive: - RequestCapture.Translated is the converted body as sent upstream. - ResponseCapture.Upstream is the server's reply before the conversion. Body stays the converted reply, as the agent got it. The archive, GET raw (translated, upstream and their Base64 forms) and the ingest line carry both fields. The archive counts them in the capture queue's size. agentop's raw view shows them with their own labels, not as a plugin rewrite. A resend records the new converted body on a copy of the capture. Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- cmd/agentop/tui/detail_raw.go | 19 +++++ cmd/agentop/tui/detail_raw_test.go | 28 +++++++ core/listener/forwardproxy/server.go | 26 ++++-- core/listener/forwardproxy/translate.go | 12 ++- core/listener/forwardproxy/translate_test.go | 46 +++++++++++ core/model/capture.go | 31 +++++-- core/session/archive/archive.go | 4 +- core/session/archive/capture.go | 86 ++++++++++++++------ core/session/archive/capture_test.go | 40 +++++++++ core/sessionapi/archive_test.go | 43 ++++++++++ core/sessionapi/ingest.go | 34 ++++---- core/sessionapi/ingest_test.go | 25 ++++++ core/sessionapi/raw.go | 17 +++- docs/session-api.md | 15 ++-- 14 files changed, 366 insertions(+), 60 deletions(-) diff --git a/cmd/agentop/tui/detail_raw.go b/cmd/agentop/tui/detail_raw.go index 2aed1d1e7..cdae49551 100644 --- a/cmd/agentop/tui/detail_raw.go +++ b/cmd/agentop/tui/detail_raw.go @@ -56,6 +56,17 @@ func withRaw(data []byte, raw *pipeline.RawCapture, e *pipeline.SessionEvent) [] case p.Body != "": resp["body"] = bodyView(p.Body, nil) } + if p.Upstream != nil || p.UpstreamBase64 != nil { + text := "" + if p.Upstream != nil { + text = *p.Upstream + } + resp["upstream"] = "server's reply before translation: " + formatBytes(int64(len(text)+len(p.UpstreamBase64))) + resp["upstreamBody"] = bodyView(text, p.UpstreamBase64) + if p.UpstreamTruncated { + resp["upstreamTruncated"] = true + } + } r["response"] = resp } else { r["request"] = bodyView(raw.Request, raw.RequestBase64) @@ -69,6 +80,14 @@ func withRaw(data []byte, raw *pipeline.RawCapture, e *pipeline.SessionEvent) [] default: r["forwarded"] = "forwarded as sent" } + if raw.Translated != nil || raw.TranslatedBase64 != nil { + text := "" + if raw.Translated != nil { + text = *raw.Translated + } + r["translated"] = "sent upstream, translated by Cortex: " + formatBytes(int64(len(text)+len(raw.TranslatedBase64))) + r["translatedBody"] = bodyView(text, raw.TranslatedBase64) + } if len(raw.RequestHeaders) > 0 { m["headers"] = flatHeaders(raw.RequestHeaders) } diff --git a/cmd/agentop/tui/detail_raw_test.go b/cmd/agentop/tui/detail_raw_test.go index badd54e24..98c6f455c 100644 --- a/cmd/agentop/tui/detail_raw_test.go +++ b/cmd/agentop/tui/detail_raw_test.go @@ -215,3 +215,31 @@ func TestDetailRaw_ReopenShowsTheHeldCapture(t *testing.T) { t.Fatalf("the reopened row shows raw %+v, want the held capture", m.detailRow.raw) } } + +// A translated request shows the body Cortex sent upstream, and its reply the server's reply +// before translation, each with its own label: neither is a plugin's rewrite. +func TestDetailRaw_ShowsTheTranslatedHop(t *testing.T) { + tr := `{"model":"c","messages":[{"role":"user","content":[{"type":"text","text":"translated text"}]}]}` + out := rawDetail(t, &pipeline.RawCapture{Request: `{"model":"m"}`, Translated: &tr}) + for _, want := range []string{"sent upstream, translated by Cortex: 95B", "translatedBody", "translated text", "forwarded as sent"} { + if !strings.Contains(out, want) { + t.Errorf("request detail lacks %q:\n%s", want, out) + } + } + + ev := &pipeline.SessionEvent{Seq: 4, Phase: pipeline.SessionResponse, Direction: pipeline.Outbound, Host: "h", + Inference: &pipeline.InferenceExtension{Model: "m"}} + m := detailModel(t, "s1", ev) + up := `{"choices":[{"message":{"content":"chat reply"}}]}` + m.applyDetailEvent(detailEventLoadedMsg{sessionID: "s1", seq: 4, event: ev, raw: &pipeline.RawCapture{ + Response: &pipeline.RawBody{Status: 200, Body: `{}`, Upstream: &up}}}) + m.detailFolds = map[string]foldChoice{"response.raw": {open: true}, "response.raw.response": {open: true}, + "response.raw.response.upstreamBody": {open: true}} + m.renderDetail() + got := stripANSI(strings.Join(lineTexts(m.detailLines), "\n")) + for _, want := range []string{"server's reply before translation: 50B", "upstreamBody"} { + if !strings.Contains(got, want) { + t.Errorf("response detail lacks %q:\n%s", want, got) + } + } +} diff --git a/core/listener/forwardproxy/server.go b/core/listener/forwardproxy/server.go index 3f95c4252..1168f330c 100644 --- a/core/listener/forwardproxy/server.go +++ b/core/listener/forwardproxy/server.go @@ -678,6 +678,9 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne httpx.WriteRejectionForRequest(w, *refusal, pctx) return } + if tr != nil && capture != nil { + capture.Translated = tr.conv.Body + } } // A WebSocket upgrade is the one hop-by-hop request this proxy forwards @@ -925,10 +928,18 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne } defer resp.Body.Close() + // The session archive's capture of the response (#1494), only when the request was + // captured, so a response never outlives its pair: the request row keeps its capture + // only for inference (recordCapturedRequest). + var rc *pipeline.ResponseCapture + if capture != nil && pctx.Extensions.Inference != nil { + rc = &pipeline.ResponseCapture{Status: resp.StatusCode, Decoded: resp.Uncompressed} + } + // Before any reply plugin or recorder reads the body: they all see the reply in // the agent's protocol. A hop's own 503 returned above and is not converted. if tr != nil { - tr.wrapReply(resp) + tr.wrapReply(resp, rc) } // Response phase: populate pctx and run plugins in reverse order. @@ -962,10 +973,9 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne // // The same wrapper keeps the body for the session archive's capture (#1494): every arm // reads through it, so what it keeps is the body as it came off the wire, whatever arm - // relays it. Only when the request was captured, so a response never outlives its pair: the - // request row keeps its capture only for inference (recordCapturedRequest). - if capture != nil && pctx.Extensions.Inference != nil { - rc := &pipeline.ResponseCapture{Status: resp.StatusCode, Header: redact.Headers(resp.Header), Decoded: resp.Uncompressed} + // relays it. For a translated reply that is the reply in the agent's protocol. + if rc != nil { + rc.Header = redact.Headers(resp.Header) pctx.ResponseCapture = rc resp.Body = bodycount.WrapCapture(resp.Body, &pctx.ResponseBytes, &rc.Body, &rc.Truncated, &rc.Ended, pipeline.MaxResponseCapture) } else { @@ -1333,6 +1343,12 @@ func (s *Server) resendIfAsked(r *http.Request, pctx *pipeline.Context, client * s.recordOutboundResponseEvent(pctx, resp.StatusCode, nil) pctx.StatusCode, pctx.ResponseBody, pctx.Extensions.Inference = 0, nil, resent pctx.RenewRequestID() + if tr != nil && capture != nil { + // A copy: the refused attempt's row keeps the body it sent. + again := *capture + again.Translated = conv.Body + capture = &again + } s.recordCapturedRequest(tl, pctx, pctx.OutboundSessionID, capture) } again := r.Clone(r.Context()) diff --git a/core/listener/forwardproxy/translate.go b/core/listener/forwardproxy/translate.go index 8550cf5d4..c6904cc8a 100644 --- a/core/listener/forwardproxy/translate.go +++ b/core/listener/forwardproxy/translate.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/rossoctl/cortex/core/listener/httpx" + "github.com/rossoctl/cortex/core/listener/internal/bodycount" "github.com/rossoctl/cortex/core/pipeline" "github.com/rossoctl/cortex/core/translate" "github.com/rossoctl/cortex/core/translate/pivot" @@ -115,8 +116,15 @@ func (t *translation) agentStream(status int) bool { } // wrapReply makes resp carry the reply in the agent's protocol, converted as it is -// read (rule 10). Every later arm then reads the converted bytes. -func (t *translation) wrapReply(resp *http.Response) { +// read (rule 10). Every later arm then reads the converted bytes. When rc is not nil, +// it keeps the server's reply before the conversion in rc.Upstream (rule 8a). +func (t *translation) wrapReply(resp *http.Response, rc *pipeline.ResponseCapture) { + if rc != nil { + // The count and the end are the converted body's, which the caller's wrapper keeps. + var unused int64 + var ended bool + resp.Body = bodycount.WrapCapture(resp.Body, &unused, &rc.Upstream, &rc.UpstreamTruncated, &ended, pipeline.MaxResponseCapture) + } stream := t.agentStream(resp.StatusCode) dec := t.dec.NewReplyDecoder(resp.StatusCode, httpx.IsEventStream(resp.Header.Get("Content-Type"))) enc := t.enc.NewReplyEncoder(stream) diff --git a/core/listener/forwardproxy/translate_test.go b/core/listener/forwardproxy/translate_test.go index 486920e82..f463197e3 100644 --- a/core/listener/forwardproxy/translate_test.go +++ b/core/listener/forwardproxy/translate_test.go @@ -477,3 +477,49 @@ func TestTranslate_NotTranslatedPassesBytes(t *testing.T) { t.Fatalf("upstream requests = %+v, want the client body unchanged", reqs) } } + +// Rule 8a: the capture of a translated request holds four bodies, so a conversion bug +// can be found from the archive: what the agent sent, what the plugins made of it (the +// input of the conversion), the chat body sent upstream, and the server's raw reply +// beside the Anthropic reply that the agent got. +func TestTranslate_CaptureHoldsFourBodies(t *testing.T) { + sse := replyFixture(t, "bob_shape1.sse") + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, sse) + }) + rewritten := strings.Replace(anthropicStreamRequest, "391", "393", 1) + proxy, rec := proxyWithCaptures(t, inferenceparser.NewInferenceParser(), + &translator{target: up.URL, translate: true}, &bodyMutatorPlugin{newBody: []byte(rewritten)}) + + resp := postAnthropic(t, proxy, anthropicStreamRequest, nil) + reply := replyText(t, resp) + if !strings.Contains(reply, "event: message_stop") { + t.Fatalf("reply = %q, want an Anthropic stream", reply) + } + + res := rec.responses() + reqs := rec.all() + if len(reqs) != 1 || len(res) != 1 { + t.Fatalf("%d request and %d response captures, want 1 and 1", len(reqs), len(res)) + } + c := reqs[0] + if string(c.Original) != anthropicStreamRequest { + t.Errorf("Original = %q, want the agent's request", c.Original) + } + if string(c.Forwarded) != rewritten { + t.Errorf("Forwarded = %q, want the plugin's body", c.Forwarded) + } + sent := up.got() + if len(sent) != 1 || !bytes.Equal(c.Translated, sent[0].body) || !strings.Contains(string(c.Translated), "is 393 prime") { + t.Errorf("Translated = %q, want the chat body sent upstream", c.Translated) + } + for _, r := range res { + if string(r.Body) != reply { + t.Errorf("Response.Body = %q, want the Anthropic reply the agent got", r.Body) + } + if string(r.Upstream) != sse || r.UpstreamTruncated { + t.Errorf("Response.Upstream = %q (truncated %v), want the server's chat stream", r.Upstream, r.UpstreamTruncated) + } + } +} diff --git a/core/model/capture.go b/core/model/capture.go index 87f0e9937..47d03007e 100644 --- a/core/model/capture.go +++ b/core/model/capture.go @@ -12,6 +12,10 @@ type RequestCapture struct { Original []byte // Forwarded is the body as Cortex sent it upstream, nil when no plugin rewrote it. Forwarded []byte + // Translated is the body as Cortex sent it upstream after it converted it to the + // server's protocol, nil when it did not convert it. Forwarded, or Original when + // Forwarded is nil, is the input of the conversion. + Translated []byte // Header is the request's headers. The producer removes credential headers before it // hands the capture over: the archive stores what it is given. Header http.Header @@ -37,6 +41,12 @@ type ResponseCapture struct { Decoded bool // Truncated is set when the body was longer than MaxResponseCapture. Truncated bool + // Upstream is the server's reply before Cortex converted it to the agent's protocol, up + // to MaxResponseCapture bytes, nil when Cortex did not convert the reply. Body is then the + // converted reply, as the agent got it. + Upstream []byte + // UpstreamTruncated is set when Upstream was longer than MaxResponseCapture. + UpstreamTruncated bool // Incomplete is set when the body did not arrive whole: the stream broke, or the client // went away first. Incomplete bool @@ -54,13 +64,16 @@ const MaxResponseCapture = 10 << 20 // text when it is valid UTF-8, which every JSON API body is, and base64 in the matching *Base64 // field otherwise, so any body survives the trip. type RawCapture struct { - Request string `json:"request,omitempty"` - RequestBase64 []byte `json:"requestBase64,omitempty"` - Forwarded *string `json:"forwarded,omitempty"` - ForwardedBase64 []byte `json:"forwardedBase64,omitempty"` - RequestHeaders http.Header `json:"requestHeaders,omitempty"` - Query string `json:"query,omitempty"` - Response *RawBody `json:"response,omitempty"` + Request string `json:"request,omitempty"` + RequestBase64 []byte `json:"requestBase64,omitempty"` + Forwarded *string `json:"forwarded,omitempty"` + ForwardedBase64 []byte `json:"forwardedBase64,omitempty"` + // Translated is the body after Cortex converted it to the server's protocol. + Translated *string `json:"translated,omitempty"` + TranslatedBase64 []byte `json:"translatedBase64,omitempty"` + RequestHeaders http.Header `json:"requestHeaders,omitempty"` + Query string `json:"query,omitempty"` + Response *RawBody `json:"response,omitempty"` } // RawBody is a captured response. Body is in its Content-Encoding unless Decoded is set, so a @@ -73,4 +86,8 @@ type RawBody struct { Decoded bool `json:"decoded,omitempty"` Truncated bool `json:"truncated,omitempty"` Incomplete bool `json:"incomplete,omitempty"` + // Upstream is the server's reply before Cortex converted it to the agent's protocol. + Upstream *string `json:"upstream,omitempty"` + UpstreamBase64 []byte `json:"upstreamBase64,omitempty"` + UpstreamTruncated bool `json:"upstreamTruncated,omitempty"` } diff --git a/core/session/archive/archive.go b/core/session/archive/archive.go index 0de17e37e..15ea3130f 100644 --- a/core/session/archive/archive.go +++ b/core/session/archive/archive.go @@ -375,9 +375,9 @@ func (a *Archive) RecordCapture(sessionID string, e *model.SessionEvent, c *mode // body read with io.ReadAll can hold up to twice that, and headers are not counted, so the // budget bounds memory within a small factor, not exactly. func captureSize(c *model.RequestCapture) int64 { - n := len(c.Original) + len(c.Forwarded) + n := len(c.Original) + len(c.Forwarded) + len(c.Translated) if c.Response != nil { - n += len(c.Response.Body) + n += len(c.Response.Body) + len(c.Response.Upstream) } return int64(n) } diff --git a/core/session/archive/capture.go b/core/session/archive/capture.go index 75ae020d6..d691d9139 100644 --- a/core/session/archive/capture.go +++ b/core/session/archive/capture.go @@ -27,15 +27,17 @@ import ( // construction. The SHA-256 of the original is stored too, and a rebuild that does not match it // is refused. Exact bytes matter: the prompt cache and thinking signatures are sensitive to them. -// captureRec is a request capture on its event record: the skeletons of the original and the -// forwarded body, and the request's headers and query. Old readers ignore the key. +// captureRec is a request capture on its event record: the skeletons of the original, the +// forwarded and the translated body, and the request's headers and query. Old readers ignore the key. // // Headers and the query are not hashed, so they must be exact by construction. JSON strings // cannot hold invalid UTF-8, which Go's server accepts in a header value, so a header set or // query that holds any is written as bytes instead: HX and QX, base64 on disk. type captureRec struct { - O *bodyRec `json:"o,omitempty"` - W *bodyRec `json:"w,omitempty"` + O *bodyRec `json:"o,omitempty"` + W *bodyRec `json:"w,omitempty"` + // X is the translated body (#1911): the request after Cortex converted it to the server's protocol. + X *bodyRec `json:"x,omitempty"` H http.Header `json:"h,omitempty"` HX map[string][][]byte `json:"hx,omitempty"` Q string `json:"q,omitempty"` @@ -47,7 +49,8 @@ type captureRec struct { } // responseRec is a ResponseCapture on disk. B is the key of the block holding the body, empty for -// an empty body; S is the hex SHA-256 of the body, checked on read. +// an empty body; S is the hex SHA-256 of the body, checked on read. UB, US and UT are the same +// for the server's reply before translation (#1911); US is empty when there is none. type responseRec struct { C int `json:"c"` H http.Header `json:"h,omitempty"` @@ -57,6 +60,9 @@ type responseRec struct { T bool `json:"t,omitempty"` IC bool `json:"ic,omitempty"` D bool `json:"d,omitempty"` + UB string `json:"ub,omitempty"` + US string `json:"us,omitempty"` + UT bool `json:"ut,omitempty"` } // bodyRec is one body's skeleton. P alternates literal bytes and block references, starting and @@ -184,13 +190,11 @@ func splitCapture(c *model.RequestCapture, seen map[string]struct{}) (*captureRe T: r.Truncated, IC: r.Incomplete, D: r.Decoded}} rec.R.H, rec.R.HX = headerRec(r.Header) var fresh []stringRec - if len(r.Body) > 0 { - v := string(r.Body) - rec.R.B = hashOf(v) - if _, ok := seen[rec.R.B]; !ok { - seen[rec.R.B] = struct{}{} - fresh = append(fresh, stringRec{h: rec.R.B, v: v}) - } + rec.R.B, fresh = wholeBlock(r.Body, seen, fresh) + if r.Upstream != nil { + sum := sha256.Sum256(r.Upstream) + rec.R.US, rec.R.UT = hex.EncodeToString(sum[:]), r.UpstreamTruncated + rec.R.UB, fresh = wholeBlock(r.Upstream, seen, fresh) } return rec, fresh } @@ -205,9 +209,29 @@ func splitCapture(c *model.RequestCapture, seen map[string]struct{}) (*captureRe f.W, more = splitBody(c.Forwarded, seen) fresh = append(fresh, more...) } + if c.Translated != nil { + var more []stringRec + f.X, more = splitBody(c.Translated, seen) + fresh = append(fresh, more...) + } return f, fresh } +// wholeBlock is the key of the one block that holds body, empty for an empty body. It adds the +// block to fresh when seen does not have it yet. +func wholeBlock(body []byte, seen map[string]struct{}, fresh []stringRec) (string, []stringRec) { + if len(body) == 0 { + return "", fresh + } + v := string(body) + h := hashOf(v) + if _, ok := seen[h]; !ok { + seen[h] = struct{}{} + fresh = append(fresh, stringRec{h: h, v: v}) + } + return h, fresh +} + // headerRec is h as a capture record holds it: as is, or as bytes in hx when a value is not valid // UTF-8, which a JSON string cannot hold. func headerRec(h http.Header) (http.Header, map[string][][]byte) { @@ -283,17 +307,18 @@ func readCapture(path string, seq uint64) (*model.RequestCapture, error) { return nil, errors.New("archive: capture record has no body") } want := map[string]struct{}{} - if f.R != nil && f.R.B != "" { - want[f.R.B] = struct{}{} - } - if f.O != nil { - for _, h := range f.O.refs() { - want[h] = struct{}{} + if f.R != nil { + for _, h := range []string{f.R.B, f.R.UB} { + if h != "" { + want[h] = struct{}{} + } } } - if f.W != nil { - for _, h := range f.W.refs() { - want[h] = struct{}{} + for _, b := range []*bodyRec{f.O, f.W, f.X} { + if b != nil { + for _, h := range b.refs() { + want[h] = struct{}{} + } } } blocks := map[string]string{} @@ -310,8 +335,18 @@ func readCapture(path string, seq uint64) (*model.RequestCapture, error) { if hex.EncodeToString(sum[:]) != r.S { return nil, errors.New("archive: response body does not match the captured hash") } - return &model.RequestCapture{Response: &model.ResponseCapture{Status: r.C, Header: headerOf(r.H, r.HX), - Body: body, Truncated: r.T, Incomplete: r.IC, Decoded: r.D}}, nil + rc := &model.ResponseCapture{Status: r.C, Header: headerOf(r.H, r.HX), + Body: body, Truncated: r.T, Incomplete: r.IC, Decoded: r.D} + if r.US != "" { + // Never nil: an empty reply from the server is not "no translation". + rc.Upstream = append([]byte{}, blocks[r.UB]...) + sum := sha256.Sum256(rc.Upstream) + if hex.EncodeToString(sum[:]) != r.US { + return nil, errors.New("archive: upstream response body does not match the captured hash") + } + rc.UpstreamTruncated = r.UT + } + return &model.RequestCapture{Response: rc}, nil } c := &model.RequestCapture{Header: headerOf(f.H, f.HX), Query: f.Q} if f.QX != nil { @@ -326,5 +361,10 @@ func readCapture(path string, seq uint64) (*model.RequestCapture, error) { return nil, err } } + if f.X != nil { + if c.Translated, err = f.X.rebuild(blocks); err != nil { + return nil, err + } + } return c, nil } diff --git a/core/session/archive/capture_test.go b/core/session/archive/capture_test.go index fbec55ac0..d8293567b 100644 --- a/core/session/archive/capture_test.go +++ b/core/session/archive/capture_test.go @@ -666,3 +666,43 @@ func TestCapture_EventReadsSkipResponseBodies(t *testing.T) { t.Fatalf("got %+v, err %v", got, err) } } + +// Rule 8a of #1911: a translated request's converted body and the server's raw reply come back +// byte for byte beside the bodies they were converted from and to, and count in the queue's size. +func TestCapture_TranslatedAndUpstreamRoundTrip(t *testing.T) { + long := strings.Repeat("x", 600) + req := &pipeline.RequestCapture{Original: []byte(`{"model":"m","system":"` + long + `"}`), + Translated: []byte(`{"model":"m","messages":[{"role":"system","content":[{"type":"text","text":"` + long + `"}]}]}`)} + res := &pipeline.RequestCapture{Response: &pipeline.ResponseCapture{Status: 200, Body: []byte(`event: message_stop` + "\n\n"), + Upstream: []byte("data: \xff[DONE]\n\n"), UpstreamTruncated: true}} + plain := &pipeline.RequestCapture{Response: &pipeline.ResponseCapture{Status: 200, Body: []byte(`{}`)}} + w, err := createSegment(filepath.Join(t.TempDir(), "sess"), time.Now()) + if err != nil { + t.Fatal(err) + } + for i, c := range []*pipeline.RequestCapture{req, res, plain} { + if err := w.appendCapture(pipeline.SessionEvent{Seq: uint64(i + 1)}, c, time.Now()); err != nil { + t.Fatal(err) + } + } + w.close() + path := filepath.Join(filepath.Dir(w.f.Name()), w.info().File) + got, err := readCapture(path, 1) + if err != nil || got == nil || !bytes.Equal(got.Original, req.Original) || !bytes.Equal(got.Translated, req.Translated) || got.Forwarded != nil { + t.Fatalf("request: got %v, err %v, want the original and translated bodies", got != nil, err) + } + got, err = readCapture(path, 2) + if err != nil || got == nil || got.Response == nil || !bytes.Equal(got.Response.Body, res.Response.Body) || + !bytes.Equal(got.Response.Upstream, res.Response.Upstream) || !got.Response.UpstreamTruncated { + t.Fatalf("response: got %v, err %v, want the body and the upstream body", got != nil, err) + } + if got, err = readCapture(path, 3); err != nil || got.Response.Upstream != nil || got.Response.UpstreamTruncated { + t.Fatalf("a reply that was not converted: got %+v, err %v, want no upstream", got, err) + } + if n := captureSize(req); n != int64(len(req.Original)+len(req.Translated)) { + t.Errorf("captureSize(request) = %d, want the translated body counted", n) + } + if n := captureSize(res); n != int64(len(res.Response.Body)+len(res.Response.Upstream)) { + t.Errorf("captureSize(response) = %d, want the upstream body counted", n) + } +} diff --git a/core/sessionapi/archive_test.go b/core/sessionapi/archive_test.go index 4b127a3c0..6d0be17d5 100644 --- a/core/sessionapi/archive_test.go +++ b/core/sessionapi/archive_test.go @@ -347,3 +347,46 @@ func TestHandleGetEvent_RawAddsTheCapture(t *testing.T) { } } } + +// Rule 8a of #1911: GET raw serves a translated request's converted body and the server's raw +// reply, each as text when it is UTF-8 and as base64 otherwise. +func TestHandleGetEvent_RawAddsTheTranslatedHop(t *testing.T) { + root := t.TempDir() + a, err := archive.Open(root) + if err != nil { + t.Fatal(err) + } + store := session.New(0, 0, 0) + store.AddRecorder(a) + store.AppendCapture("s1", pipeline.SessionEvent{At: time.Now(), Phase: pipeline.SessionRequest}, + &pipeline.RequestCapture{Original: []byte(`{"model":"m"}`), Translated: []byte(`{"model":"c"}`)}) + raw := []byte("data: \xff\n\n") + store.AppendCapture("s1", pipeline.SessionEvent{At: time.Now(), Phase: pipeline.SessionResponse}, + &pipeline.RequestCapture{Response: &pipeline.ResponseCapture{Status: 200, Body: []byte(`{}`), Upstream: raw, UpstreamTruncated: true}}) + if err := a.Close(); err != nil { + t.Fatal(err) + } + if a, err = archive.Open(root); err != nil { + t.Fatal(err) + } + defer a.Close() + ts := httptest.NewServer(New(":0", store, WithArchive(a)).server.Handler) + defer ts.Close() + + var got struct { + Raw *pipeline.RawCapture `json:"raw"` + } + code, body := getBody(t, ts.URL+"/v1/sessions/s1/events/1?raw=1") + if code != http.StatusOK || json.Unmarshal(body, &got) != nil || got.Raw == nil || + got.Raw.Translated == nil || *got.Raw.Translated != `{"model":"c"}` || got.Raw.TranslatedBase64 != nil { + t.Fatalf("GET events/1?raw=1 = %d %s, want the translated body as text", code, body) + } + got.Raw = nil + code, body = getBody(t, ts.URL+"/v1/sessions/s1/events/2?raw=1") + if code != http.StatusOK || json.Unmarshal(body, &got) != nil || got.Raw == nil || got.Raw.Response == nil { + t.Fatalf("GET events/2?raw=1 = %d %s", code, body) + } + if r := got.Raw.Response; r.Body != `{}` || r.Upstream != nil || !bytes.Equal(r.UpstreamBase64, raw) || !r.UpstreamTruncated { + t.Fatalf("response = %+v, want the non-UTF-8 upstream body as base64", r) + } +} diff --git a/core/sessionapi/ingest.go b/core/sessionapi/ingest.go index 93d9b0301..b2913f7e2 100644 --- a/core/sessionapi/ingest.go +++ b/core/sessionapi/ingest.go @@ -245,22 +245,25 @@ type ingestCost struct { // ingestCapture is the wire form of model.RequestCapture, which has no JSON tags. type ingestCapture struct { - Original []byte `json:"original"` - Forwarded []byte `json:"forwarded"` - Header http.Header `json:"header"` - Query string `json:"query"` - Response *ingestResponse `json:"response"` - Dropped bool `json:"dropped"` + Original []byte `json:"original"` + Forwarded []byte `json:"forwarded"` + Translated []byte `json:"translated"` + Header http.Header `json:"header"` + Query string `json:"query"` + Response *ingestResponse `json:"response"` + Dropped bool `json:"dropped"` } type ingestResponse struct { - Status int `json:"status"` - Header http.Header `json:"header"` - Body []byte `json:"body"` - Decoded bool `json:"decoded"` - Truncated bool `json:"truncated"` - Ended bool `json:"ended"` - Incomplete bool `json:"incomplete"` + Status int `json:"status"` + Header http.Header `json:"header"` + Body []byte `json:"body"` + Decoded bool `json:"decoded"` + Truncated bool `json:"truncated"` + Ended bool `json:"ended"` + Incomplete bool `json:"incomplete"` + Upstream []byte `json:"upstream"` + UpstreamTruncated bool `json:"upstreamTruncated"` } // handleIngest is registered behind httpx.LoopbackOnly (see New), also when the rest of the @@ -465,7 +468,7 @@ func (in *ingester) capture(l ingestLine, ev *model.SessionEvent) *model.Request defer in.reqCapsMu.Unlock() if c.Response == nil { in.reqCaps[l.RequestID] = time.Now() - return &model.RequestCapture{Original: c.Original, Forwarded: c.Forwarded, + return &model.RequestCapture{Original: c.Original, Forwarded: c.Forwarded, Translated: c.Translated, Header: redact.Headers(c.Header), Query: redact.Query(c.Query)} } in.sweepLocked(time.Now()) @@ -475,7 +478,8 @@ func (in *ingester) capture(l ingestLine, ev *model.SessionEvent) *model.Request delete(in.reqCaps, l.RequestID) res := c.Response return &model.RequestCapture{Response: &model.ResponseCapture{Status: res.Status, Header: redact.Headers(res.Header), - Body: res.Body, Decoded: res.Decoded, Truncated: res.Truncated, Ended: res.Ended, Incomplete: res.Incomplete}} + Body: res.Body, Decoded: res.Decoded, Truncated: res.Truncated, Ended: res.Ended, Incomplete: res.Incomplete, + Upstream: res.Upstream, UpstreamTruncated: res.UpstreamTruncated}} } func (s *Server) handleIngestStats(w http.ResponseWriter, r *http.Request) { diff --git a/core/sessionapi/ingest_test.go b/core/sessionapi/ingest_test.go index 12f2c8ba6..73e9baa6a 100644 --- a/core/sessionapi/ingest_test.go +++ b/core/sessionapi/ingest_test.go @@ -623,3 +623,28 @@ func TestIngest_NonStringSourceKeepsTheRow(t *testing.T) { } } } + +// Rule 8a of #1911: an ingested capture keeps a translated request's converted body and the +// server's raw reply. +func TestIngest_CaptureKeepsTheTranslatedHop(t *testing.T) { + r := newIngestRig(t) + r.post(t, nil, + line(t, map[string]any{"capture": map[string]any{ + "original": []byte(`{"a":1}`), "translated": []byte(`{"b":2}`), + }}), + line(t, map[string]any{"event": responseEvent(nil), "capture": map[string]any{ + "response": map[string]any{"status": 200, "body": []byte("{}"), "upstream": []byte("data: [DONE]\n\n"), + "upstreamTruncated": true, "ended": true}, + }}), + ) + req, res := r.caps.get("r1") + if req == nil || res == nil { + t.Fatalf("captures: request %v, response %v", req, res) + } + if string(req.Translated) != `{"b":2}` { + t.Errorf("translated = %q", req.Translated) + } + if string(res.Upstream) != "data: [DONE]\n\n" || !res.UpstreamTruncated { + t.Errorf("upstream = %q, truncated %v", res.Upstream, res.UpstreamTruncated) + } +} diff --git a/core/sessionapi/raw.go b/core/sessionapi/raw.go index 2187ffc34..76b8cdc12 100644 --- a/core/sessionapi/raw.go +++ b/core/sessionapi/raw.go @@ -43,13 +43,28 @@ func rawOf(c *pipeline.RequestCapture) *pipeline.RawCapture { } r.ForwardedBase64 = f64 } + r.Translated, r.TranslatedBase64 = optionalTextOr64(c.Translated) if p := c.Response; p != nil { - r.Response = &pipeline.RawBody{Status: p.Status, Headers: p.Header, Decoded: p.Decoded, Truncated: p.Truncated, Incomplete: p.Incomplete} + r.Response = &pipeline.RawBody{Status: p.Status, Headers: p.Header, Decoded: p.Decoded, Truncated: p.Truncated, + Incomplete: p.Incomplete, UpstreamTruncated: p.UpstreamTruncated} r.Response.Body, r.Response.BodyBase64 = textOr64(p.Body) + r.Response.Upstream, r.Response.UpstreamBase64 = optionalTextOr64(p.Upstream) } return r } +// optionalTextOr64 is textOr64 for a body that can be absent: nil and nil for a nil body. +func optionalTextOr64(b []byte) (*string, []byte) { + if b == nil { + return nil, nil + } + t, t64 := textOr64(b) + if t64 != nil { + return nil, t64 + } + return &t, nil +} + func textOr64(b []byte) (string, []byte) { if utf8.Valid(b) { return string(b), nil diff --git a/docs/session-api.md b/docs/session-api.md index b4b873726..e803e3f45 100644 --- a/docs/session-api.md +++ b/docs/session-api.md @@ -165,15 +165,19 @@ second or so before the archive flushes a new event. A request event's `raw`: |---|---| | `request` | the body as the agent sent it | | `forwarded` | the body as Cortex sent it upstream; absent when no plugin rewrote it | +| `translated` | the body after Cortex converted it to the server's protocol, as sent upstream; absent when Cortex did not convert it. `forwarded`, or `request` when `forwarded` is absent, is the input of the conversion | | `requestHeaders` | the request headers, without credential headers | | `query` | the URL's raw query | A response event's `raw` has `response`: `{status, headers, body, decoded, -truncated, incomplete}`. `body` is up to 10 MiB of the body as it came off the wire, +truncated, incomplete, upstream, upstreamTruncated}`. `body` is up to 10 MiB of the body as it came off the wire, in its `Content-Encoding`, unless `decoded` says the proxy decompressed it to read it for a plugin. `truncated` means it was longer, and `incomplete` means it was not read -to its end: the stream broke off, or the proxy, a plugin or the client stopped it. A body that is not valid UTF-8 — every compressed body — is in -`requestBase64`, `forwardedBase64` or `bodyBase64` instead. An unknown `raw` +to its end: the stream broke off, or the proxy, a plugin or the client stopped it. +When Cortex converted the reply to the agent's protocol, `body` is the converted reply +as the agent got it, and `upstream` is up to 10 MiB of the server's reply before the +conversion (`upstreamTruncated` when it was longer). A body that is not valid UTF-8 — every compressed body — is in +`requestBase64`, `forwardedBase64`, `translatedBase64`, `bodyBase64` or `upstreamBase64` instead. An unknown `raw` value is ignored, so an older proxy answers with the event alone. ## `GET /v1/sessions/{id}/timeline` — agents, tools and hosts on one time axis @@ -637,9 +641,10 @@ there. ```json {"v":1, "schemaVersion":1, "op":"append", "requestId":"…", "startedAt":"", "session":"", "event":{ "...SessionEvent wire JSON..." }, - "capture":{"original":"", "forwarded":null, "header":{"Name":["v"]}, "query":"", + "capture":{"original":"", "forwarded":null, "translated":null, "header":{"Name":["v"]}, "query":"", "response":{"status":200, "header":{}, "body":"", "decoded":true, - "truncated":false, "ended":true, "incomplete":false}}, + "truncated":false, "ended":true, "incomplete":false, + "upstream":null, "upstreamTruncated":false}}, "cost":{"headers":{"X-Litellm-Response-Cost":"0.01", "Content-Type":"application/json"}}} ``` From 79cda289f46014e284718355aac715531bc04e47 Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Tue, 17 Nov 2026 16:04:48 -0500 Subject: [PATCH 05/15] test: Check the capture of each attempt after a resend TestTranslate_ResendSeesRawReply now sets a substitute model on the resend. The test makes sure that the refused row keeps the chat body it sent and that the resent row has the new converted body. Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- core/listener/forwardproxy/translate_test.go | 45 +++++++++++++++++--- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/core/listener/forwardproxy/translate_test.go b/core/listener/forwardproxy/translate_test.go index f463197e3..89a462e8c 100644 --- a/core/listener/forwardproxy/translate_test.go +++ b/core/listener/forwardproxy/translate_test.go @@ -11,6 +11,8 @@ import ( "net/http/httptest" "net/url" "os" + "slices" + "sort" "strings" "sync" "testing" @@ -60,13 +62,28 @@ func (p *translator) OnResponse(context.Context, *pipeline.Context) pipeline.Act return pipeline.Action{Type: pipeline.Continue} } -// resendingTranslator is a translator that is also a Resender. -type resendingTranslator struct{ *translator } +// resendingTranslator is a translator that is also a Resender. When substitute is set, +// each Resend sets the request's model to it. +type resendingTranslator struct { + *translator + substitute string +} + +func (p resendingTranslator) Capabilities() pipeline.PluginCapabilities { + c := p.translator.Capabilities() + c.WritesRequestBody = p.substitute != "" + return c +} -func (p resendingTranslator) Resend(_ context.Context, _ *pipeline.Context, _ int, body []byte) bool { +func (p resendingTranslator) Resend(_ context.Context, pctx *pipeline.Context, _ int, body []byte) bool { p.mu.Lock() defer p.mu.Unlock() p.heads = append(p.heads, string(body)) + if p.substitute != "" { + if err := pctx.SetRequestModel(p.substitute); err != nil { + panic(err) + } + } return p.resend } @@ -440,8 +457,7 @@ func TestTranslate_ResendSeesRawReply(t *testing.T) { _, _ = io.WriteString(w, replyFixture(t, "bob_shape1.json")) }) tr := &translator{target: up.URL, translate: true, resend: true} - store := newStore(t) - proxy := newRedirectProxy(t, store, nil, resendingTranslator{tr}) + proxy, rec := proxyWithCaptures(t, inferenceparser.NewInferenceParser(), resendingTranslator{tr, "claude-sonnet-5"}) resp := postAnthropic(t, proxy, anthropicRequest, nil) body := replyText(t, resp) @@ -463,6 +479,25 @@ func TestTranslate_ResendSeesRawReply(t *testing.T) { t.Errorf("upstream request %d = %s %s, want a chat body", i, r.path, r.body) } } + // Each attempt's row holds the chat body that attempt sent: the refused row the + // first model, the resent row the substitute. + caps := rec.all() + if len(caps) != 2 { + t.Fatalf("%d request captures, want 2", len(caps)) + } + var translated []string + for _, c := range caps { + translated = append(translated, string(c.Translated)) + } + sort.Strings(translated) + want := []string{string(reqs[0].body), string(reqs[1].body)} + sort.Strings(want) + if !slices.Equal(translated, want) || want[0] == want[1] { + t.Errorf("Translated = %q, want the two chat bodies sent upstream, %q", translated, want) + } + if !bytes.Contains(reqs[1].body, []byte(`"model":"claude-sonnet-5"`)) { + t.Errorf("resent body = %s, want the substitute model", reqs[1].body) + } } func TestTranslate_NotTranslatedPassesBytes(t *testing.T) { From 3e2f08334298addaf6e532f9acb0111801064953 Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Tue, 17 Nov 2026 16:13:32 -0500 Subject: [PATCH 06/15] test: Cover the resend branches of a translated request Add tests for each resend branch of a translated request: - The row of the refused attempt has the refusal in Anthropic form (decision 19). - The upstream headers are set again on the resent request. - A changed request that cannot be converted is not sent again. The agent gets the refusal. - A refused translation records a denied row. When the changed request cannot be converted, the answer now goes back with the inference parse of the request that was sent. Before, the row named the model that the Resender wrote, which the upstream did not get. Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- core/listener/forwardproxy/server.go | 4 + core/listener/forwardproxy/translate_test.go | 127 ++++++++++++++++++- 2 files changed, 124 insertions(+), 7 deletions(-) diff --git a/core/listener/forwardproxy/server.go b/core/listener/forwardproxy/server.go index 1168f330c..c73495473 100644 --- a/core/listener/forwardproxy/server.go +++ b/core/listener/forwardproxy/server.go @@ -1316,6 +1316,10 @@ func (s *Server) resendIfAsked(r *http.Request, pctx *pipeline.Context, client * // A request that cannot be converted after the change is not sent again. conv, convErr = tr.convert(pctx) asked = convErr == nil + if !asked { + // The answer goes back for the request sent, so its row names that model. + pctx.Extensions.Inference = sent + } } if !asked { resp.Body = struct { diff --git a/core/listener/forwardproxy/translate_test.go b/core/listener/forwardproxy/translate_test.go index 89a462e8c..2a7e80685 100644 --- a/core/listener/forwardproxy/translate_test.go +++ b/core/listener/forwardproxy/translate_test.go @@ -63,15 +63,17 @@ func (p *translator) OnResponse(context.Context, *pipeline.Context) pipeline.Act } // resendingTranslator is a translator that is also a Resender. When substitute is set, -// each Resend sets the request's model to it. +// each Resend sets the request's model to it. When body is set, each Resend then sets +// the request body to it. type resendingTranslator struct { *translator substitute string + body string } func (p resendingTranslator) Capabilities() pipeline.PluginCapabilities { c := p.translator.Capabilities() - c.WritesRequestBody = p.substitute != "" + c.WritesRequestBody = p.substitute != "" || p.body != "" return c } @@ -84,6 +86,9 @@ func (p resendingTranslator) Resend(_ context.Context, pctx *pipeline.Context, _ panic(err) } } + if p.body != "" { + pctx.SetBody([]byte(p.body)) + } return p.resend } @@ -186,6 +191,37 @@ func translatedInvocation(store *session.Store) (*pipeline.SessionEvent, *pipeli return nil, nil } +// outboundRows are store's outbound rows of phase, in the order they were recorded. +func outboundRows(store *session.Store, phase pipeline.SessionPhase) []pipeline.SessionEvent { + var out []pipeline.SessionEvent + for _, s := range store.ListSessions() { + v := store.View(s.ID) + if v == nil { + continue + } + for _, e := range v.Events { + if e.Direction == pipeline.Outbound && e.Phase == phase { + out = append(out, e) + } + } + } + return out +} + +// waitRows waits for n outbound rows of phase: the last response row can be recorded +// after the client has read the reply. +func waitRows(t *testing.T, store *session.Store, phase pipeline.SessionPhase, n int) []pipeline.SessionEvent { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for { + rows := outboundRows(store, phase) + if len(rows) >= n || time.Now().After(deadline) { + return rows + } + time.Sleep(10 * time.Millisecond) + } +} + func newStore(t *testing.T) *session.Store { store := session.New(5*time.Minute, 100, 0) t.Cleanup(store.Close) @@ -418,6 +454,23 @@ func TestTranslate_DocumentRefused(t *testing.T) { if n := len(up.got()); n != 0 { t.Errorf("upstream got %d requests, want 0", n) } + denied := waitRows(t, store, pipeline.SessionDenied, 1) + if len(denied) != 1 { + t.Fatalf("%d denied rows, want 1", len(denied)) + } + d := denied[0] + if d.StatusCode != http.StatusBadRequest || d.Error == nil || d.Error.Code != "inference.translation_refused" { + t.Errorf("denied row = status %d error %+v, want 400 inference.translation_refused", d.StatusCode, d.Error) + } + var found bool + if d.Invocations != nil { + for _, inv := range d.Invocations.Outbound { + found = found || (inv.Plugin == "forward-proxy" && inv.Action == pipeline.ActionDeny && inv.Reason == "translation_refused") + } + } + if !found { + t.Errorf("denied row invocations = %+v, want forward-proxy deny/translation_refused", d.Invocations) + } } func TestTranslate_DroppedRecorded(t *testing.T) { @@ -440,10 +493,12 @@ func TestTranslate_DroppedRecorded(t *testing.T) { } } -func TestTranslate_ResendSeesRawReply(t *testing.T) { +// refusedThenAnswered serves a LiteLLM 403 to the first request and the chat reply to +// every later one. +func refusedThenAnswered(t *testing.T) *chatServer { var calls int var mu sync.Mutex - up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + return newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { mu.Lock() calls++ n := calls @@ -456,9 +511,16 @@ func TestTranslate_ResendSeesRawReply(t *testing.T) { } _, _ = io.WriteString(w, replyFixture(t, "bob_shape1.json")) }) - tr := &translator{target: up.URL, translate: true, resend: true} - proxy, rec := proxyWithCaptures(t, inferenceparser.NewInferenceParser(), resendingTranslator{tr, "claude-sonnet-5"}) - resp := postAnthropic(t, proxy, anthropicRequest, nil) +} + +func TestTranslate_ResendSeesRawReply(t *testing.T) { + up := refusedThenAnswered(t) + tr := &translator{target: up.URL, translate: true, resend: true, headers: http.Header{"User-Agent": {"bob-shell"}}} + store := newStore(t) + rec := &captures{} + store.AddRecorder(rec) + proxy := newRedirectProxy(t, store, nil, inferenceparser.NewInferenceParser(), resendingTranslator{translator: tr, substitute: "claude-sonnet-5"}) + resp := postAnthropic(t, proxy, anthropicRequest, func(r *http.Request) { r.Header.Set("User-Agent", "claude-cli/2") }) body := replyText(t, resp) if resp.StatusCode != http.StatusOK || !strings.Contains(body, "391 is not prime") { @@ -478,6 +540,10 @@ func TestTranslate_ResendSeesRawReply(t *testing.T) { if r.path != "/v1/chat/completions" || !bytes.Contains(r.body, []byte(`"messages":[{"role":"user","content":[`)) { t.Errorf("upstream request %d = %s %s, want a chat body", i, r.path, r.body) } + // The upstream headers go on the resent request too, after the header sync. + if got := r.header.Values("User-Agent"); len(got) != 1 || got[0] != "bob-shell" { + t.Errorf("upstream request %d User-Agent = %q, want only bob-shell", i, got) + } } // Each attempt's row holds the chat body that attempt sent: the refused row the // first model, the resent row the substitute. @@ -498,6 +564,53 @@ func TestTranslate_ResendSeesRawReply(t *testing.T) { if !bytes.Contains(reqs[1].body, []byte(`"model":"claude-sonnet-5"`)) { t.Errorf("resent body = %s, want the substitute model", reqs[1].body) } + // Decision 19: the refused attempt's row has the refusal in the agent's protocol. + // The raw body's type is team_model_access_denied; Anthropic's for a 403 is + // permission_error. + rows := waitRows(t, store, pipeline.SessionResponse, 2) + if len(rows) != 2 { + t.Fatalf("%d response rows, want 2", len(rows)) + } + refused := rows[0] + if refused.StatusCode != http.StatusForbidden || refused.Error == nil || refused.Error.Message != "permission_error" { + t.Errorf("refused row = status %d error %+v, want 403 with permission_error", refused.StatusCode, refused.Error) + } + if refused.Inference == nil || refused.Inference.Model != "claude-opus-5-5" { + t.Errorf("refused row inference = %+v, want the refused model claude-opus-5-5", refused.Inference) + } + if rows[1].StatusCode != http.StatusOK { + t.Errorf("resent row status = %d, want 200", rows[1].StatusCode) + } +} + +// A Resender asks for a resend, but its change makes a request that cannot be +// converted. The refused answer then goes to the agent, translated, and nothing is +// sent again. +func TestTranslate_ResendNotConvertible(t *testing.T) { + up := refusedThenAnswered(t) + doc := `{"model":"claude-sonnet-5","max_tokens":1024,"messages":[{"role":"user","content":[` + + `{"type":"document","source":{"type":"base64","media_type":"application/pdf","data":"JVBERi0="}}]}]}` + tr := &translator{target: up.URL, translate: true, resend: true} + store := newStore(t) + proxy := newRedirectProxy(t, store, nil, inferenceparser.NewInferenceParser(), + resendingTranslator{translator: tr, substitute: "claude-sonnet-5", body: doc}) + resp := postAnthropic(t, proxy, anthropicRequest, nil) + body := replyText(t, resp) + + if resp.StatusCode != http.StatusForbidden || !strings.Contains(body, `"permission_error"`) { + t.Errorf("reply = %d %q, want the 403 as an Anthropic permission_error", resp.StatusCode, body) + } + if n := len(up.got()); n != 1 { + t.Errorf("upstream got %d requests, want 1", n) + } + rows := waitRows(t, store, pipeline.SessionResponse, 1) + if len(rows) != 1 { + t.Fatalf("%d response rows, want 1", len(rows)) + } + // The row names the model the upstream refused, not the Resender's change. + if r := rows[0]; r.StatusCode != http.StatusForbidden || r.Inference == nil || r.Inference.Model != "claude-opus-5-5" { + t.Errorf("response row = status %d inference %+v, want 403 for claude-opus-5-5", r.StatusCode, r.Inference) + } } func TestTranslate_NotTranslatedPassesBytes(t *testing.T) { From 7eef63e295779b5d909a450bf4f43ade2f4ea6bd Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Tue, 17 Nov 2026 16:43:41 -0500 Subject: [PATCH 07/15] fix: Give a translated JSON reply the status of its error Fixes review: a not-streamed 2xx reply that converts to an error reached the agent as HTTP 200 Fixes review: the chat decoder buffer and the not-streamed reply had no size limit Fixes review: blocks of unknown kinds were dropped with no record in Dropped Fixes review: a 0-byte translated reply left the capture's Upstream nil Files: - core/listener/forwardproxy/translate.go - core/listener/forwardproxy/translate_test.go - core/translate/chat/reply.go - core/translate/chat/reply_test.go - core/translate/chat/request.go - core/translate/chat/request_test.go - core/translate/convert.go - core/translate/golden_test.go Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- core/listener/forwardproxy/translate.go | 52 +++++++++++++++++--- core/listener/forwardproxy/translate_test.go | 47 ++++++++++++++++++ core/translate/chat/reply.go | 9 ++++ core/translate/chat/reply_test.go | 8 +++ core/translate/chat/request.go | 6 +++ core/translate/chat/request_test.go | 20 ++++++++ core/translate/convert.go | 18 ++++--- core/translate/golden_test.go | 3 +- 8 files changed, 150 insertions(+), 13 deletions(-) diff --git a/core/listener/forwardproxy/translate.go b/core/listener/forwardproxy/translate.go index c6904cc8a..63d9c98d9 100644 --- a/core/listener/forwardproxy/translate.go +++ b/core/listener/forwardproxy/translate.go @@ -1,7 +1,10 @@ package forwardproxy import ( + "bytes" "errors" + "fmt" + "io" "net/http" "strings" @@ -115,33 +118,70 @@ func (t *translation) agentStream(status int) bool { return t.conv.Stream && status/100 == 2 } -// wrapReply makes resp carry the reply in the agent's protocol, converted as it is -// read (rule 10). Every later arm then reads the converted bytes. When rc is not nil, +// wrapReply makes resp carry the reply in the agent's protocol. A stream is +// converted as it is read (rule 10). Every later arm then reads the converted bytes. When rc is not nil, // it keeps the server's reply before the conversion in rc.Upstream (rule 8a). func (t *translation) wrapReply(resp *http.Response, rc *pipeline.ResponseCapture) { if rc != nil { // The count and the end are the converted body's, which the caller's wrapper keeps. var unused int64 var ended bool + rc.Upstream = []byte{} // not nil: the reply is converted, also when it is empty resp.Body = bodycount.WrapCapture(resp.Body, &unused, &rc.Upstream, &rc.UpstreamTruncated, &ended, pipeline.MaxResponseCapture) } stream := t.agentStream(resp.StatusCode) - dec := t.dec.NewReplyDecoder(resp.StatusCode, httpx.IsEventStream(resp.Header.Get("Content-Type"))) - enc := t.enc.NewReplyEncoder(stream) - resp.Body = translate.NewReplyReader(resp.Body, dec, enc) + upstreamStream := httpx.IsEventStream(resp.Header.Get("Content-Type")) if stream { + dec := t.dec.NewReplyDecoder(resp.StatusCode, upstreamStream) + resp.Body = translate.NewReplyReader(resp.Body, dec, t.enc.NewReplyEncoder(true)) resp.Header.Set("Content-Type", "text/event-stream") } else { + t.convertWhole(resp, upstreamStream) resp.Header.Set("Content-Type", "application/json") } resp.Header.Del("Content-Length") resp.Header.Del("Content-Encoding") resp.ContentLength = -1 + if rc != nil { + rc.Status = resp.StatusCode + } } +// convertWhole converts a reply that the agent gets as one body. It reads the +// whole body before the status is sent, so that an error that the decoder finds +// in a 2xx body gives its own status, not the 2xx. +func (t *translation) convertWhole(resp *http.Response, upstreamStream bool) { + upstream := resp.Body + raw, err := io.ReadAll(io.LimitReader(upstream, maxBodySize+1)) + var body io.Reader + switch { + case err != nil: + // The arm that reads the body reports the failed read. + body = &failedRead{err} + case len(raw) > maxBodySize: + resp.StatusCode = http.StatusBadGateway + body = bytes.NewReader(translate.ErrorBody(t.source, http.StatusBadGateway, "api_error", + fmt.Sprintf("the inference server's reply is over the %d-byte buffer limit", maxBodySize))) + default: + status, out := translate.Reply(t.source, t.target, resp.StatusCode, upstreamStream, false, raw) + resp.StatusCode = status + body = bytes.NewReader(out) + } + resp.Body = struct { + io.Reader + io.Closer + }{body, upstream} +} + +// failedRead gives err at the first Read. +type failedRead struct{ err error } + +func (f *failedRead) Read([]byte) (int, error) { return 0, f.err } + // replyBody converts a whole upstream body, for the row of a refused attempt // (decision 19). func (t *translation) replyBody(resp *http.Response, body []byte) []byte { - return translate.Reply(t.source, t.target, resp.StatusCode, + _, out := translate.Reply(t.source, t.target, resp.StatusCode, httpx.IsEventStream(resp.Header.Get("Content-Type")), t.agentStream(resp.StatusCode), body) + return out } diff --git a/core/listener/forwardproxy/translate_test.go b/core/listener/forwardproxy/translate_test.go index 2a7e80685..9830587f2 100644 --- a/core/listener/forwardproxy/translate_test.go +++ b/core/listener/forwardproxy/translate_test.go @@ -671,3 +671,50 @@ func TestTranslate_CaptureHoldsFourBodies(t *testing.T) { } } } + +func TestTranslate_NotStreamedErrorStatus(t *testing.T) { + for _, c := range []struct { + name, body string + want int + }{ + {"error body at 200", replyFixture(t, "chunk_error.json"), http.StatusInternalServerError}, + {"not JSON", "not json", http.StatusBadGateway}, + {"over the limit", strings.Repeat("x", maxBodySize+1), http.StatusBadGateway}, + } { + t.Run(c.name, func(t *testing.T) { + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, c.body) + }) + store := newStore(t) + proxy := newRedirectProxy(t, store, nil, &translator{target: up.URL, translate: true}) + resp := postAnthropic(t, proxy, anthropicRequest, nil) + body := replyText(t, resp) + + if resp.StatusCode != c.want || !strings.HasPrefix(body, `{"type":"error"`) { + t.Errorf("reply = %d %q, want %d with an Anthropic error body", resp.StatusCode, body, c.want) + } + if rows := waitRows(t, store, pipeline.SessionResponse, 1); rows[0].StatusCode != c.want { + t.Errorf("response row status = %d, want %d", rows[0].StatusCode, c.want) + } + }) + } +} + +func TestTranslate_EmptyReplyCapturesUpstream(t *testing.T) { + up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + }) + proxy, rec := proxyWithCaptures(t, inferenceparser.NewInferenceParser(), &translator{target: up.URL, translate: true}) + _ = replyText(t, postAnthropic(t, proxy, anthropicStreamRequest, nil)) + + res := rec.responses() + if len(res) != 1 { + t.Fatalf("%d response captures, want 1", len(res)) + } + for _, r := range res { + if r.Upstream == nil { + t.Errorf("Upstream is nil, want an empty capture of the server's reply") + } + } +} diff --git a/core/translate/chat/reply.go b/core/translate/chat/reply.go index e05c16c5f..28e24f725 100644 --- a/core/translate/chat/reply.go +++ b/core/translate/chat/reply.go @@ -14,6 +14,10 @@ import ( // maxErrorBody is the most of a non-2xx body that the decoder keeps. const maxErrorBody = 1 << 20 +// maxStreamLine is the longest stream line that the decoder keeps. It is the +// forward proxy's limit for one SSE frame. +const maxStreamLine = 10 << 20 + // maxErrorMessage is the most of a non-JSON error body that goes into the // error message (Review Focus 4). const maxErrorMessage = 512 @@ -91,6 +95,11 @@ func (d *decoder) Feed(p []byte) []pivot.Event { d.line(rest[:i]) rest = rest[i+1:] } + if len(rest) > maxStreamLine { + d.fail(http.StatusBadGateway, "api_error", "the inference server sent a stream line over the "+strconv.Itoa(maxStreamLine)+"-byte limit") + d.buf = nil + return d.out + } // Keep only the partial line, in a new array, so that the stream // read so far is not held (rule 10). d.buf = append([]byte(nil), rest...) diff --git a/core/translate/chat/reply_test.go b/core/translate/chat/reply_test.go index 159d5165f..0646f4ca7 100644 --- a/core/translate/chat/reply_test.go +++ b/core/translate/chat/reply_test.go @@ -362,3 +362,11 @@ func TestReplyDecoder_ToolCallIDsNotStream(t *testing.T) { checkEvents(t, 200, false, body, []pivot.Event{apiError(502, "api_error", msg)}) } } + +func TestReplyDecoder_StreamLineOverLimit(t *testing.T) { + d := Protocol{}.NewReplyDecoder(200, true) + evs := d.Feed([]byte("data: " + strings.Repeat("x", maxStreamLine))) + if len(evs) != 1 || evs[0].Kind != pivot.EventError || evs[0].Status != 502 { + t.Errorf("events = %+v, want one 502 error", evs) + } +} diff --git a/core/translate/chat/request.go b/core/translate/chat/request.go index 6380d8181..26ce4d2ea 100644 --- a/core/translate/chat/request.go +++ b/core/translate/chat/request.go @@ -221,6 +221,8 @@ func (e *encoder) user(blocks []pivot.Block) ([]message, error) { toolImages = append(toolImages, imagePart(c)) case pivot.BlockDocument: return nil, errDocument + default: + e.dropped[string(c.Kind)] = true } } if len(text) == 0 { @@ -230,6 +232,8 @@ func (e *encoder) user(blocks []pivot.Block) ([]message, error) { } text[len(text)-1].CacheControl = b.CacheControl msgs = append(msgs, message{Role: "tool", Content: text, ToolCallID: b.ToolUseID}) + default: + e.dropped[string(b.Kind)] = true } } if len(toolImages) > 0 { @@ -266,6 +270,8 @@ func (e *encoder) assistant(blocks []pivot.Block) message { m.ThinkingBlocks = append(m.ThinkingBlocks, thinkingBlock{Type: "thinking", Thinking: &t, Signature: b.Signature}) case pivot.BlockRedactedThinking: m.ThinkingBlocks = append(m.ThinkingBlocks, thinkingBlock{Type: "redacted_thinking", Data: b.Data}) + default: + e.dropped[string(b.Kind)] = true } } if text.Len() > 0 { diff --git a/core/translate/chat/request_test.go b/core/translate/chat/request_test.go index 7d575faaa..0ddda77c3 100644 --- a/core/translate/chat/request_test.go +++ b/core/translate/chat/request_test.go @@ -211,3 +211,23 @@ func TestEncodeRequest_PrefixStable(t *testing.T) { t.Errorf("moved marker:\n got %s\nwant %s", got, wantMoved) } } + +func TestEncodeRequest_UnknownBlocksDropped(t *testing.T) { + req := pivot.Request{Model: "m", MaxTokens: 10, Messages: []pivot.Message{ + {Role: "user", Content: []pivot.Block{ + {Kind: pivot.BlockText, Text: "hi"}, + {Kind: "search_result"}, + {Kind: pivot.BlockToolResult, ToolUseID: "t", Content: []pivot.Block{{Kind: "web_fetch_result"}}}, + }}, + {Role: "assistant", Content: []pivot.Block{{Kind: "server_tool_use"}}}, + }} + _, dropped, err := Protocol{}.EncodeRequest(req, http.Header{}) + if err != nil { + t.Fatal(err) + } + for _, k := range []string{"search_result", "server_tool_use", "web_fetch_result"} { + if !strings.Contains(strings.Join(dropped, ","), k) { + t.Errorf("dropped = %v, want it to name %s", dropped, k) + } + } +} diff --git a/core/translate/convert.go b/core/translate/convert.go index 47b4436df..65f76a8f4 100644 --- a/core/translate/convert.go +++ b/core/translate/convert.go @@ -55,16 +55,22 @@ func replyCodecs(source, target string, status int, upstreamStream, agentStream // that the source protocol's agent expects. upstreamStream tells if the // body is SSE; agentStream tells if the agent asked for a stream. When a // direction is missing, it gives a 502 error body in the source protocol -// (nil for an unknown source). -func Reply(source, target string, status int, upstreamStream, agentStream bool, body []byte) []byte { +// (nil for an unknown source). It also gives the status of the converted +// reply: the status of its error event, or status when it has none. +func Reply(source, target string, status int, upstreamStream, agentStream bool, body []byte) (int, []byte) { dec, enc, ok := replyCodecs(source, target, status, upstreamStream, agentStream) if !ok { - return ErrorBody(source, http.StatusBadGateway, "api_error", + return http.StatusBadGateway, ErrorBody(source, http.StatusBadGateway, "api_error", "Cortex cannot convert a "+target+" reply to "+source) } - out := encodeAll(enc, nil, dec.Feed(body)) - out = encodeAll(enc, out, dec.End()) - return append(out, enc.End()...) + evs := append(dec.Feed(body), dec.End()...) + for _, ev := range evs { + if ev.Kind == pivot.EventError && ev.Status != 0 { + status = ev.Status + } + } + out := encodeAll(enc, nil, evs) + return status, append(out, enc.End()...) } // encodeAll appends the encoded events to out. diff --git a/core/translate/golden_test.go b/core/translate/golden_test.go index f65272c0e..95daa2663 100644 --- a/core/translate/golden_test.go +++ b/core/translate/golden_test.go @@ -172,7 +172,8 @@ func TestReplyGolden(t *testing.T) { for _, agentStream := range c.agentForms() { name := goldenName(c, agentStream) t.Run(filepath.Base(name), func(t *testing.T) { - checkGolden(t, name, Reply(Anthropic, Chat, c.status, c.stream, agentStream, body)) + _, out := Reply(Anthropic, Chat, c.status, c.stream, agentStream, body) + checkGolden(t, name, out) }) } } From 920e4a986d659a4bde585b0098f42b56d76a7c85 Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Tue, 17 Nov 2026 16:55:00 -0500 Subject: [PATCH 08/15] fix: Limit a translated JSON reply that answers a stream request Fixes review: a 2xx JSON reply to a stream request had no buffer limit in the chat decoder Fixes review: the limit check ran again after the decoder had stopped Fixes review: the listener's over-limit test did not reach the limit branch Fixes review: ttfbMs of a not-streamed translated reply included the time to read the body Files: - core/listener/forwardproxy/server.go - core/listener/forwardproxy/translate_test.go - core/translate/chat/reply.go - core/translate/chat/reply_test.go - core/translate/chat/request.go - core/translate/convert.go Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- core/listener/forwardproxy/server.go | 4 +++- core/listener/forwardproxy/translate_test.go | 3 ++- core/translate/chat/reply.go | 22 ++++++++++++++----- core/translate/chat/reply_test.go | 23 ++++++++++++++++---- core/translate/chat/request.go | 2 +- core/translate/convert.go | 2 +- 6 files changed, 42 insertions(+), 14 deletions(-) diff --git a/core/listener/forwardproxy/server.go b/core/listener/forwardproxy/server.go index c73495473..98e19d674 100644 --- a/core/listener/forwardproxy/server.go +++ b/core/listener/forwardproxy/server.go @@ -936,6 +936,8 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne rc = &pipeline.ResponseCapture{Status: resp.StatusCode, Decoded: resp.Uncompressed} } + // Before the wrap: a not-streamed translated reply is read whole there. + headersAt := time.Now() // Before any reply plugin or recorder reads the body: they all see the reply in // the agent's protocol. A hop's own 503 returned above and is not converted. if tr != nil { @@ -952,7 +954,7 @@ func (s *Server) serveOutbound(w http.ResponseWriter, r *http.Request, tl *tunne // own flow — the moment the delivered response's headers are in hand — // so ttfb means one thing wherever a row reports it. Stays zero on the // error path above, where no headers ever arrived (#1251). - pctx.ResponseHeadersAt = time.Now() + pctx.ResponseHeadersAt = headersAt // BytesDown for the response row (#1309), counted HERE and nowhere else. Every // arm below reads this one body — buffered whole, relayed SSE block by block, relayed diff --git a/core/listener/forwardproxy/translate_test.go b/core/listener/forwardproxy/translate_test.go index 9830587f2..366e32825 100644 --- a/core/listener/forwardproxy/translate_test.go +++ b/core/listener/forwardproxy/translate_test.go @@ -679,7 +679,8 @@ func TestTranslate_NotStreamedErrorStatus(t *testing.T) { }{ {"error body at 200", replyFixture(t, "chunk_error.json"), http.StatusInternalServerError}, {"not JSON", "not json", http.StatusBadGateway}, - {"over the limit", strings.Repeat("x", maxBodySize+1), http.StatusBadGateway}, + // Valid JSON: only the limit makes it an error. + {"over the limit", replyFixture(t, "bob_shape1.json") + strings.Repeat(" ", maxBodySize), http.StatusBadGateway}, } { t.Run(c.name, func(t *testing.T) { up := newChatServer(t, func(w http.ResponseWriter, _ *http.Request) { diff --git a/core/translate/chat/reply.go b/core/translate/chat/reply.go index 28e24f725..082dbb9aa 100644 --- a/core/translate/chat/reply.go +++ b/core/translate/chat/reply.go @@ -14,9 +14,9 @@ import ( // maxErrorBody is the most of a non-2xx body that the decoder keeps. const maxErrorBody = 1 << 20 -// maxStreamLine is the longest stream line that the decoder keeps. It is the -// forward proxy's limit for one SSE frame. -const maxStreamLine = 10 << 20 +// maxReplyBuffer is the most of a 2xx reply that the decoder keeps: a whole +// not-streamed body, or one stream line. It is the forward proxy's body limit. +const maxReplyBuffer = 10 << 20 // maxErrorMessage is the most of a non-JSON error body that goes into the // error message (Review Focus 4). @@ -83,6 +83,11 @@ func (d *decoder) Feed(p []byte) []pivot.Event { } d.buf = append(d.buf, p...) if !d.stream { + if len(d.buf) > maxReplyBuffer { + d.out = nil + d.overLimit() + return d.out + } return nil } d.out = nil @@ -95,9 +100,8 @@ func (d *decoder) Feed(p []byte) []pivot.Event { d.line(rest[:i]) rest = rest[i+1:] } - if len(rest) > maxStreamLine { - d.fail(http.StatusBadGateway, "api_error", "the inference server sent a stream line over the "+strconv.Itoa(maxStreamLine)+"-byte limit") - d.buf = nil + if !d.done && len(rest) > maxReplyBuffer { + d.overLimit() return d.out } // Keep only the partial line, in a new array, so that the stream @@ -134,6 +138,12 @@ func (d *decoder) fail(status int, errType, msg string) { d.done = true } +// overLimit ends a reply that is longer than the decoder keeps. +func (d *decoder) overLimit() { + d.fail(http.StatusBadGateway, "api_error", "the inference server's reply is over the "+strconv.Itoa(maxReplyBuffer)+"-byte buffer limit") + d.buf = nil +} + // finish ends a stream at [DONE] or at the end of the body (decision 5). func (d *decoder) finish() { if d.done { diff --git a/core/translate/chat/reply_test.go b/core/translate/chat/reply_test.go index 0646f4ca7..6238e33c0 100644 --- a/core/translate/chat/reply_test.go +++ b/core/translate/chat/reply_test.go @@ -363,10 +363,25 @@ func TestReplyDecoder_ToolCallIDsNotStream(t *testing.T) { } } -func TestReplyDecoder_StreamLineOverLimit(t *testing.T) { - d := Protocol{}.NewReplyDecoder(200, true) - evs := d.Feed([]byte("data: " + strings.Repeat("x", maxStreamLine))) +func TestReplyDecoder_OverLimit(t *testing.T) { + // Valid JSON, so only the limit makes it an error. + body := `{"choices":[{"index":0,"finish_reason":"stop","message":{"role":"assistant","content":"hi"}}]}` + strings.Repeat(" ", maxReplyBuffer) + d := Protocol{}.NewReplyDecoder(200, false) + evs := append(d.Feed([]byte(body)), d.End()...) if len(evs) != 1 || evs[0].Kind != pivot.EventError || evs[0].Status != 502 { - t.Errorf("events = %+v, want one 502 error", evs) + t.Errorf("not streamed: events = %+v, want one 502 error", evs) + } + // Bytes after [DONE] are not read, so they give no error. + d = Protocol{}.NewReplyDecoder(200, true) + evs = d.Feed([]byte(sse(chunk(`{"content":"a"}`), finish("stop"), "[DONE]") + strings.Repeat("x", maxReplyBuffer+1))) + for _, ev := range evs { + if ev.Kind == pivot.EventError { + t.Errorf("stream: events = %+v, want no error after [DONE]", evs) + } + } + d = Protocol{}.NewReplyDecoder(200, true) + evs = d.Feed([]byte("data: " + strings.Repeat("x", maxReplyBuffer))) + if len(evs) != 1 || evs[0].Kind != pivot.EventError || evs[0].Status != 502 { + t.Errorf("stream line: events = %+v, want one 502 error", evs) } } diff --git a/core/translate/chat/request.go b/core/translate/chat/request.go index 26ce4d2ea..6e77293b2 100644 --- a/core/translate/chat/request.go +++ b/core/translate/chat/request.go @@ -156,7 +156,7 @@ func (Protocol) EncodeRequest(req pivot.Request, h http.Header) ([]byte, []strin return body, dropped, nil } -// encoder collects the names of the dropped fields (decision 11). +// encoder collects the names of the dropped fields and blocks (decision 11). type encoder struct { dropped map[string]bool } diff --git a/core/translate/convert.go b/core/translate/convert.go index 65f76a8f4..789905356 100644 --- a/core/translate/convert.go +++ b/core/translate/convert.go @@ -10,7 +10,7 @@ import ( // Converted is a request converted to the target protocol. type Converted struct { Body []byte - Dropped []string // the fields that the target could not carry (rule H4) + Dropped []string // the fields and blocks that the target could not carry (rule H4) Stream bool // the agent asked for a stream (decision 4) } From eeec21132868e175c3234c3cd18d9c4710e3f1e8 Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Tue, 17 Nov 2026 17:53:12 -0500 Subject: [PATCH 09/15] fix: Drop a translated-hop ingest key of the wrong type and keep its line Fixes review: a wrong JSON type in `translated`, `upstream` or `upstreamTruncated` refused the whole ingest line as bad_json. Files: - core/sessionapi/ingest.go - core/sessionapi/ingest_test.go Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- core/sessionapi/ingest.go | 47 +++++++++++++++++++++++++--------- core/sessionapi/ingest_test.go | 24 +++++++++++++++++ 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/core/sessionapi/ingest.go b/core/sessionapi/ingest.go index b2913f7e2..3d36e33cd 100644 --- a/core/sessionapi/ingest.go +++ b/core/sessionapi/ingest.go @@ -247,7 +247,7 @@ type ingestCost struct { type ingestCapture struct { Original []byte `json:"original"` Forwarded []byte `json:"forwarded"` - Translated []byte `json:"translated"` + Translated lenientBytes `json:"translated"` Header http.Header `json:"header"` Query string `json:"query"` Response *ingestResponse `json:"response"` @@ -255,15 +255,38 @@ type ingestCapture struct { } type ingestResponse struct { - Status int `json:"status"` - Header http.Header `json:"header"` - Body []byte `json:"body"` - Decoded bool `json:"decoded"` - Truncated bool `json:"truncated"` - Ended bool `json:"ended"` - Incomplete bool `json:"incomplete"` - Upstream []byte `json:"upstream"` - UpstreamTruncated bool `json:"upstreamTruncated"` + Status int `json:"status"` + Header http.Header `json:"header"` + Body []byte `json:"body"` + Decoded bool `json:"decoded"` + Truncated bool `json:"truncated"` + Ended bool `json:"ended"` + Incomplete bool `json:"incomplete"` + Upstream lenientBytes `json:"upstream"` + UpstreamTruncated lenientBool `json:"upstreamTruncated"` +} + +// lenientBytes decodes a base64 JSON string as its bytes and any other value as nil, so a +// wrong type drops the key and keeps the line, as model.LenientString does. +type lenientBytes []byte + +func (b *lenientBytes) UnmarshalJSON(data []byte) error { + var v []byte + if json.Unmarshal(data, &v) == nil { + *b = v + } + return nil +} + +// lenientBool decodes a JSON bool as itself and any other value as false. +type lenientBool bool + +func (b *lenientBool) UnmarshalJSON(data []byte) error { + var v bool + if json.Unmarshal(data, &v) == nil { + *b = lenientBool(v) + } + return nil } // handleIngest is registered behind httpx.LoopbackOnly (see New), also when the rest of the @@ -468,7 +491,7 @@ func (in *ingester) capture(l ingestLine, ev *model.SessionEvent) *model.Request defer in.reqCapsMu.Unlock() if c.Response == nil { in.reqCaps[l.RequestID] = time.Now() - return &model.RequestCapture{Original: c.Original, Forwarded: c.Forwarded, Translated: c.Translated, + return &model.RequestCapture{Original: c.Original, Forwarded: c.Forwarded, Translated: []byte(c.Translated), Header: redact.Headers(c.Header), Query: redact.Query(c.Query)} } in.sweepLocked(time.Now()) @@ -479,7 +502,7 @@ func (in *ingester) capture(l ingestLine, ev *model.SessionEvent) *model.Request res := c.Response return &model.RequestCapture{Response: &model.ResponseCapture{Status: res.Status, Header: redact.Headers(res.Header), Body: res.Body, Decoded: res.Decoded, Truncated: res.Truncated, Ended: res.Ended, Incomplete: res.Incomplete, - Upstream: res.Upstream, UpstreamTruncated: res.UpstreamTruncated}} + Upstream: []byte(res.Upstream), UpstreamTruncated: bool(res.UpstreamTruncated)}} } func (s *Server) handleIngestStats(w http.ResponseWriter, r *http.Request) { diff --git a/core/sessionapi/ingest_test.go b/core/sessionapi/ingest_test.go index 73e9baa6a..060081c37 100644 --- a/core/sessionapi/ingest_test.go +++ b/core/sessionapi/ingest_test.go @@ -648,3 +648,27 @@ func TestIngest_CaptureKeepsTheTranslatedHop(t *testing.T) { t.Errorf("upstream = %q, truncated %v", res.Upstream, res.UpstreamTruncated) } } + +// A wrong JSON type in a translated-hop key drops that key, not the line. +func TestIngest_BadTranslatedHopKeysKeepTheRow(t *testing.T) { + for _, bad := range []any{123, "{not base64}", map[string]any{"a": 1}, true} { + r := newIngestRig(t) + r.post(t, nil, + line(t, map[string]any{"capture": map[string]any{"original": []byte(`{"a":1}`), "translated": bad}}), + line(t, map[string]any{"event": responseEvent(nil), "capture": map[string]any{ + "response": map[string]any{"status": 200, "upstream": bad, "upstreamTruncated": bad, "ended": true}, + }}), + ) + if evs := r.events("s1"); len(evs) != 2 { + t.Errorf("value %v: stored %d events, want 2", bad, len(evs)) + continue + } + req, res := r.caps.get("r1") + if req == nil || res == nil { + t.Fatalf("value %v: captures: request %v, response %v", bad, req, res) + } + if wantTrunc := bad == true; req.Translated != nil || res.Upstream != nil || res.UpstreamTruncated != wantTrunc { + t.Errorf("value %v: translated %q, upstream %q, truncated %v; want empty, empty, %v", bad, req.Translated, res.Upstream, res.UpstreamTruncated, wantTrunc) + } + } +} From 47285e6264b375cbaecf9c9403b217781dd832a6 Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Tue, 17 Nov 2026 18:33:43 -0500 Subject: [PATCH 10/15] fix: Drop a system block that is not text and name it in Dropped Fixes review: a system block that is not text was sent as an empty text part, with no record in Dropped. Files: - core/translate/chat/request.go - core/translate/testdata/request/rule13_system_nontext.{json,golden} - core/translate/testdata/request/rule13_system_only_nontext.{json,golden} Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- core/translate/chat/request.go | 8 ++++++-- .../testdata/request/rule13_system_nontext.golden | 2 ++ .../translate/testdata/request/rule13_system_nontext.json | 3 +++ .../testdata/request/rule13_system_only_nontext.golden | 2 ++ .../testdata/request/rule13_system_only_nontext.json | 3 +++ 5 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 core/translate/testdata/request/rule13_system_nontext.golden create mode 100644 core/translate/testdata/request/rule13_system_nontext.json create mode 100644 core/translate/testdata/request/rule13_system_only_nontext.golden create mode 100644 core/translate/testdata/request/rule13_system_only_nontext.json diff --git a/core/translate/chat/request.go b/core/translate/chat/request.go index 6e77293b2..8c9b3d9b8 100644 --- a/core/translate/chat/request.go +++ b/core/translate/chat/request.go @@ -109,8 +109,8 @@ func (Protocol) EncodeRequest(req pivot.Request, h http.Header) ([]byte, []strin if req.Stream { out.StreamOptions = &streamOptions{IncludeUsage: true} } - if len(req.System) > 0 { - out.Messages = append(out.Messages, message{Role: "system", Content: e.textParts(req.System)}) + if parts := e.textParts(req.System); len(parts) > 0 { + out.Messages = append(out.Messages, message{Role: "system", Content: parts}) } for _, m := range req.Messages { e.extra(m.Extra) @@ -175,6 +175,10 @@ func (e *encoder) textParts(blocks []pivot.Block) []part { parts := make([]part, 0, len(blocks)) for _, b := range blocks { e.extra(b.Extra) + if b.Kind != pivot.BlockText { + e.dropped[string(b.Kind)] = true + continue + } parts = append(parts, textPart(b.Text, b.CacheControl)) } return parts diff --git a/core/translate/testdata/request/rule13_system_nontext.golden b/core/translate/testdata/request/rule13_system_nontext.golden new file mode 100644 index 000000000..522561982 --- /dev/null +++ b/core/translate/testdata/request/rule13_system_nontext.golden @@ -0,0 +1,2 @@ +dropped: content,image,search_result,source,title +{"model":"claude-opus-5-5","messages":[{"role":"system","content":[{"type":"text","text":"hi"}]},{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule13_system_nontext.json b/core/translate/testdata/request/rule13_system_nontext.json new file mode 100644 index 000000000..7ca1bca41 --- /dev/null +++ b/core/translate/testdata/request/rule13_system_nontext.json @@ -0,0 +1,3 @@ +{"model":"claude-opus-5-5","max_tokens":1024, + "system":[{"type":"text","text":"hi"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"iVBORw0KGgo="}},{"type":"search_result","source":"https://example.com","title":"t","content":[{"type":"text","text":"r"}]}], + "messages":[{"role":"user","content":"hi"}]} diff --git a/core/translate/testdata/request/rule13_system_only_nontext.golden b/core/translate/testdata/request/rule13_system_only_nontext.golden new file mode 100644 index 000000000..db122278a --- /dev/null +++ b/core/translate/testdata/request/rule13_system_only_nontext.golden @@ -0,0 +1,2 @@ +dropped: document +{"model":"claude-opus-5-5","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}],"max_tokens":1024} diff --git a/core/translate/testdata/request/rule13_system_only_nontext.json b/core/translate/testdata/request/rule13_system_only_nontext.json new file mode 100644 index 000000000..c597a24ad --- /dev/null +++ b/core/translate/testdata/request/rule13_system_only_nontext.json @@ -0,0 +1,3 @@ +{"model":"claude-opus-5-5","max_tokens":1024, + "system":[{"type":"document","source":{"type":"text","media_type":"text/plain","data":"doc"}}], + "messages":[{"role":"user","content":"hi"}]} From d8d6548e77b817691d7109d59a1bfaf46aef0db9 Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Tue, 17 Nov 2026 16:21:24 -0500 Subject: [PATCH 11/15] docs: Add the translation server page and the router's api fields Add docs/translation-server.md (rule F1): how to run LiteLLM on loopback for a protocol pair that Cortex cannot translate, and how to add it with agentop server add. Add the api, auth_header and headers fields to the inference router page, and a section that gives the routing rules, the auth header table, the refused header names and the modify/translated record. Add an index row in docs/README.md. Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- docs/README.md | 1 + docs/inference-router-plugin.md | 87 ++++++++++++++++++++++++++++++++- docs/translation-server.md | 60 +++++++++++++++++++++++ 3 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 docs/translation-server.md diff --git a/docs/README.md b/docs/README.md index 81ff42f4e..c01baa8d1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -15,6 +15,7 @@ holds the repo-level pieces; this file is the index. | Enforce policy on agent traffic in praxis mode | [`praxis-policy.md`](praxis-policy.md) | | Customize Cortex with a small Python or shell script | [`script-plugin.md`](script-plugin.md) | | Run an agent on a local model, or switch inference servers | [`laptop-local-model.md`](laptop-local-model.md) | +| Use a server whose protocol Cortex cannot translate to | [`translation-server.md`](translation-server.md) | | Use the `agentop` TUI | [`cmd/agentop/README.md`](../cmd/agentop/README.md) | | Add your agent's sessions from before Cortex | [`agentop import`](../cmd/agentop/README.md#importing-agent-history-agentop-import) | | Export sessions to files | [`session-dump.md`](session-dump.md) | diff --git a/docs/inference-router-plugin.md b/docs/inference-router-plugin.md index 89a8fc1c0..bea3e5e62 100644 --- a/docs/inference-router-plugin.md +++ b/docs/inference-router-plugin.md @@ -14,9 +14,12 @@ at startup, and in-cluster use has not been examined. - `servers` (map, required) — inference servers by name: lowercase letters, digits, `.`, `_` and `-`. Each has: - `url` (string, required) — `scheme://host[:port]`, `http` or `https`, no path, query or fragment. A port must be 1–65535; the scheme's default is dropped, so `https://x:443` is `x`. No two servers may share a host, compared without case and, except on a loopback host such as `localhost`, without port, since one machine serves many things there: a session's server is named from the host its requests went to. A plain-`http` server on another machine logs a WARN at load, since routed requests would cross the network decrypted. - - `key` (string, required) — the API key sent to this server in place of the client's, in the header the client used: `X-Api-Key` when it sent one, `Authorization: Bearer` otherwise, both when it sent both. Printable ASCII, no spaces. `/config` and `/v1/pipeline` redact it. + - `key` (string, required) — the API key sent to this server in place of the client's, in the header the client used: `X-Api-Key` when it sent one, `Authorization: Bearer` otherwise, both when it sent both. `auth_header` and a translated request change this (see **Servers that serve another protocol**). Printable ASCII, no spaces. `/config` and `/v1/pipeline` redact it. - `main` (string, required) — a word naming this server's model for an agent's main work, a request that carries tools, sent in place of a model the server refuses (see **Models**). It resolves to the newest model on the server's list whose name, without its provider prefix, contains the word, case ignored: `opus` against `aws/claude-opus-5-5` and `claude-opus-4-8` is the first. No `/` and no whitespace. The fields `opus`, `sonnet` and `haiku` it replaced are refused by name. - `helper` (string, required) — the same for a request without tools, such as a title. + - `api` (map) — protocols the server serves, by protocol name, to the path it serves each at. Empty: requests go in the agent's protocol, to the agent's path. Valid keys: `anthropic-messages`, `chat-completions` and `responses`. Each path must be absolute, with no query, fragment or `..` segment. See **Servers that serve another protocol**. + - `auth_header` (string) — header that carries `key`. `authorization` sends `Bearer `; any other name sends the raw key. Empty: the agent's header, or the target protocol's usual one for a translated request. + - `headers` (map) — fixed headers set on every request to this server after the plugins run. Values may use `${ENV_VAR}`. - `agents` (map) — agent name, as agentop shows it (`claude-code`, `opencode`), to a server name. Each value must name a listed server. `unknown`, the name for requests with no User-Agent, cannot be routed. - `sessions` (map) — session id to a server name, moving one running session to that server. Each value must name a listed server, or be `""` to stop routing that session. An entry beats the session's pin and every rule below, and the pin follows it, so removing the entry does not move the session back; `agentop server move` and the TUI's `S` on a session row write these; a `server move` from the shell also prunes an entry whose session is neither live nor archived. A synthetic id (`default`, `pending:`) is refused: it holds many conversations, not one to move. @@ -254,3 +257,85 @@ server's host, and each event's `requestedHost` keeps the one asked for when it differs. The router also rewrites the request body to send a substitute, so the framework holds it after every body reader: a chain with a parser after the router is refused, at startup and on reload. + +## Servers that serve another protocol + +A server's `api` map says which protocols it serves, and at which path. DeepSeek +serves two, at different paths: + +```yaml +servers: + deepseek: + url: https://api.deepseek.com + key: sk-… + main: chat + helper: chat + api: + chat-completions: /chat/completions + anthropic-messages: /anthropic/v1/messages +``` + +The router uses the agent's protocol, from the request path, and these rules in +this order: + +1. The map is empty: the request goes in the agent's protocol, to the agent's path, + as before. +2. The request is `/v1/messages/count_tokens`, and the map has no + `anthropic-messages`: the proxy answers a 404 Anthropic `not_found_error` and + sends nothing upstream (`deny/count_tokens_local`). This applies only to a + `count_tokens` request that is routed to this server. The router does not + capture `count_tokens`, so when the agent sends it to another host, it goes there. +3. The agent's protocol is a key of the map: the request goes to that key's path, + with no translation. Claude Code's `/v1/messages` goes to DeepSeek's + `/anthropic/v1/messages`. +4. Cortex can read the agent's protocol (only `anthropic-messages` at this time): + Cortex translates the request. The target is the first of `chat-completions`, + `anthropic-messages`, `responses` that is in the map and that Cortex can write + (only `chat-completions` at this time). The request goes to the target's path, + with no query string. If no protocol in the map is a target, Cortex sends + nothing and answers a 502 Anthropic `api_error` + (`deny/cannot_translate`): "Cortex cannot translate anthropic-messages to + \ yet. Use a translation server for this pair". The message names each + map key, sorted, joined with " or ". For that pair, see + [`translation-server.md`](translation-server.md). +5. Otherwise (a chat-completions or Responses request): the request goes as it is, + with no translation and no 502. + +**A translated request.** Every plugin, the session store, the timeline and the +cost ledger see only the Anthropic side: the request as the agent sent it, and the +reply as the agent gets it. The proxy converts the request after the last request +plugin, and it converts the reply before a reply plugin reads it. Streams stay +incremental. The request row carries the framework's `modify/translated` record, +with `from`, `to`, `path` (the path sent upstream), and `dropped` when the +conversion removed fields (for example `metadata,top_k`). A request that the target +cannot carry, such as one with a PDF document, gets a 400 `invalid_request_error` +(`deny/translation_refused`), and nothing goes upstream. The raw capture keeps the +translated request and the server's reply before translation. + +**The auth header.** The sent request has only the header that this table picks. +The router removes the agent's `Authorization`, `X-Api-Key` and `Api-Key` first, +except in the first row: + +| `auth_header` | Request | Header sent | +|---|---|---| +| empty | not translated | as before: the header the agent used (see `key`) | +| empty | translated to `chat-completions` | `Authorization: Bearer ` | +| empty | translated to `anthropic-messages` | `X-Api-Key: ` | +| `authorization` | each request | `Authorization: Bearer ` | +| any other name | each request | that header, with the raw key | + +For example, Azure uses `api-key`. Bob accepts its key as `x-api-key`, and refuses +it as Bearer. + +**Fixed headers.** `headers` values replace the agent's header of the same name, +after the plugins run, on translated and not translated requests. The config load +refuses these names: `Host`, `Content-Length`, the hop-by-hop headers +(`Connection`, `Keep-Alive`, `Proxy-Connection`, `Transfer-Encoding`, `Upgrade`, +`TE`, `Trailer`, `Proxy-Authenticate`, `Proxy-Authorization`), the auth headers +(`Authorization`, `X-Api-Key`, `Api-Key`) and the server's own `auth_header`. It +also refuses a value with CR, LF or NUL. `/config` and `/v1/pipeline` show the +header names and hide the values, as they hide `key`. Bob needs +`User-Agent: bob-shell`. + +`auth_header` and `headers` apply only to the requests that the router routes. The +model-list fetch (`GET /v1/models`) still sends `Authorization: Bearer ` only. diff --git a/docs/translation-server.md b/docs/translation-server.md new file mode 100644 index 000000000..2f5266693 --- /dev/null +++ b/docs/translation-server.md @@ -0,0 +1,60 @@ +# Use a translation server + +The [inference router](inference-router-plugin.md#servers-that-serve-another-protocol) +can translate one protocol pair only: Anthropic Messages (the agent's request) to +chat completions (the server). For each other pair, Cortex does not send the +request. It answers with a 502 in the agent's protocol: + +> Cortex cannot translate \ to \ yet. Use a translation server +> for this pair (see this page). + +For such a pair, run a translation server on loopback, and add it to the router as +the server for the agent. + +## Run LiteLLM + +This LiteLLM config worked on 2026-11-16. It accepts Anthropic Messages and sends +chat completions to the server: + +```yaml +model_list: + - model_name: + litellm_params: + model: hosted_vllm/ + api_base: + api_key: os.environ/UPSTREAM_API_KEY + additional_drop_params: ["prompt_cache_key"] +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY +``` + +- Use the `hosted_vllm/` provider, not `openai/`. LiteLLM's `openai/` provider + sent the requests to `/v1/responses`, and the server answered 403. +- `additional_drop_params` removes `prompt_cache_key`, which the server does not + accept. + +Start it: + +```sh +litellm --config litellm.yaml --host 127.0.0.1 --port 4000 +``` + +Bind it to loopback only, and always set a master key. Without them, any client that can +get to port 4000 can use your server key. + +## Add it to the router + +```sh +agentop server add --main --helper \ + --api anthropic-messages=/v1/messages http://127.0.0.1:4000 +``` + +Give the master key as the server key: at the prompt, or on stdin with +`--key-stdin`. Because `anthropic-messages` is in the server's `api` map, Cortex +sends Claude Code's requests to LiteLLM with no translation. + +## What Cortex records + +Cortex records only the agent's side of the hop: the request to LiteLLM and the +reply from LiteLLM. The cost and the errors of the hop from LiteLLM to the server +are LiteLLM's. Cortex does not see them. From 2b0fb82cd22d019e79045287bb9bd5bc8ee9da08 Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Tue, 17 Nov 2026 16:21:05 -0500 Subject: [PATCH 12/15] feat: Add the api, auth_header and headers server fields Add three optional fields to an inference router server: - api: the protocols that the server serves, each to its path. A key must be a known protocol. A path must be absolute, with no query, fragment or ".." segment. - auth_header: the header that carries the key. Host, Content-Length and the hop-by-hop headers are refused. - headers: fixed headers for each request to the server. The auth headers, the server's own auth_header, Host, Content-Length and the hop-by-hop headers are refused. A value with CR, LF or NUL is refused. redact.JSON now hides each value under a field named exactly "headers" and keeps the names. Fields such as id_headers do not change. Refs #1911 Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- .../inferencerouter/routerconfig/config.go | 101 +++++++++++++++++- .../routerconfig/config_test.go | 90 ++++++++++++++++ core/redact/redact.go | 10 ++ core/redact/redact_test.go | 24 +++++ 4 files changed, 221 insertions(+), 4 deletions(-) diff --git a/core/plugins/inferencerouter/routerconfig/config.go b/core/plugins/inferencerouter/routerconfig/config.go index 63ddb31ae..3749e003a 100644 --- a/core/plugins/inferencerouter/routerconfig/config.go +++ b/core/plugins/inferencerouter/routerconfig/config.go @@ -42,10 +42,94 @@ type Config struct { // Server is one inference server. type Server struct { - URL string `json:"url" required:"true" description:"scheme://host[:port] of the server; no path."` - Key string `json:"key" required:"true" description:"API key sent to this server in place of the client's."` - Main string `json:"main" required:"true" description:"A word naming this server's model for an agent's main work, a request with tools, sent when the server refuses the model the agent asked for. The newest listed model whose name contains it."` - Helper string `json:"helper" required:"true" description:"A word naming this server's model for an agent's helper calls, a request without tools. Resolved as main is."` + URL string `json:"url" required:"true" description:"scheme://host[:port] of the server; no path."` + Key string `json:"key" required:"true" description:"API key sent to this server in place of the client's."` + Main string `json:"main" required:"true" description:"A word naming this server's model for an agent's main work, a request with tools, sent when the server refuses the model the agent asked for. The newest listed model whose name contains it."` + Helper string `json:"helper" required:"true" description:"A word naming this server's model for an agent's helper calls, a request without tools. Resolved as main is."` + API map[string]string `json:"api" description:"Protocols the server serves, by protocol name, to the path it serves each at. Empty: requests go in the agent's protocol, to the agent's path."` + AuthHeader string `json:"auth_header" description:"Header that carries key. authorization sends Bearer ; any other name sends the raw key. Empty: the agent's header, or the target protocol's usual one for a translated request."` + Headers map[string]string `json:"headers" description:"Fixed headers set on every request to this server after the plugins run. Values may use ${ENV_VAR}."` +} + +// APIProtocols are the protocol names an api key can have. It is translate.Names(), +// copied rather than imported to keep this package free of core/translate; a test in +// the plugin package holds the two equal. +var APIProtocols = []string{"anthropic-messages", "chat-completions", "responses"} + +// CheckAPI reports whether api maps known protocols to paths a request can be sent +// to. Keys are checked in sorted order, so the same map always gives the same first +// error. Each error starts with "api". The path rule is pipeline.Context.SetRedirectPath's. +func CheckAPI(api map[string]string) error { + for _, k := range slices.Sorted(maps.Keys(api)) { + if !slices.Contains(APIProtocols, k) { + return fmt.Errorf("api: unknown protocol %q; valid protocols: %s", k, strings.Join(APIProtocols, ", ")) + } + p := api[k] + if !strings.HasPrefix(p, "/") || strings.ContainsAny(p, "?#") || slices.Contains(strings.Split(p, "/"), "..") { + return fmt.Errorf("api.%s: path %q must be absolute, with no query, fragment or \"..\" segment", k, p) + } + } + return nil +} + +// framingHeaders are the headers neither auth_header nor headers can name: the +// transport sets them, and a fixed value would break the request's framing. +var framingHeaders = []string{ + "host", "content-length", + "connection", "keep-alive", "proxy-connection", "transfer-encoding", "upgrade", "te", "trailer", + "proxy-authenticate", "proxy-authorization", +} + +// authHeaders are the headers the router puts the key in. headers cannot name them. +var authHeaders = []string{"authorization", "x-api-key", "api-key"} + +// CheckAuthHeader reports whether name can carry a server's key. "" is valid: it +// means the agent's header, or the target protocol's usual one. +func CheckAuthHeader(name string) error { + if name == "" { + return nil + } + if !validHeaderName(name) { + return fmt.Errorf("%q is not a valid header name", name) + } + if slices.Contains(framingHeaders, strings.ToLower(name)) { + return fmt.Errorf("%q cannot carry the key: it is a hop-by-hop or framing header", name) + } + return nil +} + +// CheckHeaders reports whether h can be set on every request to a server whose +// auth_header is authHeader. Names are checked in sorted order for a stable first +// error. The errors never repeat a value. +func CheckHeaders(h map[string]string, authHeader string) error { + for _, name := range slices.Sorted(maps.Keys(h)) { + lower := strings.ToLower(name) + switch { + case !validHeaderName(name): + return fmt.Errorf("headers: %q is not a valid header name", name) + case slices.Contains(authHeaders, lower) || (authHeader != "" && lower == strings.ToLower(authHeader)): + return fmt.Errorf("headers: %q cannot be set here: Cortex sets it", name) + case slices.Contains(framingHeaders, lower): + return fmt.Errorf("headers: %q cannot be set here: it is a hop-by-hop or framing header", name) + case strings.ContainsAny(h[name], "\r\n\x00"): + return fmt.Errorf("headers: the value of %q has a CR, LF or NUL", name) + } + } + return nil +} + +// validHeaderName reports whether name is an RFC 9110 token. +func validHeaderName(name string) bool { + if name == "" { + return false + } + for i := 0; i < len(name); i++ { + c := name[i] + if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || strings.IndexByte("!#$%&'*+-.^_`|~", c) >= 0) { + return false + } + } + return true } // CheckWord reports whether word can name a server's main or helper model. A word is @@ -151,6 +235,15 @@ func (c Config) Validate() error { if err := CheckWord(s.Helper); err != nil { return fmt.Errorf("servers.%s.helper: %w", name, err) } + if err := CheckAPI(s.API); err != nil { + return fmt.Errorf("servers.%s.%w", name, err) + } + if err := CheckAuthHeader(s.AuthHeader); err != nil { + return fmt.Errorf("servers.%s.auth_header: %w", name, err) + } + if err := CheckHeaders(s.Headers, s.AuthHeader); err != nil { + return fmt.Errorf("servers.%s.%w", name, err) + } } for _, agent := range slices.Sorted(maps.Keys(c.Agents)) { if err := CheckAgent(agent); err != nil { diff --git a/core/plugins/inferencerouter/routerconfig/config_test.go b/core/plugins/inferencerouter/routerconfig/config_test.go index 4304956ae..2bc40f9ce 100644 --- a/core/plugins/inferencerouter/routerconfig/config_test.go +++ b/core/plugins/inferencerouter/routerconfig/config_test.go @@ -373,3 +373,93 @@ func TestDecode_Sessions(t *testing.T) { }) } } +func TestCheckAPI(t *testing.T) { + for _, api := range []map[string]string{ + nil, + {"chat-completions": "/v1/chat/completions"}, + {"chat-completions": "/chat/completions", "anthropic-messages": "/anthropic/v1/messages"}, + {"responses": "/v1/responses"}, + } { + if err := CheckAPI(api); err != nil { + t.Errorf("CheckAPI(%v) = %v, want nil", api, err) + } + } + for api, want := range map[string]string{ + `{"chat": "/v1/chat/completions"}`: `unknown protocol "chat"; valid protocols: anthropic-messages, chat-completions, responses`, + `{"chat-completions": "v1/chat/completions"}`: `api.chat-completions: path "v1/chat/completions" must be absolute, with no query, fragment or ".." segment`, + `{"chat-completions": "/v1/chat/completions?x=1"}`: `api.chat-completions: path "/v1/chat/completions?x=1" must be absolute`, + `{"chat-completions": "/v1/../chat/completions"}`: `".." segment`, + `{"chat-completions": "/v1/chat#x"}`: `must be absolute`, + `{"anthropic-messages": "x", "chat-completions": "y"}`: `api.anthropic-messages:`, // sorted: first key first + } { + var m map[string]string + if err := json.Unmarshal([]byte(api), &m); err != nil { + t.Fatal(err) + } + err := CheckAPI(m) + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("CheckAPI(%s) = %v, want %q", api, err, want) + } + } +} + +func TestCheckAuthHeader(t *testing.T) { + for _, name := range []string{"", "authorization", "x-api-key", "api-key", "Authorization"} { + if err := CheckAuthHeader(name); err != nil { + t.Errorf("CheckAuthHeader(%q) = %v, want nil", name, err) + } + } + for _, name := range []string{"Host", "Connection", "content-length", "Transfer-Encoding", "TE", "Proxy-Authorization", "bad header", "x:y"} { + if err := CheckAuthHeader(name); err == nil { + t.Errorf("CheckAuthHeader(%q) = nil, want an error", name) + } + } +} + +func TestCheckHeaders(t *testing.T) { + if err := CheckHeaders(map[string]string{"User-Agent": "bob-shell", "X-Team": "a b"}, ""); err != nil { + t.Errorf("CheckHeaders = %v, want nil", err) + } + for name, want := range map[string]string{ + "Authorization": `headers: "Authorization" cannot be set here: Cortex sets it`, + "X-Api-Key": `headers: "X-Api-Key" cannot be set here: Cortex sets it`, + "api-key": `Cortex sets it`, + "Content-Length": `headers: "Content-Length" cannot be set here: it is a hop-by-hop or framing header`, + "Transfer-Encoding": `it is a hop-by-hop or framing header`, + "host": `it is a hop-by-hop or framing header`, + "X-Key": `Cortex sets it`, // the server's own auth_header + "bad name": `not a valid header name`, + } { + err := CheckHeaders(map[string]string{name: "v"}, "x-key") + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("CheckHeaders(%q) = %v, want %q", name, err, want) + } + } + for _, v := range []string{"a\nb", "a\rb", "a\x00b"} { + if err := CheckHeaders(map[string]string{"User-Agent": v}, ""); err == nil { + t.Errorf("CheckHeaders(value %q) = nil, want an error", v) + } + } +} + +func TestDecode_APIAuthHeaderHeaders(t *testing.T) { + c, err := Decode(json.RawMessage(`{"servers": {"bob": {"url": "https://bob.example.com", "key": "k", "main": "m", "helper": "h", + "api": {"chat-completions": "/v1/chat/completions"}, "auth_header": "x-api-key", "headers": {"User-Agent": "bob-shell"}}}}`)) + if err != nil { + t.Fatalf("Decode: %v", err) + } + s := c.Servers["bob"] + if s.API["chat-completions"] != "/v1/chat/completions" || s.AuthHeader != "x-api-key" || s.Headers["User-Agent"] != "bob-shell" { + t.Errorf("decoded %+v", s) + } + for field, want := range map[string]string{ + `"api": {"chat": "/x"}`: "servers.bob.api: unknown protocol", + `"auth_header": "Host"`: "servers.bob.auth_header:", + `"headers": {"Authorization": "Bearer x"}`: "servers.bob.headers: \"Authorization\" cannot be set here", + } { + _, err := Decode(json.RawMessage(`{"servers": {"bob": {"url": "https://bob.example.com", "key": "k", "main": "m", "helper": "h", ` + field + `}}}`)) + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("Decode(%s) = %v, want %q", field, err, want) + } + } +} diff --git a/core/redact/redact.go b/core/redact/redact.go index bc7e448d3..7748ac0ca 100644 --- a/core/redact/redact.go +++ b/core/redact/redact.go @@ -47,6 +47,16 @@ func redactMap(m map[string]any) { m[k] = "[REDACTED]" continue } + // A field named exactly "headers" (the inference router's fixed headers) + // holds values that can be secrets under names that are not. Keep the + // names. Matched exactly, not by suffix: id_headers and remove_headers + // hold header names, which must stay visible. + if h, ok := v.(map[string]any); ok && k == "headers" { + for name := range h { + h[name] = "[REDACTED]" + } + continue + } switch val := v.(type) { case map[string]any: redactMap(val) diff --git a/core/redact/redact_test.go b/core/redact/redact_test.go index e6c6e17ca..8ed247203 100644 --- a/core/redact/redact_test.go +++ b/core/redact/redact_test.go @@ -149,3 +149,27 @@ func TestJSON_NestedArrayRedaction(t *testing.T) { t.Errorf("nested array url = %v, want https://a", obj["url"]) } } + +func TestJSON_RedactsHeadersValuesKeepsNames(t *testing.T) { + in := json.RawMessage(`{"servers":{"bob":{"headers":{"User-Agent":"bob-shell"}}},"session":{"id_headers":["X-Session"]},` + + `"cpex":{"delegated_credential_headers":["X-Cred"]},"script":{"remove_headers":["X-Drop"]}}`) + var m struct { + Servers map[string]struct { + Headers map[string]any `json:"headers"` + } `json:"servers"` + Session map[string]any `json:"session"` + Cpex map[string]any `json:"cpex"` + Script map[string]any `json:"script"` + } + if err := json.Unmarshal(JSON(in), &m); err != nil { + t.Fatal(err) + } + if got := m.Servers["bob"].Headers["User-Agent"]; got != "[REDACTED]" { + t.Errorf("headers.User-Agent = %v, want [REDACTED]", got) + } + for name, got := range map[string]any{"id_headers": m.Session["id_headers"], "delegated_credential_headers": m.Cpex["delegated_credential_headers"], "remove_headers": m.Script["remove_headers"]} { + if s, ok := got.([]any); !ok || len(s) != 1 { + t.Errorf("%s = %v, want it unchanged", name, got) + } + } +} From 920fbd9573d88602bc190d14c4c708f8478a667d Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Tue, 17 Nov 2026 16:22:53 -0500 Subject: [PATCH 13/15] fix: Compare router servers with DeepEqual in agentop routerconfig.Server now holds maps (api, headers), so it is not comparable and maps.Equal does not compile on it. sameRouter now compares the servers with reflect.DeepEqual. Refs #1911 Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- cmd/agentop/tui/server_route.go | 5 ++++- cmd/agentop/tui/server_route_test.go | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/cmd/agentop/tui/server_route.go b/cmd/agentop/tui/server_route.go index e1e790210..ae58a63e4 100644 --- a/cmd/agentop/tui/server_route.go +++ b/cmd/agentop/tui/server_route.go @@ -5,6 +5,7 @@ import ( "maps" "os" "path/filepath" + "reflect" "strings" "github.com/rossoctl/cortex/cmd/agentop/servers" @@ -88,7 +89,9 @@ func homeTilde(path string) string { // sameRouter reports whether two router configs route alike, so a pipeline refetch repaints the // SERVER columns only when what they show has changed. func sameRouter(a, b routerconfig.Config) bool { - return maps.Equal(a.Servers, b.Servers) && maps.Equal(a.Agents, b.Agents) + // DeepEqual: a Server holds maps (api, headers), so it is not comparable. + return maps.EqualFunc(a.Servers, b.Servers, func(x, y routerconfig.Server) bool { return reflect.DeepEqual(x, y) }) && + maps.Equal(a.Agents, b.Agents) } // localCortexTarget is the config file and stats URL of the Cortex on this machine, when the diff --git a/cmd/agentop/tui/server_route_test.go b/cmd/agentop/tui/server_route_test.go index 98f8cf412..11d10d560 100644 --- a/cmd/agentop/tui/server_route_test.go +++ b/cmd/agentop/tui/server_route_test.go @@ -481,3 +481,15 @@ func TestAgentsPane_TheAgentsNameSurvivesTheServerColumnAt80(t *testing.T) { t.Errorf("the row does not show the agent's name:\n%s", view) } } + +func TestSameRouter_SeesAHeadersChange(t *testing.T) { + cfg := func(ua string) routerconfig.Config { + return routerconfig.Config{Servers: map[string]routerconfig.Server{"bob": {URL: "https://bob.example.com", Headers: map[string]string{"User-Agent": ua}}}} + } + if !sameRouter(cfg("bob-shell"), cfg("bob-shell")) { + t.Error("sameRouter = false for equal configs") + } + if sameRouter(cfg("bob-shell"), cfg("other")) { + t.Error("sameRouter = true for configs with different headers") + } +} From 6ca1bfb0ddfbdc5ec277dbaad6e7e9861396ab9c Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Tue, 17 Nov 2026 16:29:24 -0500 Subject: [PATCH 14/15] feat: Add --api, --auth-header and --header to agentop server add agentop server add now writes the api, auth_header and headers server fields (rule C4). It checks each flag with the routerconfig checks before it asks for the key. It refuses a header value with $, because config.Load expands it. agentop server shows the api, the auth header and the header names under each server. It does not show header values. Refs #1911 Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- cmd/agentop/README.md | 13 +++++ cmd/agentop/cmd_server.go | 33 ++++++++++++- cmd/agentop/cmd_server_test.go | 38 ++++++++++++++ cmd/agentop/cmd_server_write.go | 74 +++++++++++++++++++++++++++- cmd/agentop/cmd_server_write_test.go | 61 +++++++++++++++++++++++ 5 files changed, 216 insertions(+), 3 deletions(-) diff --git a/cmd/agentop/README.md b/cmd/agentop/README.md index 48dae05f0..11170d7e7 100644 --- a/cmd/agentop/README.md +++ b/cmd/agentop/README.md @@ -704,6 +704,19 @@ refusal is believed for an hour, and a session keeps the model it was given. creates the plugin's entry, last in the outbound pipeline. A key or a word containing `$` is refused: the config loader would read it as an environment variable. + - `--api =` (repeatable) names a protocol the server serves and + the path it serves it at: `anthropic-messages`, `chat-completions` or + `responses`. A second `--api` for one protocol replaces the first. With no + `--api`, requests go in the agent's protocol to the agent's path. + - `--auth-header ` names the header that carries the key: `authorization` + sends `Bearer `, any other name sends the raw key. + - `--header =` (repeatable) sets a fixed header on every request to + the server. A value with `$` is refused, as a key is: write + `headers: {: ${NAME}}` by hand. The key headers and the hop-by-hop + headers are refused. + + `agentop server` shows these under the server: its `api`, its auth header, and + the header names, never their values. - **`use --agent `** routes the agent's new sessions to the server, including an agent that has not run yet. **`reset --agent `** stops routing it. Either way a session already running stays where it is: its pin holds diff --git a/cmd/agentop/cmd_server.go b/cmd/agentop/cmd_server.go index 78f315f2b..73105e80b 100644 --- a/cmd/agentop/cmd_server.go +++ b/cmd/agentop/cmd_server.go @@ -33,6 +33,7 @@ const serverUsage = `agentop server — choose the inference server each coding Usage: agentop server [--config PATH] agentop server add --main WORD --helper WORD [--key-stdin] [--yes] [--config PATH] + [--api PROTOCOL=PATH]... [--auth-header NAME] [--header NAME=VALUE]... agentop server remove [--config PATH] agentop server use --agent [--config PATH] agentop server reset --agent [--config PATH] @@ -54,7 +55,9 @@ agent sends it. "add" reads the server's API key at a prompt that does not echo, or from stdin with --key-stdin; it is never an argument. Adding a name that exists asks before replacing its URL, key and models (--yes skips the question), which is also how a -key is rotated. +key is rotated. --api names a protocol the server serves and its path +(anthropic-messages, chat-completions or responses), --auth-header the header +that carries the key, and --header a fixed header for every request. A request goes with the model the agent asked for. Only when the server refuses that model is it sent again, once and before the agent sees the refusal, with @@ -256,8 +259,16 @@ func serverList(args []string, stdout, stderr io.Writer) int { fmt.Fprintf(tw, " %s\t%s\t%s\t%s\n", name, servers.Host(s), resolvedWords(s), strings.Join(agentsOn(c, name), ", ")) } tw.Flush() + // The detail lines go under each row after the table is aligned: a line with no + // tab would restart tabwriter's columns below it. + names := slices.Sorted(maps.Keys(c.Servers)) + i := 0 for line := range strings.Lines(table.String()) { fmt.Fprintln(stdout, strings.TrimRight(line, " \n")) + for _, d := range protocolLines(c.Servers[names[i]]) { + fmt.Fprintln(stdout, " "+d) + } + i++ } fmt.Fprintln(stdout) @@ -267,6 +278,26 @@ func serverList(args []string, stdout, stderr io.Writer) int { return 0 } +// protocolLines is s's api, auth_header and header names, one line each that s has. +// Header values are never shown: they can hold a credential. +func protocolLines(s routerconfig.Server) []string { + var out []string + if len(s.API) > 0 { + var api []string + for _, p := range slices.Sorted(maps.Keys(s.API)) { + api = append(api, p+"="+s.API[p]) + } + out = append(out, "api: "+strings.Join(api, ", ")) + } + if s.AuthHeader != "" { + out = append(out, "auth header: "+s.AuthHeader) + } + if len(s.Headers) > 0 { + out = append(out, "headers: "+strings.Join(slices.Sorted(maps.Keys(s.Headers)), ", ")) + } + return out +} + // resolvedWords is a server's main and helper words with the models they name on its // list now, read with the server's own key, as the router resolves them. A server // whose list cannot be read says why rather than showing a model it may not name. diff --git a/cmd/agentop/cmd_server_test.go b/cmd/agentop/cmd_server_test.go index 09bae4e60..d3ccafa68 100644 --- a/cmd/agentop/cmd_server_test.go +++ b/cmd/agentop/cmd_server_test.go @@ -295,3 +295,41 @@ func TestServer_SaysWhenAServersModelListIsUnavailable(t *testing.T) { t.Errorf("the listing prints a key:\n%s", out) } } + +// Rule C4: the listing shows api, auth_header and the header names, never a header value. +func TestServer_ListsTheAPIAuthHeaderAndHeaderNames(t *testing.T) { + block := strings.Replace(routerBlock, " helper: haiku\n", ` helper: haiku + api: + chat-completions: /v1/chat/completions + anthropic-messages: /v1/messages + auth_header: x-api-key + headers: + X-Client: cortex + User-Agent: bob-shell +`, 1) + path := serverEnv(t, closedAddr(t), block) + code, out, errOut := runServerCmd(t, "", "--config", path) + if code != 0 { + t.Fatalf("exit %d: %s", code, errOut) + } + lines := strings.Split(out, "\n") + want := []string{ + "api: anthropic-messages=/v1/messages, chat-completions=/v1/chat/completions", + "auth header: x-api-key", + "headers: User-Agent, X-Client", + } + if len(lines) < 5 || !strings.HasPrefix(flat(lines[0]), "ete ") || !strings.HasPrefix(flat(lines[4]), "glm ") { + t.Fatalf("the detail lines are not under ete:\n%s", out) + } + for i, w := range want { + if got := flat(lines[i+1]); got != w { + t.Errorf("line %d = %q, want %q", i+2, got, w) + } + } + if strings.Index(lines[0], "main") != strings.Index(lines[4], "main") { + t.Errorf("columns do not line up:\n%s", out) + } + if strings.Contains(out, "bob-shell") || strings.Contains(out, "cortex\n") { + t.Errorf("the listing shows a header value:\n%s", out) + } +} diff --git a/cmd/agentop/cmd_server_write.go b/cmd/agentop/cmd_server_write.go index dbba7fea0..45bdadc36 100644 --- a/cmd/agentop/cmd_server_write.go +++ b/cmd/agentop/cmd_server_write.go @@ -13,6 +13,7 @@ import ( "time" "github.com/charmbracelet/x/term" + "gopkg.in/yaml.v3" "github.com/rossoctl/cortex/cmd/agentop/apiclient" "github.com/rossoctl/cortex/cmd/agentop/edit" @@ -82,6 +83,10 @@ func serverAdd(args []string, stdin io.Reader, stdout, stderr io.Writer) int { var models routerconfig.Server flags.StringVar(&models.Main, "main", "", "a word naming the server's model for an agent's main work, such as opus or glm") flags.StringVar(&models.Helper, "helper", "", "a word naming the server's model for an agent's helper calls, such as haiku") + api, headers := pairFlag{flag: "--api", want: "="}, pairFlag{flag: "--header", want: "="} + flags.Var(&api, "api", "a protocol the server serves and its path, such as chat-completions=/v1/chat/completions (repeatable)") + flags.StringVar(&models.AuthHeader, "auth-header", "", "the header that carries the key, such as x-api-key (default: the agent's)") + flags.Var(&headers, "header", "a fixed header set on every request to the server, as = (repeatable)") pos, code, ok := serverFlags(flags, args, stdout, stderr) if !ok { return code @@ -112,6 +117,11 @@ func serverAdd(args []string, stdin io.Reader, stdout, stderr io.Writer) int { fmt.Fprintf(stderr, "agentop server add: %v\n", err) return 2 } + models.API, models.Headers = api.m, headers.m + if err := checkProtocolFlags(models); err != nil { + fmt.Fprintf(stderr, "agentop server add: %v\n", err) + return 2 + } // Before anything is asked or read: --key-stdin reads stdin as typed, so on a // terminal the key would echo, character by character, onto the screen and into // any recording of it. The prompt without the flag reads it without echo. @@ -186,9 +196,18 @@ func serverAdd(args []string, stdin io.Reader, stdout, stderr io.Writer) int { fmt.Fprintf(stdout, "%s %s → %s\n", strings.TrimPrefix(w.flag, "--"), w.word, m) } - pairs := []string{"url", ep.URL(), "key", key, "main", models.Main, "helper", models.Helper} + value := edit.MapValue("url", ep.URL(), "key", key, "main", models.Main, "helper", models.Helper) + if len(models.API) > 0 { + value.Content = append(value.Content, edit.ScalarValue("api"), sortedMap(models.API)) + } + if models.AuthHeader != "" { + value.Content = append(value.Content, edit.ScalarValue("auth_header"), edit.ScalarValue(models.AuthHeader)) + } + if len(models.Headers) > 0 { + value.Content = append(value.Content, edit.ScalarValue("headers"), sortedMap(models.Headers)) + } ch := edit.ConfigChange{Chain: "outbound", Plugin: routerName, Path: []string{"servers", name}, - Value: edit.MapValue(pairs...), CreatePlugin: true} + Value: value, CreatePlugin: true} done := "Added " + name + "." if replacing { done = "Replaced " + name + "." @@ -217,6 +236,57 @@ func checkWordFlags(models routerconfig.Server) error { return nil } +// pairFlag is a repeatable = flag. A second value for one key replaces the first. +type pairFlag struct { + flag, want string + m map[string]string +} + +func (f *pairFlag) String() string { return "" } + +func (f *pairFlag) Set(s string) error { + k, v, ok := strings.Cut(s, "=") + if !ok || k == "" { + return fmt.Errorf("want %s", f.want) + } + if f.m == nil { + f.m = map[string]string{} + } + f.m[k] = v + return nil +} + +// checkProtocolFlags checks add's --api, --auth-header and --header with the rules the +// router loads them by, and refuses a header value with $, which config.Load would expand. +// The errors never repeat a header value. +func checkProtocolFlags(s routerconfig.Server) error { + if err := routerconfig.CheckAPI(s.API); err != nil { + return fmt.Errorf("--%w", err) + } + if err := routerconfig.CheckAuthHeader(s.AuthHeader); err != nil { + return fmt.Errorf("--auth-header: %w", err) + } + if err := routerconfig.CheckHeaders(s.Headers, s.AuthHeader); err != nil { + return fmt.Errorf("--header: %w", err) + } + for _, name := range slices.Sorted(maps.Keys(s.Headers)) { + if strings.Contains(s.Headers[name], "$") { + return fmt.Errorf("--header %s: the value contains $, which Cortex reads as an environment variable when it "+ + "loads the config; write headers: {%s: ${NAME}} by hand", name, name) + } + } + return nil +} + +// sortedMap is m as a YAML mapping, its keys sorted, so the same flags write the same file. +func sortedMap(m map[string]string) *yaml.Node { + var pairs []string + for _, k := range slices.Sorted(maps.Keys(m)) { + pairs = append(pairs, k, m[k]) + } + return edit.MapValue(pairs...) +} + func serverRemove(args []string, stdout, stderr io.Writer) int { flags := flag.NewFlagSet("server remove", flag.ContinueOnError) cfgPath := flags.String("config", "", "Cortex config file (default ~/.cortex/config.yaml)") diff --git a/cmd/agentop/cmd_server_write_test.go b/cmd/agentop/cmd_server_write_test.go index f218f6d16..b841fb88c 100644 --- a/cmd/agentop/cmd_server_write_test.go +++ b/cmd/agentop/cmd_server_write_test.go @@ -1058,3 +1058,64 @@ func TestServerMove_AnEmptyListingPrunesNothing(t *testing.T) { t.Errorf("config:\n%s", cfg) } } + +// Rule C4: the three fields are written under the server, the maps sorted, and a +// second --api for one protocol replaces the first. +func TestServerAdd_WritesTheAPIAuthHeaderAndHeaders(t *testing.T) { + path := serverEnv(t, newFakeStats(t, 0).addr(), "") + code, _, errOut := runServerCmd(t, "sk-bob", "add", "bob", "https://bob.example.com", "--main", "opus", "--helper", "haiku", + "--api", "chat-completions=/x", "--api", "chat-completions=/v1/chat/completions", "--api", "anthropic-messages=/v1/messages", + "--auth-header", "x-api-key", "--header", "User-Agent=bob-shell", "--header", "X-Client=cortex", + "--key-stdin", "--config", path) + if code != 0 { + t.Fatalf("exit %d: %s", code, errOut) + } + want := ` bob: + url: https://bob.example.com + key: sk-bob + main: opus + helper: haiku + api: + anthropic-messages: /v1/messages + chat-completions: /v1/chat/completions + auth_header: x-api-key + headers: + User-Agent: bob-shell + X-Client: cortex +` + if got := readConfig(t, path); !strings.HasSuffix(got, want) { + t.Errorf("config does not end with the new entry:\n%s", got) + } +} + +// Each bad flag is refused with exit 2 before the key is asked for, and the file is +// not written. +func TestServerAdd_RefusesABadAPIOrHeaderFlagBeforeAskingForAKey(t *testing.T) { + asked := stubKeyPrompt(t, "sk") + path := serverEnv(t, newFakeStats(t, 0).addr(), "") + before := readConfig(t, path) + for _, tc := range []struct { + flags []string + want string + }{ + {[]string{"--api", "bogus=/x"}, "valid protocols: anthropic-messages, chat-completions, responses"}, + {[]string{"--api", "chat-completions"}, "want ="}, + {[]string{"--api", "chat-completions=v1"}, "must be absolute"}, + {[]string{"--auth-header", "Host"}, "cannot carry the key"}, + {[]string{"--header", "X-A=$HOME"}, "--header X-A: the value contains $"}, + {[]string{"--header", "Authorization=x"}, `"Authorization" cannot be set here`}, + {[]string{"--header", "X-A"}, "want ="}, + } { + args := append([]string{"add", "bob", "https://bob.example.com", "--main", "opus", "--helper", "haiku", "--config", path}, tc.flags...) + code, _, errOut := runServerCmd(t, "", args...) + if code != 2 || !strings.Contains(errOut, tc.want) { + t.Errorf("%v: exit %d, stderr:\n%s", tc.flags, code, errOut) + } + if strings.Contains(errOut, "$HOME") { + t.Errorf("%v: stderr repeats the header value:\n%s", tc.flags, errOut) + } + } + if *asked != "" || readConfig(t, path) != before { + t.Error("a bad flag asked for a key or wrote the file") + } +} From 81494c54d4508ce68726964d44714cee75a05410 Mon Sep 17 00:00:00 2001 From: Hai Huang Date: Tue, 17 Nov 2026 16:34:24 -0500 Subject: [PATCH 15/15] feat: Route requests by the server api map and translate them The inference router now reads the api, auth_header and headers fields of a server (rules 4 to 7 and C1 to C3 of #1911): - When the protocol of the agent is a key of api, the router sends the request to the path of that key. - When the protocol of the agent can be decoded and is not a key, the router asks the listener to translate the request to the first protocol of the map that has an encoder. It does not change the path. - When no protocol of the map has an encoder, the router refuses the request with a 502 in the protocol of the agent. - count_tokens to a server with no anthropic-messages entry gets a local 404 not_found_error. - setAuth puts the key in auth_header, or in the usual header of the target protocol for a translated request, and removes the other auth headers of the agent. Otherwise setKey applies as before. - The fixed headers go to SetUpstreamHeaders. inference.ProtocolFor gives the protocol of a path with the rule of dialectFor, so that the parser and the router cannot disagree. Assisted-By: Claude (Anthropic AI) Signed-off-by: Hai Huang --- .../forwardproxy/inferencerouter_test.go | 248 ++++++++++++++++++ core/parse/inference/dialect.go | 22 ++ core/parse/inference/dialect_test.go | 18 ++ core/plugins/inferencerouter/api.go | 122 +++++++++ core/plugins/inferencerouter/api_test.go | 215 +++++++++++++++ core/plugins/inferencerouter/plugin.go | 52 +++- 6 files changed, 671 insertions(+), 6 deletions(-) create mode 100644 core/plugins/inferencerouter/api.go create mode 100644 core/plugins/inferencerouter/api_test.go diff --git a/core/listener/forwardproxy/inferencerouter_test.go b/core/listener/forwardproxy/inferencerouter_test.go index 75b4d7e3c..f1aac9ad3 100644 --- a/core/listener/forwardproxy/inferencerouter_test.go +++ b/core/listener/forwardproxy/inferencerouter_test.go @@ -1,10 +1,13 @@ package forwardproxy import ( + "bytes" + "context" "encoding/json" "io" "net/http" "net/http/httptest" + "path/filepath" "slices" "strings" "sync" @@ -16,6 +19,7 @@ import ( "github.com/rossoctl/cortex/core/plugins/inferenceparser" "github.com/rossoctl/cortex/core/plugins/inferencerouter" "github.com/rossoctl/cortex/core/session" + "github.com/rossoctl/cortex/core/storage/filestore" ) // seen is every request an origin received, by method and Authorization header — @@ -378,3 +382,247 @@ func TestForwardProxy_ASessionsOverrideMovesARunningSession(t *testing.T) { t.Errorf("ete's own turn received %v, want its own e1 and e2", got) } } + +const routedKey = "s-key" + +// routedServer is an upstream for the router's api tests. It lists claude-sonnet-5 at +// /v1/models, answers every other request with answer, and keeps those requests. +func routedServer(t *testing.T, answer http.HandlerFunc) *chatServer { + t.Helper() + return newChatServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && r.URL.Path == "/v1/models" { + _, _ = io.WriteString(w, `{"data":[{"id":"claude-sonnet-5"}]}`) + return + } + answer(w, r) + }) +} + +// sent are the requests up got, without the router's model-list fetches. +func sent(up *chatServer) []upstreamRequest { + var out []upstreamRequest + for _, r := range up.got() { + if r.path != "/v1/models" { + out = append(out, r) + } + } + return out +} + +// routedProxy is a forward proxy with inference-parser and an inference router that +// sends claude-code and opencode to up, a server with the extra fields given. The +// router's model list for up is claude-sonnet-5 from the start, so that a resend has +// a substitute. +func routedProxy(t *testing.T, up *chatServer, fields string) (*httptest.Server, *session.Store) { + t.Helper() + st, err := filestore.Open(filepath.Join(t.TempDir(), "store.json")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = st.Close() }) + _ = st.Set(context.Background(), inferencerouter.Name+"/models/s", `["claude-sonnet-5"]`, 0) + router := inferencerouter.New() + router.SetStore(st) + if err := router.Configure(json.RawMessage(`{ + "servers": {"s": {"url": "` + up.URL + `", "key": "` + routedKey + `", "main": "sonnet", "helper": "sonnet"` + fields + `}}, + "agents": {"claude-code": "s", "opencode": "s"} + }`)); err != nil { + t.Fatalf("Configure: %v", err) + } + if err := router.Init(context.Background()); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = router.Shutdown(context.Background()) }) + p, err := pipeline.New([]pipeline.Plugin{inferenceparser.NewInferenceParser(), router}) + if err != nil { + t.Fatalf("pipeline.New: %v", err) + } + store := newStore(t) + shared := memstore.New() + t.Cleanup(shared.Close) + srv := &Server{ + OutboundPipeline: pipeline.NewHolder(p), + Sessions: store, + Shared: shared, + SessionIDHeaders: []string{session.ClaudeCodeSessionHeader}, + Client: http.DefaultClient, + } + proxy := httptest.NewServer(srv.Handler()) + t.Cleanup(proxy.Close) + return proxy, store +} + +// claudeCodeRequest is Claude Code's request header set, with its X-Api-Key. +func claudeCodeRequest(r *http.Request) { + r.Header.Set("User-Agent", "claude-cli/2.1.286 (external, cli)") + r.Header.Set("X-Api-Key", "client-key") + r.Header.Set(session.ClaudeCodeSessionHeader, "s1") +} + +// Rule C2 and the auth test: Claude Code's X-Api-Key request reaches a chat server +// with only Authorization: Bearer , and a chat body. +func TestRouterAPI_ChatServerGetsBearerAndAChatBody(t *testing.T) { + up := routedServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, replyFixture(t, "bob_shape1.json")) + }) + proxy, _ := routedProxy(t, up, `, "api": {"chat-completions": "/v1/chat/completions"}`) + resp := postAnthropic(t, proxy, anthropicRequest, claudeCodeRequest) + body := replyText(t, resp) + + if resp.StatusCode != http.StatusOK || !strings.Contains(body, `"type":"message"`) { + t.Errorf("reply = %d %s, want an Anthropic message", resp.StatusCode, body) + } + reqs := sent(up) + if len(reqs) != 1 { + t.Fatalf("upstream got %d requests, want 1", len(reqs)) + } + r := reqs[0] + if r.path != "/v1/chat/completions" || !bytes.Contains(r.body, []byte(`"messages":[{"role":"user","content":[`)) { + t.Errorf("upstream got %s %s, want a chat body at /v1/chat/completions", r.path, r.body) + } + if got := r.header.Values("Authorization"); len(got) != 1 || got[0] != "Bearer "+routedKey { + t.Errorf("Authorization = %q, want only Bearer %s", got, routedKey) + } + for _, h := range []string{"X-Api-Key", "Api-Key", "Anthropic-Version"} { + if got := r.header.Values(h); len(got) != 0 { + t.Errorf("%s = %q, want none", h, got) + } + } +} + +// Rule 4a: nothing goes upstream, and the agent gets an Anthropic api_error. +func TestRouterAPI_ResponsesOnlyServerIsA502(t *testing.T) { + up := routedServer(t, func(w http.ResponseWriter, _ *http.Request) { t.Error("the server got a request") }) + proxy, _ := routedProxy(t, up, `, "api": {"responses": "/v1/responses"}`) + resp := postAnthropic(t, proxy, anthropicRequest, claudeCodeRequest) + body := replyText(t, resp) + + const msg = "Cortex cannot translate anthropic-messages to responses yet. Use a translation server for this pair " + + "(see https://github.com/rossoctl/cortex/blob/main/docs/translation-server.md)." + var got struct { + Type string `json:"type"` + Error struct{ Type, Message string } + } + if resp.StatusCode != http.StatusBadGateway || json.Unmarshal([]byte(body), &got) != nil || + got.Type != "error" || got.Error.Type != "api_error" || got.Error.Message != msg { + t.Errorf("reply = %d %s, want a 502 Anthropic api_error with %q", resp.StatusCode, body, msg) + } + if reqs := sent(up); len(reqs) != 0 { + t.Errorf("upstream got %d requests, want none", len(reqs)) + } +} + +// Rule C1: the agent's protocol is a key, so the request goes untranslated to its path. +func TestRouterAPI_AnthropicPathEntry(t *testing.T) { + up := routedServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"type":"message","role":"assistant","content":[]}`) + }) + proxy, _ := routedProxy(t, up, `, "api": {"anthropic-messages": "/anthropic/v1/messages"}`) + resp := postAnthropic(t, proxy, anthropicRequest, claudeCodeRequest) + _ = replyText(t, resp) + + reqs := sent(up) + if len(reqs) != 1 || reqs[0].path != "/anthropic/v1/messages" || string(reqs[0].body) != anthropicRequest { + t.Fatalf("upstream got %+v, want the body unchanged at /anthropic/v1/messages", reqs) + } + // Not translated, empty auth_header: setKey keeps the agent's header. + if got := reqs[0].header.Get("X-Api-Key"); got != routedKey { + t.Errorf("X-Api-Key = %q, want %s", got, routedKey) + } +} + +// Rule 7: a chat request cannot be decoded, so it is not translated and gets no 502. +func TestRouterAPI_AChatRequestIsNotTranslated(t *testing.T) { + up := routedServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, replyFixture(t, "bob_shape1.json")) + }) + proxy, _ := routedProxy(t, up, `, "api": {"anthropic-messages": "/v1/messages"}`) + const chatBody = `{"model":"claude-opus-5-5","messages":[{"role":"user","content":"hi"}]}` + req, _ := http.NewRequest(http.MethodPost, "http://api.openai.example/v1/chat/completions", strings.NewReader(chatBody)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "opencode/latest/2.0.21/cli") + resp, err := proxyClient(proxy, nil).Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + _, _ = io.ReadAll(resp.Body) + + if resp.StatusCode != http.StatusOK { + t.Errorf("status = %d, want 200", resp.StatusCode) + } + reqs := sent(up) + if len(reqs) != 1 || reqs[0].path != "/v1/chat/completions" || string(reqs[0].body) != chatBody { + t.Errorf("upstream got %+v, want the chat body unchanged at /v1/chat/completions", reqs) + } +} + +// Rule 6: count_tokens routed to a chat server is answered here with a 404. +func TestRouterAPI_CountTokensIsALocal404(t *testing.T) { + up := routedServer(t, func(w http.ResponseWriter, _ *http.Request) { t.Error("the server got a request") }) + proxy, _ := routedProxy(t, up, `, "api": {"chat-completions": "/v1/chat/completions"}`) + req, _ := http.NewRequest(http.MethodPost, up.URL+"/v1/messages/count_tokens", strings.NewReader(anthropicRequest)) + req.Header.Set("Content-Type", "application/json") + claudeCodeRequest(req) + resp, err := proxyClient(proxy, nil).Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + + if resp.StatusCode != http.StatusNotFound || !bytes.Contains(body, []byte(`"not_found_error"`)) { + t.Errorf("reply = %d %s, want a 404 not_found_error", resp.StatusCode, body) + } + if reqs := sent(up); len(reqs) != 0 { + t.Errorf("upstream got %d requests, want none", len(reqs)) + } +} + +// A translated request that the server refuses for its model is resent with the +// router's substitute, as an untranslated one is: LiteLLM's 403 and Bob's 404. +func TestRouterAPI_ATranslatedRefusalIsResent(t *testing.T) { + for _, refusal := range []struct { + name string + status int + fixture string + }{ + {"litellm 403", http.StatusForbidden, "error_403_litellm.json"}, + {"bob 404", http.StatusNotFound, "error_404_bob.json"}, + } { + t.Run(refusal.name, func(t *testing.T) { + var mu sync.Mutex + calls := 0 + up := routedServer(t, func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + calls++ + n := calls + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + if n == 1 { + w.WriteHeader(refusal.status) + _, _ = io.WriteString(w, replyFixture(t, refusal.fixture)) + return + } + _, _ = io.WriteString(w, replyFixture(t, "bob_shape1.json")) + }) + proxy, _ := routedProxy(t, up, `, "api": {"chat-completions": "/v1/chat/completions"}`) + resp := postAnthropic(t, proxy, anthropicRequest, claudeCodeRequest) + body := replyText(t, resp) + + if resp.StatusCode != http.StatusOK || !strings.Contains(body, "391 is not prime") { + t.Errorf("reply = %d %s, want the second answer, translated", resp.StatusCode, body) + } + reqs := sent(up) + if len(reqs) != 2 { + t.Fatalf("upstream got %d requests, want 2", len(reqs)) + } + if !bytes.Contains(reqs[1].body, []byte(`"model":"claude-sonnet-5"`)) || reqs[1].path != "/v1/chat/completions" { + t.Errorf("resent request = %s %s, want a chat body with the substitute", reqs[1].path, reqs[1].body) + } + }) + } +} diff --git a/core/parse/inference/dialect.go b/core/parse/inference/dialect.go index 4a8ebee70..69979b253 100644 --- a/core/parse/inference/dialect.go +++ b/core/parse/inference/dialect.go @@ -60,3 +60,25 @@ func dialectFor(path string) dialect { } return dialectNone } + +// chatCompletionsSuffix ends a chat-completions path. The OpenAI dialect also reads +// legacy completions, which have no translation protocol. +const chatCompletionsSuffix = "/chat/completions" + +// ProtocolFor names the translation protocol of path, by the same rule as dialectFor, +// so that the parser and the translator cannot disagree: "anthropic-messages", +// "chat-completions", "responses", or "" for a path with no protocol (legacy +// completions included). +func ProtocolFor(path string) string { + switch dialectFor(path) { + case dialectAnthropic: + return "anthropic-messages" + case dialectOpenAI: + if strings.HasSuffix(path, chatCompletionsSuffix) { + return "chat-completions" + } + case dialectResponses: + return "responses" + } + return "" +} diff --git a/core/parse/inference/dialect_test.go b/core/parse/inference/dialect_test.go index 5d4ea3eb9..ef812dbe0 100644 --- a/core/parse/inference/dialect_test.go +++ b/core/parse/inference/dialect_test.go @@ -47,3 +47,21 @@ func TestDialectFor(t *testing.T) { } } } + +// ProtocolFor names the protocol of a path with the rule that dialectFor uses +// (decision 14). Legacy completions and count_tokens have no protocol. +func TestProtocolFor(t *testing.T) { + for path, want := range map[string]string{ + "/v1/messages": "anthropic-messages", + "/anthropic/v1/messages": "anthropic-messages", + "/v1/chat/completions": "chat-completions", + "/v1/completions": "", + "/v1/responses": "responses", + "/v1/messages/count_tokens": "", + "/v1/models": "", + } { + if got := ProtocolFor(path); got != want { + t.Errorf("ProtocolFor(%q) = %q, want %q", path, got, want) + } + } +} diff --git a/core/plugins/inferencerouter/api.go b/core/plugins/inferencerouter/api.go new file mode 100644 index 000000000..dff4fd9ad --- /dev/null +++ b/core/plugins/inferencerouter/api.go @@ -0,0 +1,122 @@ +package inferencerouter + +import ( + "net/http" + "slices" + "strings" + + "github.com/rossoctl/cortex/core/parse/inference" + "github.com/rossoctl/cortex/core/pipeline" + "github.com/rossoctl/cortex/core/translate" +) + +// planKind is what the server's api map does to a request (rules 4 to 7, C1). +type planKind int + +const ( + planAsIs planKind = iota // the agent's protocol, to the agent's path + planPath // the agent's protocol, to the api map's path + planTranslate // converted to target, sent to path + planRefuse // no protocol of the map can be translated to (rule 4a) + planCountTokens // answered here with a 404 (rule 6) +) + +type apiPlan struct { + kind planKind + source, target, path string +} + +// countTokensSuffix ends Anthropic's token-count path. +const countTokensSuffix = "/v1/messages/count_tokens" + +// translationServerDoc is the fallback doc that the rule 4a error names. +const translationServerDoc = "https://github.com/rossoctl/cortex/blob/main/docs/translation-server.md" + +// plan says how a request to path goes to r, from r's api map. +func (r route) plan(path string) apiPlan { + if len(r.api) == 0 { + return apiPlan{kind: planAsIs} + } + if _, ok := r.api[translate.Anthropic]; !ok && strings.HasSuffix(path, countTokensSuffix) { + return apiPlan{kind: planCountTokens} + } + src := inference.ProtocolFor(path) + if p, ok := r.api[src]; ok { + return apiPlan{kind: planPath, path: p} + } + if translate.Decodes(src) { + tgt, ok := translate.Target(apiKeys(r)) + if !ok { + return apiPlan{kind: planRefuse, source: src} + } + return apiPlan{kind: planTranslate, source: src, target: tgt, path: r.api[tgt]} + } + return apiPlan{kind: planAsIs} +} + +// apiKeys are the protocols of r's api map, sorted. +func apiKeys(r route) []string { + keys := make([]string, 0, len(r.api)) + for k := range r.api { + keys = append(keys, k) + } + slices.Sort(keys) + return keys +} + +// cannotTranslate is the rule 4a message (decision 13). +func cannotTranslate(source string, r route) string { + return "Cortex cannot translate " + source + " to " + strings.Join(apiKeys(r), " or ") + + " yet. Use a translation server for this pair (see " + translationServerDoc + ")." +} + +// refusePlan records and gives the local answer for planRefuse and planCountTokens. +func refusePlan(pctx *pipeline.Context, r route, pl apiPlan, details map[string]string) pipeline.Action { + if pl.kind == planCountTokens { + pctx.Record(pipeline.Invocation{Action: pipeline.ActionDeny, Reason: "count_tokens_local", Details: details}) + const msg = "this inference server does not count tokens" + return pipeline.Action{Type: pipeline.Reject, Violation: &pipeline.Violation{ + Code: "inference.count_tokens_unavailable", Reason: msg, Status: http.StatusNotFound, + Body: translate.ErrorBody(translate.Anthropic, http.StatusNotFound, "not_found_error", msg), + BodyType: "application/json", + }} + } + details["from"] = pl.source + pctx.Record(pipeline.Invocation{Action: pipeline.ActionDeny, Reason: "cannot_translate", Details: details}) + msg := cannotTranslate(pl.source, r) + return pipeline.Action{Type: pipeline.Reject, Violation: &pipeline.Violation{ + Code: "inference.translation_unavailable", Reason: msg, Status: http.StatusBadGateway, + Body: translate.ErrorBody(pl.source, http.StatusBadGateway, "api_error", msg), + BodyType: "application/json", + }} +} + +// authHeaderNames are the headers an agent can carry its key in. +var authHeaderNames = []string{"Authorization", "X-Api-Key", "Api-Key"} + +// setAuth puts r's key on the request (rule C2). translatedTo is the target protocol, +// "" when the request is not translated. With no auth_header and no translation, it +// is setKey: the header the agent used. +func setAuth(pctx *pipeline.Context, r route, translatedTo string) { + name := r.authHeader + switch { + case name == "" && translatedTo == translate.Chat: + name = "authorization" + case name == "" && translatedTo == translate.Anthropic: + name = "x-api-key" + case name == "": + setKey(pctx, r.key) + return + } + if pctx.Headers == nil { + pctx.Headers = http.Header{} + } + for _, h := range authHeaderNames { + pctx.Headers.Del(h) + } + if strings.EqualFold(name, "authorization") { + pctx.Headers.Set("Authorization", "Bearer "+r.key) + return + } + pctx.Headers.Set(name, r.key) +} diff --git a/core/plugins/inferencerouter/api_test.go b/core/plugins/inferencerouter/api_test.go new file mode 100644 index 000000000..79eb0e4e1 --- /dev/null +++ b/core/plugins/inferencerouter/api_test.go @@ -0,0 +1,215 @@ +package inferencerouter + +import ( + "encoding/json" + "net/http" + "slices" + "strings" + "testing" + + "github.com/rossoctl/cortex/core/pipeline" + "github.com/rossoctl/cortex/core/plugins/inferencerouter/routerconfig" + "github.com/rossoctl/cortex/core/translate" +) + +// routerconfig keeps its own list, because it must not import core/translate +// (decision 15). This pins the two lists equal. +func TestAPIProtocolsMatchRegistry(t *testing.T) { + if !slices.Equal(routerconfig.APIProtocols, translate.Names()) { + t.Errorf("routerconfig.APIProtocols = %v, translate.Names() = %v; want equal", routerconfig.APIProtocols, translate.Names()) + } +} + +func TestPlan(t *testing.T) { + for _, tc := range []struct { + name string + api map[string]string + path string + want apiPlan + }{ + {"empty map", nil, "/v1/messages", apiPlan{kind: planAsIs}}, + {"chat only", map[string]string{"chat-completions": "/v1/chat/completions"}, "/v1/messages", + apiPlan{kind: planTranslate, source: "anthropic-messages", target: "chat-completions", path: "/v1/chat/completions"}}, + {"anthropic at its own path", map[string]string{"anthropic-messages": "/anthropic/v1/messages"}, "/v1/messages", + apiPlan{kind: planPath, path: "/anthropic/v1/messages"}}, + {"responses only", map[string]string{"responses": "/v1/responses"}, "/v1/messages", + apiPlan{kind: planRefuse, source: "anthropic-messages"}}, + // Rule 7: a chat request cannot be decoded, so it is not translated. + {"no decoder", map[string]string{"anthropic-messages": "/v1/messages", "responses": "/v1/responses"}, "/v1/chat/completions", + apiPlan{kind: planAsIs}}, + {"count_tokens to a chat server", map[string]string{"chat-completions": "/v1/chat/completions"}, "/v1/messages/count_tokens", + apiPlan{kind: planCountTokens}}, + {"count_tokens to an anthropic server", map[string]string{"anthropic-messages": "/v1/messages"}, "/v1/messages/count_tokens", + apiPlan{kind: planAsIs}}, + {"DeepSeek chat", map[string]string{"chat-completions": "/chat/completions", "anthropic-messages": "/anthropic/v1/messages"}, "/v1/chat/completions", + apiPlan{kind: planPath, path: "/chat/completions"}}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := (route{api: tc.api}).plan(tc.path); got != tc.want { + t.Errorf("plan(%q) = %+v, want %+v", tc.path, got, tc.want) + } + }) + } +} + +// apiConfig is one server, "s", with the extra server fields given, and claude-code +// routed to it. +func apiConfig(fields string) string { + return `{ + "servers": {"s": {"url": "https://s.example.com", "key": "s-key", "main": "opus", "helper": "haiku"` + fields + `}}, + "agents": {"claude-code": "s"} + }` +} + +const chatAPI = `, "api": {"chat-completions": "/v1/chat/completions"}` + +// Rule C2: the key goes in one header, and the agent's other auth headers are gone. +func TestRouter_SetAuth(t *testing.T) { + for _, tc := range []struct { + name string + fields string + sent http.Header + want http.Header // every auth header the request must carry + }{ + {"not translated, empty auth_header: setKey", "", http.Header{"X-Api-Key": {"client-key"}}, + http.Header{"X-Api-Key": {"s-key"}}}, + {"translated to chat", chatAPI, http.Header{"X-Api-Key": {"client-key"}, "Api-Key": {"client-key"}}, + http.Header{"Authorization": {"Bearer s-key"}}}, + {"path entry, not translated: setKey", `, "api": {"anthropic-messages": "/x/v1/messages"}`, http.Header{"Authorization": {"Bearer client-key"}}, + http.Header{"Authorization": {"Bearer s-key"}}}, + {"auth_header x-api-key", chatAPI + `, "auth_header": "x-api-key"`, http.Header{"Authorization": {"Bearer client-key"}}, + http.Header{"X-Api-Key": {"s-key"}}}, + {"auth_header api-key", `, "auth_header": "api-key"`, http.Header{"X-Api-Key": {"client-key"}, "Authorization": {"Bearer client-key"}}, + http.Header{"Api-Key": {"s-key"}}}, + {"auth_header authorization", `, "auth_header": "authorization"`, http.Header{"X-Api-Key": {"client-key"}}, + http.Header{"Authorization": {"Bearer s-key"}}}, + } { + t.Run(tc.name, func(t *testing.T) { + p := build(t, apiConfig(tc.fields)) + pctx := request(newStore(t), "api.anthropic.com", claudeUA, "s1") + pctx.Headers = tc.sent + pctx.Headers.Set("User-Agent", claudeUA) + if a := run(t, p, pctx); a.Type != pipeline.Continue { + t.Fatalf("action = %+v, want Continue", a) + } + for _, h := range []string{"Authorization", "X-Api-Key", "Api-Key"} { + if got, want := pctx.Headers.Values(h), tc.want.Values(h); !slices.Equal(got, want) { + t.Errorf("%s = %q, want %q", h, got, want) + } + } + }) + } +} + +// A request translated to Anthropic Messages gets the key in X-Api-Key only. No +// protocol of this issue can be translated to it, so setAuth is called directly. +func TestSetAuth_TranslatedToAnthropic(t *testing.T) { + pctx := &pipeline.Context{Headers: http.Header{"Authorization": {"Bearer client-key"}, "Api-Key": {"client-key"}}} + setAuth(pctx, route{key: "s-key"}, "anthropic-messages") + if got := pctx.Headers; len(got) != 1 || got.Get("X-Api-Key") != "s-key" { + t.Errorf("headers = %v, want only X-Api-Key: s-key", got) + } +} + +// Rule 5: the router does not change Path and does not call SetRedirectPath. +func TestRouter_ATranslatedRequestKeepsItsPath(t *testing.T) { + p := build(t, apiConfig(chatAPI)) + pctx := request(newStore(t), "api.anthropic.com", claudeUA, "s1") + pctx.Path = "/zen/v1/messages" + run(t, p, pctx) + + if pctx.Path != "/zen/v1/messages" { + t.Errorf("Path = %q, want the client's path", pctx.Path) + } + if path, ok := pctx.RedirectPath(); ok { + t.Errorf("RedirectPath = %q, want none for a translated request", path) + } + src, tgt, path, ok := pctx.TranslateTarget() + if !ok || src != "anthropic-messages" || tgt != "chat-completions" || path != "/v1/chat/completions" { + t.Errorf("TranslateTarget = %q %q %q %v, want anthropic-messages to chat-completions at /v1/chat/completions", src, tgt, path, ok) + } + assertRecord(t, pctx, pipeline.ActionModify, "routed", map[string]string{"server": "s"}) +} + +func TestRouter_APathEntrySetsTheRedirectPath(t *testing.T) { + p := build(t, apiConfig(`, "api": {"anthropic-messages": "/anthropic/v1/messages"}`)) + pctx := request(newStore(t), "api.anthropic.com", claudeUA, "s1") + run(t, p, pctx) + + if path, ok := pctx.RedirectPath(); !ok || path != "/anthropic/v1/messages" { + t.Errorf("RedirectPath = %q %v, want /anthropic/v1/messages", path, ok) + } + if _, _, _, ok := pctx.TranslateTarget(); ok { + t.Error("TranslateTarget is set, want no translation") + } +} + +func TestRouter_HeadersReachUpstreamHeaders(t *testing.T) { + p := build(t, apiConfig(`, "headers": {"user-agent": "bob-shell"}`)) + pctx := request(newStore(t), "api.anthropic.com", claudeUA, "s1") + run(t, p, pctx) + + if got := pctx.UpstreamHeaders().Values("User-Agent"); !slices.Equal(got, []string{"bob-shell"}) { + t.Errorf("UpstreamHeaders User-Agent = %q, want bob-shell", got) + } +} + +func TestRouter_NoHeadersSetsNoUpstreamHeaders(t *testing.T) { + p := build(t, apiConfig("")) + pctx := request(newStore(t), "api.anthropic.com", claudeUA, "s1") + run(t, p, pctx) + + if h := pctx.UpstreamHeaders(); h != nil { + t.Errorf("UpstreamHeaders = %v, want nil", h) + } +} + +// Rule 4a, with the decision 13 message. +func TestRouter_RefusesAPairItCannotTranslate(t *testing.T) { + p := build(t, apiConfig(`, "api": {"responses": "/v1/responses"}`)) + pctx := request(newStore(t), "api.anthropic.com", claudeUA, "s1") + a := run(t, p, pctx) + + const msg = "Cortex cannot translate anthropic-messages to responses yet. Use a translation server for this pair " + + "(see https://github.com/rossoctl/cortex/blob/main/docs/translation-server.md)." + if a.Type != pipeline.Reject || a.Violation == nil { + t.Fatalf("action = %+v, want Reject", a) + } + v := a.Violation + if v.Code != "inference.translation_unavailable" || v.Status != http.StatusBadGateway || v.Reason != msg || v.BodyType != "application/json" { + t.Errorf("violation = %+v, want a 502 inference.translation_unavailable with the decision 13 message", v) + } + var body struct { + Type string `json:"type"` + Error struct{ Type, Message string } + } + if err := json.Unmarshal(v.Body, &body); err != nil || body.Type != "error" || body.Error.Type != "api_error" || body.Error.Message != msg { + t.Errorf("body = %s (%v), want an Anthropic api_error with the message", v.Body, err) + } + assertRecord(t, pctx, pipeline.ActionDeny, "cannot_translate", map[string]string{"server": "s", "from": "anthropic-messages"}) +} + +func TestRouter_TheRefusalNamesEveryAPIKey(t *testing.T) { + r := route{api: map[string]string{"responses": "/v1/responses", "anthropic-messages": "/v1/messages"}} + got := cannotTranslate("chat-completions", r) + if !strings.HasPrefix(got, "Cortex cannot translate chat-completions to anthropic-messages or responses yet.") { + t.Errorf("message = %q, want the sorted keys joined with or", got) + } +} + +// Rule 6: count_tokens to a server with no anthropic-messages entry is answered here. +func TestRouter_AnswersCountTokensLocally(t *testing.T) { + p := build(t, apiConfig(chatAPI)) + pctx := request(newStore(t), "s.example.com", claudeUA, "s1") + pctx.Path, pctx.Extensions.Inference = "/v1/messages/count_tokens", nil + a := run(t, p, pctx) + + if a.Type != pipeline.Reject || a.Violation == nil || a.Violation.Status != http.StatusNotFound { + t.Fatalf("action = %+v, want a 404 Reject", a) + } + if !strings.Contains(string(a.Violation.Body), `"not_found_error"`) || + !strings.Contains(string(a.Violation.Body), "this inference server does not count tokens") { + t.Errorf("body = %s, want an Anthropic not_found_error", a.Violation.Body) + } + assertRecord(t, pctx, pipeline.ActionDeny, "count_tokens_local", map[string]string{"server": "s"}) +} diff --git a/core/plugins/inferencerouter/plugin.go b/core/plugins/inferencerouter/plugin.go index 6e0c1526e..9e9247860 100644 --- a/core/plugins/inferencerouter/plugin.go +++ b/core/plugins/inferencerouter/plugin.go @@ -187,6 +187,11 @@ type route struct { // main and helper are the words naming the server's substitute models, resolved // against its model list when one is needed (see substitute). main, helper string + // api, authHeader and headers are the server's api, auth_header and headers + // fields (rules C1 to C3). headers is nil when the server has none. + api map[string]string + authHeader string + headers http.Header } // target is the server's own scheme and host, where every request routed to it is @@ -280,7 +285,15 @@ func (p *Router) Configure(raw json.RawMessage) error { "crosses the network decrypted, its key and prompt included; use https unless the network is trusted", "server", name, "url", ep.URL()) } - servers[name] = route{endpoint: ep, key: s.Key, main: s.Main, helper: s.Helper} + var headers http.Header + for k, v := range s.Headers { + if headers == nil { + headers = http.Header{} + } + headers.Set(k, v) + } + servers[name] = route{endpoint: ep, key: s.Key, main: s.Main, helper: s.Helper, + api: s.API, authHeader: s.AuthHeader, headers: headers} byHost[ep.Key()] = name } // p.agents is keyed by CANONICAL agent name, folded from whatever spelling the @@ -366,11 +379,21 @@ func (p *Router) OnRequest(_ context.Context, pctx *pipeline.Context) pipeline.A // sent it. Redirected is the signal that it moved, and it is this router's // redirect: a pipeline admits one WritesDestination plugin. redirected := redirectErr == nil && pctx.Redirected() + var pl apiPlan var pathErr error - if redirected && captured != "" { - // The table's paths are absolute with no query, so this is refused only by a - // drift between the table and SetRedirectPath's rule. - pathErr = pctx.SetRedirectPath(captured) + if redirected { + // The paths of the table and of the api map are absolute with no query, so + // these are refused only by a drift between the checks and the pipeline's. + switch pl = srv.plan(pctx.Path); pl.kind { + case planPath: + pathErr = pctx.SetRedirectPath(pl.path) + case planTranslate: + pathErr = pctx.Translate(pl.source, pl.target, pl.path) + case planAsIs: + if captured != "" { + pathErr = pctx.SetRedirectPath(captured) + } + } } err := cmp.Or(redirectErr, pathErr) outcome := redirectMoved @@ -397,6 +420,13 @@ func (p *Router) OnRequest(_ context.Context, pctx *pipeline.Context) pipeline.A pctx.Record(pipeline.Invocation{Action: pipeline.ActionObserve, Reason: "would_route", Details: details()}) return cont } + if pl.kind == planRefuse || pl.kind == planCountTokens { + // The session is on this server; only this request is answered here. + if settle { + pin.settle(name) + } + return refusePlan(pctx, srv, pl, details()) + } // The model is chosen only once the request has moved, so an observed router // changes nothing. p.chooseModel(pctx, name) @@ -409,7 +439,17 @@ func (p *Router) OnRequest(_ context.Context, pctx *pipeline.Context) pipeline.A if addressed != name { dropProviderCredentials(pctx.Headers) } - setKey(pctx, srv.key) + translatedTo := "" + if pl.kind == planTranslate { + translatedTo = pl.target + } + setAuth(pctx, srv, translatedTo) + if len(srv.headers) > 0 { + if err := pctx.SetUpstreamHeaders(srv.headers); err != nil { + pctx.Record(pipeline.Invocation{Action: pipeline.ActionDeny, Reason: "redirect_failed", Details: details()}) + return pipeline.Deny(codeUnavailable, fmt.Sprintf("inference-router could not send this request to %q: %v", name, err)) + } + } d := details() if captured != "" { d["captured"] = captured