diff --git a/core/translate/anthropic/error.go b/core/translate/anthropic/error.go new file mode 100644 index 000000000..b3d58a3e9 --- /dev/null +++ b/core/translate/anthropic/error.go @@ -0,0 +1,35 @@ +package anthropic + +import ( + "bytes" + "encoding/json" +) + +type errorDetail struct { + Type string `json:"type"` + Message string `json:"message"` +} + +type errorBody struct { + Type string `json:"type"` + Error errorDetail `json:"error"` +} + +// EncodeError gives an Anthropic error body (rule 24). The status goes on +// the HTTP reply, not in the body. +func (Protocol) EncodeError(_ int, errType, message string) []byte { + return marshal(errorBody{Type: "error", Error: errorDetail{Type: errType, Message: message}}) +} + +// marshal encodes v with no HTML escapes, so text goes to the agent as +// the server wrote it. v holds only fixed types and checked raw JSON, so +// an error is a bug. +func marshal(v any) []byte { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(v); err != nil { + panic("anthropic: marshal: " + err.Error()) + } + return bytes.TrimSuffix(buf.Bytes(), []byte("\n")) +} diff --git a/core/translate/anthropic/protocol.go b/core/translate/anthropic/protocol.go new file mode 100644 index 000000000..4d042f01e --- /dev/null +++ b/core/translate/anthropic/protocol.go @@ -0,0 +1,24 @@ +// Package anthropic is the anthropic-messages translation protocol: the +// Anthropic Messages API. It decodes a request to the pivot form and +// encodes pivot reply events as an Anthropic reply (rule H6). +package anthropic + +import "github.com/rossoctl/cortex/core/translate/pivot" + +// Name is the protocol name. +const Name = "anthropic-messages" + +// extraPrefix starts each pivot.Extra key that this protocol writes. +const extraPrefix = "anthropic." + +// Protocol is the anthropic-messages protocol. +type Protocol struct{} + +var ( + _ pivot.Protocol = Protocol{} + _ pivot.RequestDecoder = Protocol{} + _ pivot.ReplyEncoding = Protocol{} +) + +// Name gives "anthropic-messages". +func (Protocol) Name() string { return Name } diff --git a/core/translate/anthropic/reply.go b/core/translate/anthropic/reply.go new file mode 100644 index 000000000..6de890336 --- /dev/null +++ b/core/translate/anthropic/reply.go @@ -0,0 +1,275 @@ +package anthropic + +import ( + "encoding/json" + + "github.com/rossoctl/cortex/core/translate/pivot" +) + +// The output types have a fixed field order, so the bytes are the same +// each time (rule 9a). + +type startUsage struct { + Input int `json:"input_tokens"` + Output int `json:"output_tokens"` +} + +type fullUsage struct { + Input int `json:"input_tokens"` + CacheCreation int `json:"cache_creation_input_tokens"` + CacheRead int `json:"cache_read_input_tokens"` + Output int `json:"output_tokens"` +} + +func usageOf(u pivot.Usage) fullUsage { + return fullUsage{Input: u.Input, CacheCreation: u.CacheCreation, CacheRead: u.CacheRead, Output: u.Output} +} + +type message struct { + ID string `json:"id"` + Type string `json:"type"` + Role string `json:"role"` + Model string `json:"model"` + Content []any `json:"content"` + StopReason *string `json:"stop_reason"` + StopSequence *string `json:"stop_sequence"` + Usage any `json:"usage"` +} + +type textBlock struct { + Type string `json:"type"` + Text string `json:"text"` +} + +type thinkingBlock struct { + Type string `json:"type"` + Thinking string `json:"thinking"` + Signature string `json:"signature"` +} + +type toolUseBlock struct { + Type string `json:"type"` + ID string `json:"id"` + Name string `json:"name"` + Input json.RawMessage `json:"input"` +} + +type kindBlock struct { + Type string `json:"type"` +} + +// block gives the Anthropic content block of one kind. +func block(kind pivot.BlockKind, text, signature, id, name string, input json.RawMessage) any { + switch kind { + case pivot.BlockText: + return textBlock{Type: "text", Text: text} + case pivot.BlockThinking: + return thinkingBlock{Type: "thinking", Thinking: text, Signature: signature} + case pivot.BlockToolUse: + return toolUseBlock{Type: "tool_use", ID: id, Name: name, Input: input} + } + return kindBlock{Type: string(kind)} +} + +type messageStart struct { + Type string `json:"type"` + Message message `json:"message"` +} + +type blockStart struct { + Type string `json:"type"` + Index int `json:"index"` + ContentBlock any `json:"content_block"` +} + +type blockDelta struct { + Type string `json:"type"` + Index int `json:"index"` + Delta any `json:"delta"` +} + +type blockStop struct { + Type string `json:"type"` + Index int `json:"index"` +} + +type textDelta struct { + Type string `json:"type"` + Text string `json:"text"` +} + +type thinkingDelta struct { + Type string `json:"type"` + Thinking string `json:"thinking"` +} + +type signatureDelta struct { + Type string `json:"type"` + Signature string `json:"signature"` +} + +type inputDelta struct { + Type string `json:"type"` + PartialJSON string `json:"partial_json"` +} + +type stopDelta struct { + StopReason string `json:"stop_reason"` + StopSequence *string `json:"stop_sequence"` +} + +type messageDelta struct { + Type string `json:"type"` + Delta stopDelta `json:"delta"` + Usage fullUsage `json:"usage"` +} + +func delta(kind pivot.DeltaKind, v string) any { + switch kind { + case pivot.DeltaThinking: + return thinkingDelta{Type: "thinking_delta", Thinking: v} + case pivot.DeltaSignature: + return signatureDelta{Type: "signature_delta", Signature: v} + case pivot.DeltaInputJSON: + return inputDelta{Type: "input_json_delta", PartialJSON: v} + } + return textDelta{Type: "text_delta", Text: v} +} + +func frame(event string, v any) []byte { + b := append([]byte("event: "+event+"\ndata: "), marshal(v)...) + return append(b, "\n\n"...) +} + +// collected is one content block of a not-streamed reply. +type collected struct { + index int + kind pivot.BlockKind + text, signature, input string + id, name string +} + +type replyEncoder struct { + stream bool + done bool // a stream has sent its error or its message_stop + id, model string + usage pivot.Usage + stop string + blocks []*collected + err *pivot.Event +} + +// NewReplyEncoder makes an encoder for one reply to the agent. With +// stream, each event gives its SSE frames at once (rule 23). Without +// stream, the events are collected and End gives one message. +func (Protocol) NewReplyEncoder(stream bool) pivot.ReplyEncoder { + return &replyEncoder{stream: stream} +} + +func (e *replyEncoder) Encode(ev pivot.Event) []byte { + if e.stream { + return e.encodeStream(ev) + } + e.collect(ev) + return nil +} + +func (e *replyEncoder) encodeStream(ev pivot.Event) []byte { + if e.done { + return nil + } + switch ev.Kind { + case pivot.EventStart: + return frame("message_start", messageStart{Type: "message_start", Message: message{ + ID: ev.ID, Type: "message", Role: "assistant", Model: ev.Model, + Content: []any{}, Usage: startUsage{}, + }}) + case pivot.EventBlockStart: + return frame("content_block_start", blockStart{Type: "content_block_start", Index: ev.Index, + ContentBlock: block(ev.Block, "", "", ev.ToolID, ev.ToolName, json.RawMessage("{}"))}) + case pivot.EventDelta: + return frame("content_block_delta", blockDelta{Type: "content_block_delta", Index: ev.Index, Delta: delta(ev.Delta, ev.Value)}) + case pivot.EventBlockStop: + return frame("content_block_stop", blockStop{Type: "content_block_stop", Index: ev.Index}) + case pivot.EventUsage: + e.usage = ev.Usage + case pivot.EventStop: + e.done = true + out := frame("message_delta", messageDelta{Type: "message_delta", + Delta: stopDelta{StopReason: ev.StopReason}, Usage: usageOf(e.usage)}) + return append(out, frame("message_stop", kindBlock{Type: "message_stop"})...) + case pivot.EventError: + e.done = true + return frame("error", errorBody{Type: "error", Error: errorDetail{Type: ev.ErrorType, Message: ev.Message}}) + } + return nil +} + +func (e *replyEncoder) collect(ev pivot.Event) { + if e.err != nil { + return + } + switch ev.Kind { + case pivot.EventStart: + e.id, e.model = ev.ID, ev.Model + case pivot.EventBlockStart: + e.blocks = append(e.blocks, &collected{index: ev.Index, kind: ev.Block, id: ev.ToolID, name: ev.ToolName}) + case pivot.EventDelta: + b := e.block(ev.Index) + if b == nil { + return + } + switch ev.Delta { + case pivot.DeltaText, pivot.DeltaThinking: + b.text += ev.Value + case pivot.DeltaSignature: + b.signature += ev.Value + case pivot.DeltaInputJSON: + b.input += ev.Value + } + case pivot.EventUsage: + e.usage = ev.Usage + case pivot.EventStop: + e.stop = ev.StopReason + case pivot.EventError: + e.err = &ev + } +} + +func (e *replyEncoder) block(index int) *collected { + for i := len(e.blocks) - 1; i >= 0; i-- { + if e.blocks[i].index == index { + return e.blocks[i] + } + } + return nil +} + +// End gives nothing for a stream. For a not-streamed reply it gives the +// one message, or only the error body when an error event came. +func (e *replyEncoder) End() []byte { + if e.stream { + return nil + } + if e.err != nil { + return Protocol{}.EncodeError(e.err.Status, e.err.ErrorType, e.err.Message) + } + content := make([]any, 0, len(e.blocks)) + for _, b := range e.blocks { + input := json.RawMessage(b.input) + if b.kind == pivot.BlockToolUse { + if len(input) == 0 { + input = json.RawMessage("{}") + } + if !json.Valid(input) { + return Protocol{}.EncodeError(502, "api_error", "the inference server sent tool "+b.name+" input that is not valid JSON") + } + } + content = append(content, block(b.kind, b.text, b.signature, b.id, b.name, input)) + } + msg := message{ID: e.id, Type: "message", Role: "assistant", Model: e.model, Content: content, Usage: usageOf(e.usage)} + if e.stop != "" { + msg.StopReason = &e.stop + } + return marshal(msg) +} diff --git a/core/translate/anthropic/reply_test.go b/core/translate/anthropic/reply_test.go new file mode 100644 index 000000000..fab8bed88 --- /dev/null +++ b/core/translate/anthropic/reply_test.go @@ -0,0 +1,160 @@ +package anthropic + +import ( + "testing" + + "github.com/rossoctl/cortex/core/translate/pivot" +) + +const replyToolID = "call_1__thought__a+b/c=" + +// replyEvents is one reply with a signed thinking block, a text block and +// a tool call in two JSON pieces. +var replyEvents = []pivot.Event{ + {Kind: pivot.EventStart, ID: "msg_1", Model: "m"}, + {Kind: pivot.EventBlockStart, Index: 0, Block: pivot.BlockThinking}, + {Kind: pivot.EventDelta, Index: 0, Delta: pivot.DeltaThinking, Value: "391"}, + {Kind: pivot.EventDelta, Index: 0, Delta: pivot.DeltaSignature, Value: "EvkB"}, + {Kind: pivot.EventBlockStop, Index: 0}, + {Kind: pivot.EventBlockStart, Index: 1, Block: pivot.BlockText}, + {Kind: pivot.EventDelta, Index: 1, Delta: pivot.DeltaText, Value: "391 is prime"}, + {Kind: pivot.EventBlockStop, Index: 1}, + {Kind: pivot.EventBlockStart, Index: 2, Block: pivot.BlockToolUse, ToolID: replyToolID, ToolName: "Bash"}, + {Kind: pivot.EventDelta, Index: 2, Delta: pivot.DeltaInputJSON, Value: `{"comma`}, + {Kind: pivot.EventDelta, Index: 2, Delta: pivot.DeltaInputJSON, Value: `nd": "echo done"}`}, + {Kind: pivot.EventBlockStop, Index: 2}, + {Kind: pivot.EventUsage, Usage: pivot.Usage{Input: 5, Output: 55, CacheRead: 3, CacheCreation: 2}}, + {Kind: pivot.EventStop, StopReason: "tool_use"}, +} + +// encodeAll gives the bytes of each Encode call and of End, joined. It +// fails the test when a not-streamed encoder gives bytes before End. +func encodeAll(t *testing.T, stream bool, events []pivot.Event) string { + t.Helper() + enc := Protocol{}.NewReplyEncoder(stream) + var out []byte + for _, ev := range events { + b := enc.Encode(ev) + if !stream && b != nil { + t.Fatalf("not-streamed Encode(%v) = %q, want nil", ev.Kind, b) + } + out = append(out, b...) + } + return string(append(out, enc.End()...)) +} + +func TestReplyEncoder_Stream(t *testing.T) { + want := `event: message_start +data: {"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","model":"m","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"391"}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"EvkB"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: content_block_start +data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"391 is prime"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":1} + +event: content_block_start +data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"call_1__thought__a+b/c=","name":"Bash","input":{}}} + +event: content_block_delta +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\"comma"}} + +event: content_block_delta +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"nd\": \"echo done\"}"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":2} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"input_tokens":5,"cache_creation_input_tokens":2,"cache_read_input_tokens":3,"output_tokens":55}} + +event: message_stop +data: {"type":"message_stop"} + +` + if got := encodeAll(t, true, replyEvents); got != want { + t.Errorf("stream:\ngot %s\nwant %s", got, want) + } +} + +func TestReplyEncoder_NotStream(t *testing.T) { + want := `{"id":"msg_1","type":"message","role":"assistant","model":"m","content":[` + + `{"type":"thinking","thinking":"391","signature":"EvkB"},` + + `{"type":"text","text":"391 is prime"},` + + `{"type":"tool_use","id":"call_1__thought__a+b/c=","name":"Bash","input":{"command":"echo done"}}],` + + `"stop_reason":"tool_use","stop_sequence":null,` + + `"usage":{"input_tokens":5,"cache_creation_input_tokens":2,"cache_read_input_tokens":3,"output_tokens":55}}` + if got := encodeAll(t, false, replyEvents); got != want { + t.Errorf("not stream:\ngot %s\nwant %s", got, want) + } +} + +func TestReplyEncoder_NotStreamToolInput(t *testing.T) { + start := []pivot.Event{ + {Kind: pivot.EventStart, ID: "msg_2", Model: "m"}, + {Kind: pivot.EventBlockStart, Index: 0, Block: pivot.BlockToolUse, ToolID: "t1", ToolName: "Ls"}, + } + empty := append(append([]pivot.Event{}, start...), pivot.Event{Kind: pivot.EventBlockStop, Index: 0}) + want := `{"id":"msg_2","type":"message","role":"assistant","model":"m","content":[` + + `{"type":"tool_use","id":"t1","name":"Ls","input":{}}],"stop_reason":null,"stop_sequence":null,` + + `"usage":{"input_tokens":0,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":0}}` + if got := encodeAll(t, false, empty); got != want { + t.Errorf("empty input:\ngot %s\nwant %s", got, want) + } + + bad := append(append([]pivot.Event{}, start...), + pivot.Event{Kind: pivot.EventDelta, Index: 0, Delta: pivot.DeltaInputJSON, Value: `{"a":`}) + want = `{"type":"error","error":{"type":"api_error","message":"the inference server sent tool Ls input that is not valid JSON"}}` + if got := encodeAll(t, false, bad); got != want { + t.Errorf("bad input:\ngot %s\nwant %s", got, want) + } +} + +func TestReplyEncoder_Error(t *testing.T) { + events := []pivot.Event{ + {Kind: pivot.EventStart, ID: "msg_1", Model: "m"}, + {Kind: pivot.EventBlockStart, Index: 0, Block: pivot.BlockText}, + {Kind: pivot.EventError, Status: 502, ErrorType: "api_error", Message: "boom"}, + {Kind: pivot.EventStop, StopReason: "end_turn"}, + } + want := `event: message_start +data: {"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","model":"m","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} + +event: error +data: {"type":"error","error":{"type":"api_error","message":"boom"}} + +` + if got := encodeAll(t, true, events); got != want { + t.Errorf("stream:\ngot %s\nwant %s", got, want) + } + want = `{"type":"error","error":{"type":"api_error","message":"boom"}}` + if got := encodeAll(t, false, events); got != want { + t.Errorf("not stream:\ngot %s\nwant %s", got, want) + } +} + +func TestEncodeError(t *testing.T) { + got := string(Protocol{}.EncodeError(404, "not_found_error", `model "x" found`)) + want := `{"type":"error","error":{"type":"not_found_error","message":"model \"x\" found"}}` + if got != want { + t.Errorf("EncodeError = %s, want %s", got, want) + } +} diff --git a/core/translate/anthropic/request.go b/core/translate/anthropic/request.go new file mode 100644 index 000000000..46f2f3532 --- /dev/null +++ b/core/translate/anthropic/request.go @@ -0,0 +1,291 @@ +package anthropic + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/rossoctl/cortex/core/translate/pivot" +) + +// DecodeRequest converts an Anthropic Messages request body to the pivot +// form. Each field that the pivot has no place for goes to the Extra of +// its request, message or block, under "anthropic.". A body that +// is not a valid request gives a *pivot.RequestError with status 400. +func (Protocol) DecodeRequest(body []byte) (pivot.Request, error) { + req, err := decodeRequest(body) + if err != nil { + return pivot.Request{}, &pivot.RequestError{ + Status: 400, + Type: "invalid_request_error", + Message: "the request body is not a valid Anthropic Messages request: " + err.Error(), + } + } + return req, nil +} + +func decodeRequest(body []byte) (pivot.Request, error) { + m, err := object(body) + if err != nil { + return pivot.Request{}, err + } + var req pivot.Request + var system, messages, tools json.RawMessage + var choice *struct { + Type string `json:"type"` + Name string `json:"name"` + DisableParallel bool `json:"disable_parallel_tool_use"` + } + var thinking *struct { + Type string `json:"type"` + BudgetTokens int `json:"budget_tokens"` + } + err = errors.Join( + take(m, "model", &req.Model), + take(m, "system", &system), + take(m, "messages", &messages), + take(m, "tools", &tools), + take(m, "tool_choice", &choice), + take(m, "max_tokens", &req.MaxTokens), + take(m, "stop_sequences", &req.Stop), + take(m, "temperature", &req.Temperature), + take(m, "top_p", &req.TopP), + take(m, "stream", &req.Stream), + take(m, "thinking", &thinking), + takeEffort(m, &req.Effort), + ) + if err != nil { + return pivot.Request{}, err + } + if choice != nil { + req.ToolChoice = &pivot.ToolChoice{Type: choice.Type, Name: choice.Name, DisableParallel: choice.DisableParallel} + } + if thinking != nil { + req.Thinking = &pivot.Thinking{Type: thinking.Type, BudgetTokens: thinking.BudgetTokens} + } + if req.System, err = decodeContent(system); err != nil { + return pivot.Request{}, fmt.Errorf("system: %w", err) + } + if req.Messages, err = decodeMessages(messages); err != nil { + return pivot.Request{}, err + } + if req.Tools, err = decodeTools(tools); err != nil { + return pivot.Request{}, err + } + req.Extra = extra(m) + return req, nil +} + +// takeEffort moves output_config.effort to effort. The other members of +// output_config stay in m, so they go to Extra as anthropic.output_config. +func takeEffort(m map[string]json.RawMessage, effort *string) error { + var cfg map[string]json.RawMessage + if err := take(m, "output_config", &cfg); err != nil || cfg == nil { + return err + } + if err := take(cfg, "effort", effort); err != nil { + return fmt.Errorf("output_config.%w", err) + } + if len(cfg) > 0 { + m["output_config"], _ = json.Marshal(cfg) // a map of raw JSON that was just parsed + } + return nil +} + +func decodeMessages(raw json.RawMessage) ([]pivot.Message, error) { + var items []json.RawMessage + if err := json.Unmarshal(orNull(raw), &items); err != nil { + return nil, fmt.Errorf("messages: %w", err) + } + var out []pivot.Message + for i, item := range items { + m, err := object(item) + if err != nil { + return nil, fmt.Errorf("messages[%d]: %w", i, err) + } + var msg pivot.Message + var content json.RawMessage + if err := errors.Join(take(m, "role", &msg.Role), take(m, "content", &content)); err != nil { + return nil, fmt.Errorf("messages[%d]: %w", i, err) + } + if msg.Content, err = decodeContent(content); err != nil { + return nil, fmt.Errorf("messages[%d].content: %w", i, err) + } + msg.Extra = extra(m) + out = append(out, msg) + } + return out, nil +} + +// decodeContent reads content that is a string (one text block) or a list +// of blocks. +func decodeContent(raw json.RawMessage) ([]pivot.Block, error) { + if isNull(raw) { + return nil, nil + } + var s string + if json.Unmarshal(raw, &s) == nil { + return []pivot.Block{{Kind: pivot.BlockText, Text: s}}, nil + } + var items []json.RawMessage + if err := json.Unmarshal(raw, &items); err != nil { + return nil, errors.New("must be a string or a list of blocks") + } + out := make([]pivot.Block, 0, len(items)) + for i, item := range items { + b, err := decodeBlock(item) + if err != nil { + return nil, fmt.Errorf("[%d]: %w", i, err) + } + out = append(out, b) + } + return out, nil +} + +func decodeBlock(raw json.RawMessage) (pivot.Block, error) { + m, err := object(raw) + if err != nil { + return pivot.Block{}, err + } + var b pivot.Block + var kind string + if err := errors.Join(take(m, "type", &kind), takeRaw(m, "cache_control", &b.CacheControl)); err != nil { + return pivot.Block{}, err + } + if kind == "" { + return pivot.Block{}, errors.New("a block has no type") + } + b.Kind = pivot.BlockKind(kind) + switch b.Kind { + case pivot.BlockText: + err = take(m, "text", &b.Text) + case pivot.BlockImage, pivot.BlockDocument: + err = takeSource(m, &b) + case pivot.BlockToolUse: + err = errors.Join(take(m, "id", &b.ID), take(m, "name", &b.Name), takeRaw(m, "input", &b.Input)) + case pivot.BlockToolResult: + var content json.RawMessage + err = errors.Join(take(m, "tool_use_id", &b.ToolUseID), take(m, "content", &content), take(m, "is_error", &b.IsError)) + if err == nil { + b.Content, err = decodeContent(content) + } + case pivot.BlockThinking: + err = errors.Join(take(m, "thinking", &b.Text), take(m, "signature", &b.Signature)) + case pivot.BlockRedactedThinking: + err = take(m, "data", &b.Data) + } + if err != nil { + return pivot.Block{}, fmt.Errorf("%s block: %w", kind, err) + } + b.Extra = extra(m) + return b, nil +} + +// takeSource reads the source of an image or a document. The members of +// source that the pivot has no place for stay in m as source. +func takeSource(m map[string]json.RawMessage, b *pivot.Block) error { + var src map[string]json.RawMessage + if err := take(m, "source", &src); err != nil || src == nil { + return err + } + if err := errors.Join( + take(src, "type", &b.SourceType), + take(src, "media_type", &b.MediaType), + take(src, "data", &b.Data), + take(src, "url", &b.URL), + ); err != nil { + return fmt.Errorf("source.%w", err) + } + if len(src) > 0 { + m["source"], _ = json.Marshal(src) // a map of raw JSON that was just parsed + } + return nil +} + +func decodeTools(raw json.RawMessage) ([]pivot.Tool, error) { + var items []json.RawMessage + if err := json.Unmarshal(orNull(raw), &items); err != nil { + return nil, fmt.Errorf("tools: %w", err) + } + var out []pivot.Tool + for i, item := range items { + m, err := object(item) + if err != nil { + return nil, fmt.Errorf("tools[%d]: %w", i, err) + } + var t pivot.Tool + if err := errors.Join( + take(m, "name", &t.Name), + take(m, "description", &t.Description), + takeRaw(m, "input_schema", &t.InputSchema), + take(m, "type", &t.Type), + takeRaw(m, "cache_control", &t.CacheControl), + ); err != nil { + return nil, fmt.Errorf("tools[%d]: %w", i, err) + } + if t.Type == "custom" { // Anthropic's name for a function tool + t.Type = "" + } + t.Extra = extra(m) + out = append(out, t) + } + return out, nil +} + +// object parses a JSON object into its members. +func object(raw []byte) (map[string]json.RawMessage, error) { + var m map[string]json.RawMessage + if err := json.Unmarshal(raw, &m); err != nil { + return nil, err + } + if m == nil { + return nil, errors.New("must be a JSON object") + } + return m, nil +} + +// take parses the member key of m into dst and deletes it from m. A +// missing member leaves dst unchanged. +func take(m map[string]json.RawMessage, key string, dst any) error { + raw, ok := m[key] + if !ok { + return nil + } + delete(m, key) + if err := json.Unmarshal(raw, dst); err != nil { + return fmt.Errorf("%s: %w", key, err) + } + return nil +} + +// takeRaw moves the member key of m to dst as raw JSON. A null member +// gives nil. +func takeRaw(m map[string]json.RawMessage, key string, dst *json.RawMessage) error { + raw, ok := m[key] + delete(m, key) + if ok && !isNull(raw) { + *dst = raw + } + return nil +} + +// extra gives the members left in m as Extra keys, or nil when none. +func extra(m map[string]json.RawMessage) pivot.Extra { + if len(m) == 0 { + return nil + } + e := make(pivot.Extra, len(m)) + for k, v := range m { + e[extraPrefix+k] = v + } + return e +} + +func isNull(raw json.RawMessage) bool { return len(raw) == 0 || string(raw) == "null" } + +func orNull(raw json.RawMessage) json.RawMessage { + if len(raw) == 0 { + return json.RawMessage("null") + } + return raw +} diff --git a/core/translate/anthropic/request_test.go b/core/translate/anthropic/request_test.go new file mode 100644 index 000000000..da839754e --- /dev/null +++ b/core/translate/anthropic/request_test.go @@ -0,0 +1,128 @@ +package anthropic + +import ( + "encoding/json" + "errors" + "reflect" + "testing" + + "github.com/rossoctl/cortex/core/translate/pivot" +) + +func raw(s string) json.RawMessage { return json.RawMessage(s) } + +func ptr(f float64) *float64 { return &f } + +// thoughtID is a tool id with a thought signature in it (rule 20a). It has +// "+", "/" and "=", and must pass byte-exact. +const thoughtID = "call_3833__thought__AY89a1+8zd/x=" + +func TestDecodeRequest_Fields(t *testing.T) { + body := `{"model":"claude-opus-5-5","max_tokens":32000,"stream":true,"temperature":0.5,"top_p":0.9,` + + `"top_k":40,"metadata":{"user_id":"u1"},"stop_sequences":["END"],` + + `"system":[{"type":"text","text":"You are Claude Code.","cache_control":{"type":"ephemeral"}}],` + + `"output_config":{"effort":"xhigh","format":{"type":"json"}},` + + `"thinking":{"type":"enabled","budget_tokens":1024},` + + `"tool_choice":{"type":"auto","disable_parallel_tool_use":true},` + + `"tools":[{"name":"Bash","description":"Run a command","input_schema":{"type":"object"},"cache_control":{"type":"ephemeral"}},` + + `{"type":"custom","name":"Read","input_schema":{"type":"object"}},` + + `{"type":"web_search_20250305","name":"web_search","max_uses":5}],` + + `"messages":[` + + `{"role":"user","content":"Is 391 prime?"},` + + `{"role":"system","content":"hook output"},` + + `{"role":"assistant","content":[{"type":"thinking","thinking":"391 = 17*23","signature":"EvkB"},` + + `{"type":"redacted_thinking","data":"cmVk"},{"type":"text","text":"Let me check."},` + + `{"type":"tool_use","id":"` + thoughtID + `","name":"Bash","input":{"command":"factor 391"}}]},` + + `{"role":"user","content":[{"type":"tool_result","tool_use_id":"` + thoughtID + `","content":"391: 17 23","is_error":true},` + + `{"type":"image","source":{"type":"base64","media_type":"image/png","data":"iVBO"}},` + + `{"type":"image","source":{"type":"url","url":"https://example.com/a.png","foo":1}},` + + `{"type":"document","source":{"type":"base64","media_type":"application/pdf","data":"JVBE"},"title":"a.pdf"},` + + `{"type":"text","text":"Go on.","cache_control":{"type":"ephemeral","ttl":"1h"}}],"turn":7}]}` + + want := pivot.Request{ + Model: "claude-opus-5-5", + System: []pivot.Block{ + {Kind: pivot.BlockText, Text: "You are Claude Code.", CacheControl: raw(`{"type":"ephemeral"}`)}, + }, + Messages: []pivot.Message{ + {Role: "user", Content: []pivot.Block{{Kind: pivot.BlockText, Text: "Is 391 prime?"}}}, + {Role: "system", Content: []pivot.Block{{Kind: pivot.BlockText, Text: "hook output"}}}, + {Role: "assistant", Content: []pivot.Block{ + {Kind: pivot.BlockThinking, Text: "391 = 17*23", Signature: "EvkB"}, + {Kind: pivot.BlockRedactedThinking, Data: "cmVk"}, + {Kind: pivot.BlockText, Text: "Let me check."}, + {Kind: pivot.BlockToolUse, ID: thoughtID, Name: "Bash", Input: raw(`{"command":"factor 391"}`)}, + }}, + {Role: "user", Content: []pivot.Block{ + {Kind: pivot.BlockToolResult, ToolUseID: thoughtID, IsError: true, + Content: []pivot.Block{{Kind: pivot.BlockText, Text: "391: 17 23"}}}, + {Kind: pivot.BlockImage, SourceType: "base64", MediaType: "image/png", Data: "iVBO"}, + {Kind: pivot.BlockImage, SourceType: "url", URL: "https://example.com/a.png", + Extra: pivot.Extra{"anthropic.source": raw(`{"foo":1}`)}}, + {Kind: pivot.BlockDocument, SourceType: "base64", MediaType: "application/pdf", Data: "JVBE", + Extra: pivot.Extra{"anthropic.title": raw(`"a.pdf"`)}}, + {Kind: pivot.BlockText, Text: "Go on.", CacheControl: raw(`{"type":"ephemeral","ttl":"1h"}`)}, + }, Extra: pivot.Extra{"anthropic.turn": raw(`7`)}}, + }, + Tools: []pivot.Tool{ + {Name: "Bash", Description: "Run a command", InputSchema: raw(`{"type":"object"}`), CacheControl: raw(`{"type":"ephemeral"}`)}, + {Name: "Read", InputSchema: raw(`{"type":"object"}`)}, + {Name: "web_search", Type: "web_search_20250305", Extra: pivot.Extra{"anthropic.max_uses": raw(`5`)}}, + }, + ToolChoice: &pivot.ToolChoice{Type: "auto", DisableParallel: true}, + MaxTokens: 32000, + Stop: []string{"END"}, + Temperature: ptr(0.5), + TopP: ptr(0.9), + Stream: true, + Effort: "xhigh", + Thinking: &pivot.Thinking{Type: "enabled", BudgetTokens: 1024}, + Extra: pivot.Extra{ + "anthropic.top_k": raw(`40`), + "anthropic.metadata": raw(`{"user_id":"u1"}`), + "anthropic.output_config": raw(`{"format":{"type":"json"}}`), + }, + } + + got, err := Protocol{}.DecodeRequest([]byte(body)) + if err != nil { + t.Fatalf("DecodeRequest: %v", err) + } + if !reflect.DeepEqual(got, want) { + g, _ := json.MarshalIndent(got, "", " ") + w, _ := json.MarshalIndent(want, "", " ") + t.Errorf("DecodeRequest:\ngot %s\nwant %s", g, w) + } +} + +func TestDecodeRequest_SystemString(t *testing.T) { + got, err := Protocol{}.DecodeRequest([]byte(`{"model":"m","system":"Be brief.","output_config":{"effort":"low"}}`)) + if err != nil { + t.Fatalf("DecodeRequest: %v", err) + } + want := pivot.Request{Model: "m", System: []pivot.Block{{Kind: pivot.BlockText, Text: "Be brief."}}, Effort: "low"} + if !reflect.DeepEqual(got, want) { + t.Errorf("DecodeRequest = %+v, want %+v", got, want) + } +} + +func TestDecodeRequest_BadJSON(t *testing.T) { + for _, body := range []string{ + `{"model":`, + `[]`, + `null`, + `{"model":5}`, + `{"messages":[{"role":"user","content":5}]}`, + `{"messages":[{"role":"user","content":[{"text":"no type"}]}]}`, + } { + _, err := Protocol{}.DecodeRequest([]byte(body)) + var re *pivot.RequestError + if !errors.As(err, &re) { + t.Errorf("%s: err = %v, want *pivot.RequestError", body, err) + continue + } + if re.Status != 400 || re.Type != "invalid_request_error" { + t.Errorf("%s: got %d %s, want 400 invalid_request_error", body, re.Status, re.Type) + } + } +} diff --git a/core/translate/chat/error.go b/core/translate/chat/error.go new file mode 100644 index 000000000..51bca8985 --- /dev/null +++ b/core/translate/chat/error.go @@ -0,0 +1,15 @@ +package chat + +// EncodeError gives an error body in the chat-completions shape. +func (Protocol) EncodeError(status int, errType, message string) []byte { + type body struct { + Message string `json:"message"` + Type string `json:"type"` + Code *int `json:"code"` + } + // The value holds no json.RawMessage, so marshal cannot fail. + b, _ := marshal(struct { + Error body `json:"error"` + }{body{Message: message, Type: errType}}) + return b +} diff --git a/core/translate/chat/protocol.go b/core/translate/chat/protocol.go new file mode 100644 index 000000000..96c4c5edf --- /dev/null +++ b/core/translate/chat/protocol.go @@ -0,0 +1,35 @@ +// Package chat is the chat-completions protocol of the translation hub. +// It encodes pivot requests as OpenAI /v1/chat/completions bodies, and +// decodes chat-completions replies to pivot events (rules 11-24 of #1911). +package chat + +import ( + "bytes" + "encoding/json" + + "github.com/rossoctl/cortex/core/translate/pivot" +) + +// Protocol is the chat-completions protocol. +type Protocol struct{} + +var ( + _ pivot.Protocol = Protocol{} + _ pivot.RequestEncoder = Protocol{} + _ pivot.ReplyDecoding = Protocol{} +) + +// Name gives the protocol name. +func (Protocol) Name() string { return "chat-completions" } + +// marshal writes v with no HTML escapes, so that "<" in a prompt goes to +// the server as "<". The output is the same for the same value (rule 9a). +func marshal(v any) ([]byte, error) { + var b bytes.Buffer + enc := json.NewEncoder(&b) + enc.SetEscapeHTML(false) + if err := enc.Encode(v); err != nil { + return nil, err + } + return bytes.TrimSuffix(b.Bytes(), []byte("\n")), nil +} diff --git a/core/translate/chat/reply.go b/core/translate/chat/reply.go new file mode 100644 index 000000000..e05c16c5f --- /dev/null +++ b/core/translate/chat/reply.go @@ -0,0 +1,472 @@ +package chat + +import ( + "bytes" + "encoding/json" + "net/http" + "strconv" + "strings" + "unicode/utf8" + + "github.com/rossoctl/cortex/core/translate/pivot" +) + +// maxErrorBody is the most of a non-2xx body that the decoder keeps. +const maxErrorBody = 1 << 20 + +// maxErrorMessage is the most of a non-JSON error body that goes into the +// error message (Review Focus 4). +const maxErrorMessage = 512 + +const msgEndedEarly = "the inference server's stream ended before the reply was complete" + +// The Anthropic error types. An upstream type in this set is kept. +var anthropicErrorTypes = map[string]bool{ + "invalid_request_error": true, "authentication_error": true, "permission_error": true, + "not_found_error": true, "request_too_large": true, "rate_limit_error": true, + "api_error": true, "overloaded_error": true, +} + +type openKind int + +const ( + kindNone openKind = iota + kindThinking + kindText + kindTool +) + +// decoder converts one chat-completions reply to pivot events. +type decoder struct { + status int + stream bool + buf []byte // a stream: the partial line; else the body + + out []pivot.Event + started bool + finished bool // finish_reason came + done bool // stop or error emitted; ignore the rest + stopReason string + usage pivot.Usage + + open openKind + openIndex int // the pivot block index of the open block + openTool int // the chat tool-call index of the open tool_use block + openID string // the tool-call id of the open tool_use block + next int // the next pivot block index + closed map[int]bool // chat tool-call indexes whose block ended + closedIDs map[string]bool // tool-call ids whose block ended + sawTool bool // a tool_use block was opened +} + +// NewReplyDecoder makes a decoder for one upstream reply (decision 3). +func (Protocol) NewReplyDecoder(status int, stream bool) pivot.ReplyDecoder { + return &decoder{status: status, stream: stream, closed: map[int]bool{}, closedIDs: map[string]bool{}} +} + +func (d *decoder) ok() bool { return d.status >= 200 && d.status < 300 } + +// Feed reads the next bytes of the body. +func (d *decoder) Feed(p []byte) []pivot.Event { + if d.done { + return nil + } + if !d.ok() { + if room := maxErrorBody - len(d.buf); room > 0 { + d.buf = append(d.buf, p[:min(len(p), room)]...) + } + return nil + } + d.buf = append(d.buf, p...) + if !d.stream { + return nil + } + d.out = nil + rest := d.buf + for !d.done { + i := bytes.IndexByte(rest, '\n') + if i < 0 { + break + } + d.line(rest[:i]) + rest = rest[i+1:] + } + // Keep only the partial line, in a new array, so that the stream + // read so far is not held (rule 10). + d.buf = append([]byte(nil), rest...) + return d.out +} + +// End reads the end of the body. +func (d *decoder) End() []pivot.Event { + if d.done { + return nil + } + d.out = nil + switch { + case !d.ok(): + d.fail(d.status, pivot.ErrorType(d.status), errorMessage(d.status, d.buf)) + case !d.stream: + d.whole(d.buf) + default: + if len(d.buf) > 0 { + d.line(d.buf) + } + d.finish() + } + d.buf = nil + return d.out +} + +func (d *decoder) emit(ev pivot.Event) { d.out = append(d.out, ev) } + +func (d *decoder) fail(status int, errType, msg string) { + d.emit(pivot.Event{Kind: pivot.EventError, Status: status, ErrorType: errType, Message: msg}) + d.done = true +} + +// finish ends a stream at [DONE] or at the end of the body (decision 5). +func (d *decoder) finish() { + if d.done { + return + } + if !d.finished { + d.fail(http.StatusBadGateway, "api_error", msgEndedEarly) + return + } + d.closeOpen() + // vLLM and OpenAI send finish_reason stop for a forced named tool. + if d.sawTool && d.stopReason == "end_turn" { + d.stopReason = "tool_use" + } + d.emit(pivot.Event{Kind: pivot.EventUsage, Usage: d.usage}) + d.emit(pivot.Event{Kind: pivot.EventStop, StopReason: d.stopReason}) + d.done = true +} + +func (d *decoder) start(id, model string) { + if !d.started { + d.started = true + d.emit(pivot.Event{Kind: pivot.EventStart, ID: id, Model: model}) + } +} + +func (d *decoder) openBlock(k openKind, block pivot.BlockKind, toolIndex int, id, name string) { + d.closeOpen() + d.emit(pivot.Event{Kind: pivot.EventBlockStart, Index: d.next, Block: block, ToolID: id, ToolName: name}) + d.open, d.openIndex, d.openTool, d.openID = k, d.next, toolIndex, id + d.sawTool = d.sawTool || k == kindTool + d.next++ +} + +func (d *decoder) closeOpen() { + if d.open == kindNone { + return + } + d.emit(pivot.Event{Kind: pivot.EventBlockStop, Index: d.openIndex}) + if d.open == kindTool { + d.closed[d.openTool] = true + d.closedIDs[d.openID] = true + } + d.open = kindNone +} + +func (d *decoder) delta(k pivot.DeltaKind, v string) { + d.emit(pivot.Event{Kind: pivot.EventDelta, Index: d.openIndex, Delta: k, Value: v}) +} + +// The chat-completions reply shapes. Only the fields that rules 20-24 read. +type ( + chatError struct { + Message string `json:"message"` + Type string `json:"type"` + Code json.RawMessage `json:"code"` + } + chatUsage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + CacheRead *int `json:"cache_read_input_tokens"` + CacheCreation int `json:"cache_creation_input_tokens"` + PromptTokensDetails *struct { + CachedTokens int `json:"cached_tokens"` + } `json:"prompt_tokens_details"` + } + replyThinking struct { + Type string `json:"type"` + Thinking string `json:"thinking"` + Signature string `json:"signature"` + } + replyToolCall struct { + Index int `json:"index"` + ID string `json:"id"` + Function struct { + Name string `json:"name"` + Arguments string `json:"arguments"` + } `json:"function"` + } + // replyMessage is a stream delta or a whole message. It does not read + // provider_specific_fields (rule 23). + replyMessage struct { + Content string `json:"content"` + Reasoning string `json:"reasoning"` + ReasoningContent string `json:"reasoning_content"` + ThinkingBlocks []replyThinking `json:"thinking_blocks"` + ToolCalls []replyToolCall `json:"tool_calls"` + } + replyChoice struct { + Delta replyMessage `json:"delta"` + Message replyMessage `json:"message"` + FinishReason string `json:"finish_reason"` + } + reply struct { + ID string `json:"id"` + Model string `json:"model"` + Choices []replyChoice `json:"choices"` + Usage *chatUsage `json:"usage"` + Error *chatError `json:"error"` + } +) + +// line reads one SSE line of a stream (rule 23). +func (d *decoder) line(l []byte) { + l = bytes.TrimSuffix(l, []byte("\r")) + payload, ok := bytes.CutPrefix(l, []byte("data:")) + if !ok { + return // a blank line, a comment (": ping") or another field + } + payload = bytes.TrimSpace(payload) + if string(payload) == "[DONE]" { + d.finish() + return + } + var c reply + if err := json.Unmarshal(payload, &c); err != nil { + d.fail(http.StatusBadGateway, "api_error", "the inference server sent a stream chunk that is not valid JSON") + return + } + if c.Error != nil { + d.chunkError(c.Error) + return + } + d.start(c.ID, c.Model) + if c.Usage != nil { + d.usage = convertUsage(c.Usage) + } + if len(c.Choices) == 0 { + return + } + ch := c.Choices[0] + d.streamDelta(ch.Delta) + if ch.FinishReason != "" && !d.done { + d.closeOpen() + d.finished = true + d.stopReason = stopReason(ch.FinishReason) + } +} + +func (d *decoder) streamDelta(m replyMessage) { + // Thinking text comes only from reasoning or reasoning_content; Bob + // repeats it in thinking_blocks (rule 23). + think := m.Reasoning + if think == "" { + think = m.ReasoningContent + } + if think != "" { + if d.open != kindThinking { + d.openBlock(kindThinking, pivot.BlockThinking, 0, "", "") + } + d.delta(pivot.DeltaThinking, think) + } + for _, tb := range m.ThinkingBlocks { + if tb.Signature == "" { + continue + } + // A signature with no open thinking block opens an empty one. + if d.open != kindThinking { + d.openBlock(kindThinking, pivot.BlockThinking, 0, "", "") + } + d.delta(pivot.DeltaSignature, tb.Signature) + } + if m.Content != "" { + if d.open != kindText { + d.openBlock(kindText, pivot.BlockText, 0, "", "") + } + d.delta(pivot.DeltaText, m.Content) + } + for _, tc := range m.ToolCalls { + // Rule 20a: the id identifies the call, so a new id on the open + // index is a new call, not more of the open one. + isOpen := d.open == kindTool && d.openTool == tc.Index && (tc.ID == "" || tc.ID == d.openID) + if tc.ID != "" && !isOpen { + // Key by id, not index: servers that put every call at index 0 + // close that index once per call. + if d.closedIDs[tc.ID] { + d.fail(http.StatusBadGateway, "api_error", "the inference server sent tool call "+tc.ID+" again after that tool call ended") + return + } + // Rule 20a: the id goes to the agent byte-exact. + d.openBlock(kindTool, pivot.BlockToolUse, tc.Index, tc.ID, tc.Function.Name) + isOpen = true + } + if tc.Function.Arguments == "" { + continue + } + switch { + case isOpen: + d.delta(pivot.DeltaInputJSON, tc.Function.Arguments) + case d.closed[tc.Index]: + // Decision 6. + d.fail(http.StatusBadGateway, "api_error", "the inference server sent arguments for tool call "+strconv.Itoa(tc.Index)+" after that tool call ended") + return + default: + d.fail(http.StatusBadGateway, "api_error", "the inference server sent arguments for tool call "+strconv.Itoa(tc.Index)+" before its id") + return + } + } +} + +// chunkError converts an {"error":…} chunk or body at status 2xx. +func (d *decoder) chunkError(e *chatError) { + status := http.StatusInternalServerError + if n, err := strconv.Atoi(string(e.Code)); err == nil && n >= 400 && n < 600 { + status = n + } + t := "api_error" + if anthropicErrorTypes[e.Type] { + t = e.Type + } + d.fail(status, t, e.Message) +} + +// whole converts a not-streamed 2xx body by rule 20. +func (d *decoder) whole(body []byte) { + var r reply + if err := json.Unmarshal(body, &r); err != nil { + d.fail(http.StatusBadGateway, "api_error", "the inference server sent a reply that is not valid JSON") + return + } + if r.Error != nil { + d.chunkError(r.Error) + return + } + if len(r.Choices) == 0 { + d.fail(http.StatusBadGateway, "api_error", "the inference server sent a reply with no choices") + return + } + m := r.Choices[0].Message + // Decision 7: check every tool input before any event goes out. + ids := map[string]bool{} + for _, tc := range m.ToolCalls { + if a := tc.Function.Arguments; a != "" && !json.Valid([]byte(a)) { + d.fail(http.StatusBadGateway, "api_error", "the inference server sent tool "+tc.Function.Name+" arguments that are not valid JSON") + return + } + // Rule 20a: the agent's tool_result finds its call by id. + if tc.ID == "" { + d.fail(http.StatusBadGateway, "api_error", "the inference server sent tool "+tc.Function.Name+" with no id") + return + } + if ids[tc.ID] { + d.fail(http.StatusBadGateway, "api_error", "the inference server sent tool call "+tc.ID+" twice") + return + } + ids[tc.ID] = true + } + d.start(r.ID, r.Model) + if len(m.ThinkingBlocks) > 0 { + for _, tb := range m.ThinkingBlocks { + // shortcut: a redacted_thinking entry is not sent, because the + // pivot events have no field for its data; add one when a + // server sends it. + if tb.Type != "thinking" { + continue + } + d.openBlock(kindThinking, pivot.BlockThinking, 0, "", "") + if tb.Thinking != "" { + d.delta(pivot.DeltaThinking, tb.Thinking) + } + if tb.Signature != "" { + d.delta(pivot.DeltaSignature, tb.Signature) + } + } + } else if think := cmpOr(m.Reasoning, m.ReasoningContent); think != "" { + d.openBlock(kindThinking, pivot.BlockThinking, 0, "", "") + d.delta(pivot.DeltaThinking, think) + } + if m.Content != "" { + d.openBlock(kindText, pivot.BlockText, 0, "", "") + d.delta(pivot.DeltaText, m.Content) + } + for _, tc := range m.ToolCalls { + d.openBlock(kindTool, pivot.BlockToolUse, tc.Index, tc.ID, tc.Function.Name) + if tc.Function.Arguments != "" { + d.delta(pivot.DeltaInputJSON, tc.Function.Arguments) + } + } + if r.Usage != nil { + d.usage = convertUsage(r.Usage) + } + d.finished = true + d.stopReason = stopReason(r.Choices[0].FinishReason) + d.finish() +} + +func cmpOr(a, b string) string { + if a != "" { + return a + } + return b +} + +// stopReason maps finish_reason by rule 21. +func stopReason(r string) string { + switch r { + case "length": + return "max_tokens" + case "tool_calls": + return "tool_use" + case "content_filter": + return "refusal" + } + return "end_turn" +} + +// convertUsage converts usage to the Anthropic meaning by rule 22. +func convertUsage(u *chatUsage) pivot.Usage { + read := 0 + if u.CacheRead != nil { + read = *u.CacheRead + } else if u.PromptTokensDetails != nil { + read = u.PromptTokensDetails.CachedTokens + } + return pivot.Usage{ + Input: max(0, u.PromptTokens-read-u.CacheCreation), + Output: u.CompletionTokens, + CacheRead: read, + CacheCreation: u.CacheCreation, + } +} + +// errorMessage gives the message of a non-2xx reply (decision 3). +func errorMessage(status int, body []byte) string { + var e struct { + Error struct { + Message string `json:"message"` + } `json:"error"` + } + if json.Unmarshal(body, &e) == nil && e.Error.Message != "" { + return e.Error.Message + } + t := strings.TrimSpace(string(body)) + if len(t) > maxErrorMessage { + n := maxErrorMessage + for n > 0 && !utf8.RuneStart(t[n]) { + n-- + } + t = t[:n] + } + if t == "" { + return http.StatusText(status) + } + return t +} diff --git a/core/translate/chat/reply_test.go b/core/translate/chat/reply_test.go new file mode 100644 index 000000000..159d5165f --- /dev/null +++ b/core/translate/chat/reply_test.go @@ -0,0 +1,364 @@ +package chat + +import ( + "reflect" + "strings" + "testing" + + "github.com/rossoctl/cortex/core/translate/pivot" +) + +// sse writes each chunk as one data line, as Bob does, with a padded +// ping comment line first. +func sse(chunks ...string) string { + var b strings.Builder + b.WriteString(": ping\n\n") + for _, c := range chunks { + b.WriteString("data: " + c + "\n\n") + } + return b.String() +} + +func decodeAll(status int, stream bool, body string) []pivot.Event { + d := Protocol{}.NewReplyDecoder(status, stream) + evs := d.Feed([]byte(body)) + return append(evs, d.End()...) +} + +// decodeBytewise feeds the body one byte at a time. +func decodeBytewise(status int, stream bool, body string) []pivot.Event { + d := Protocol{}.NewReplyDecoder(status, stream) + var evs []pivot.Event + for i := 0; i < len(body); i++ { + evs = append(evs, d.Feed([]byte{body[i]})...) + } + return append(evs, d.End()...) +} + +func checkEvents(t *testing.T, status int, stream bool, body string, want []pivot.Event) { + t.Helper() + if got := decodeAll(status, stream, body); !reflect.DeepEqual(got, want) { + t.Errorf("events:\n got %+v\nwant %+v", got, want) + } + if got := decodeBytewise(status, stream, body); !reflect.DeepEqual(got, want) { + t.Errorf("events, fed one byte at a time:\n got %+v\nwant %+v", got, want) + } +} + +func chunk(delta string) string { + return `{"id":"chatcmpl-1","model":"claude","choices":[{"index":0,"delta":` + delta + `,"finish_reason":null}]}` +} + +func finish(reason string) string { + return `{"id":"chatcmpl-1","model":"claude","choices":[{"index":0,"delta":{},"finish_reason":"` + reason + `"}]}` +} + +func usageChunk(u string) string { + return `{"id":"chatcmpl-1","model":"claude","choices":[],"usage":` + u + `}` +} + +func start() pivot.Event { + return pivot.Event{Kind: pivot.EventStart, ID: "chatcmpl-1", Model: "claude"} +} +func open(i int, k pivot.BlockKind) pivot.Event { + return pivot.Event{Kind: pivot.EventBlockStart, Index: i, Block: k} +} +func openTool(i int, id, name string) pivot.Event { + return pivot.Event{Kind: pivot.EventBlockStart, Index: i, Block: pivot.BlockToolUse, ToolID: id, ToolName: name} +} +func delta(i int, k pivot.DeltaKind, v string) pivot.Event { + return pivot.Event{Kind: pivot.EventDelta, Index: i, Delta: k, Value: v} +} +func closeBlock(i int) pivot.Event { return pivot.Event{Kind: pivot.EventBlockStop, Index: i} } +func usage(u pivot.Usage) pivot.Event { return pivot.Event{Kind: pivot.EventUsage, Usage: u} } +func stop(r string) pivot.Event { return pivot.Event{Kind: pivot.EventStop, StopReason: r} } +func apiError(status int, typ, msg string) pivot.Event { + return pivot.Event{Kind: pivot.EventError, Status: status, ErrorType: typ, Message: msg} +} + +func TestReplyDecoder_BobShape1(t *testing.T) { + body := sse( + chunk(`{"reasoning":"391","thinking_blocks":[{"type":"thinking","thinking":"391"}],"role":"assistant"}`), + chunk(`{"reasoning":" Not","thinking_blocks":[{"type":"thinking","thinking":" Not"}]}`), + chunk(`{"thinking_blocks":[{"type":"thinking","thinking":"","signature":"EvkB"}]}`), + chunk(`{"content":"391 is not prime"}`), + chunk(`{"tool_calls":[{"id":"tooluse_9nf","function":{"arguments":"","name":"Bash"},"type":"function","index":0}]}`), + chunk(`{"tool_calls":[{"function":{"arguments":"{\"comma"},"type":"function","index":0}]}`), + chunk(`{"tool_calls":[{"function":{"arguments":"nd\": \"echo done\"}"},"type":"function","index":0}]}`), + finish("tool_calls"), + usageChunk(`{"prompt_tokens":578,"completion_tokens":55,"total_tokens":633,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"prompt_tokens_details":{"cached_tokens":0},"completion_tokens_details":{"reasoning_tokens":0}}`), + "[DONE]", + ) + checkEvents(t, 200, true, body, []pivot.Event{ + start(), + open(0, pivot.BlockThinking), + delta(0, pivot.DeltaThinking, "391"), + delta(0, pivot.DeltaThinking, " Not"), + delta(0, pivot.DeltaSignature, "EvkB"), + closeBlock(0), + open(1, pivot.BlockText), + delta(1, pivot.DeltaText, "391 is not prime"), + closeBlock(1), + openTool(2, "tooluse_9nf", "Bash"), + delta(2, pivot.DeltaInputJSON, `{"comma`), + delta(2, pivot.DeltaInputJSON, `nd": "echo done"}`), + closeBlock(2), + usage(pivot.Usage{Input: 578, Output: 55}), + stop("tool_use"), + }) +} + +func TestReplyDecoder_BobShape2(t *testing.T) { + body := sse( + chunk(`{"reasoning":"","thinking_blocks":[{"type":"thinking","thinking":""}],"content":"","role":"assistant"}`), + chunk(`{"reasoning":"","thinking_blocks":[{"type":"thinking","thinking":"","signature":"CAQS"}],"content":""}`), + chunk(`{"content":"hello"}`), + chunk(`{"content":"","tool_calls":[{"id":"tooluse_Z","function":{"arguments":"","name":"execute_command"},"type":"function","index":0}]}`), + chunk(`{"content":"","tool_calls":[{"function":{"arguments":"{}"},"type":"function","index":0}]}`), + finish("tool_calls"), + usageChunk(`{"prompt_tokens":22333,"completion_tokens":216,"cache_read_input_tokens":21649,"cache_creation_input_tokens":682,"completion_tokens_details":{"reasoning_tokens":0}}`), + "[DONE]", + ) + checkEvents(t, 200, true, body, []pivot.Event{ + start(), + open(0, pivot.BlockThinking), + delta(0, pivot.DeltaSignature, "CAQS"), + closeBlock(0), + open(1, pivot.BlockText), + delta(1, pivot.DeltaText, "hello"), + closeBlock(1), + openTool(2, "tooluse_Z", "execute_command"), + delta(2, pivot.DeltaInputJSON, "{}"), + closeBlock(2), + usage(pivot.Usage{Input: 2, Output: 216, CacheRead: 21649, CacheCreation: 682}), + stop("tool_use"), + }) +} + +func TestReplyDecoder_BobShape3(t *testing.T) { + const id = "call_3833__thought__AY89a1+8zd/x==" + body := sse( + chunk(`{"provider_specific_fields":{"thought_signatures":["AY89"]},"role":"assistant","tool_calls":[{"id":"`+id+`","function":{"arguments":"{\"task\":\"x\"}","name":"spawn_subagent"},"type":"function","index":0}]}`), + finish("tool_calls"), + usageChunk(`{"prompt_tokens":13314,"completion_tokens":296,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"completion_tokens_details":{"reasoning_tokens":249}}`), + "[DONE]", + ) + checkEvents(t, 200, true, body, []pivot.Event{ + start(), + openTool(0, id, "spawn_subagent"), + delta(0, pivot.DeltaInputJSON, `{"task":"x"}`), + closeBlock(0), + usage(pivot.Usage{Input: 13314, Output: 296}), + stop("tool_use"), + }) +} + +func TestReplyDecoder_FinishReasons(t *testing.T) { + for reason, want := range map[string]string{ + "stop": "end_turn", "length": "max_tokens", "tool_calls": "tool_use", + "content_filter": "refusal", "something_new": "end_turn", + } { + evs := decodeAll(200, true, sse(chunk(`{"content":"a"}`), finish(reason), "[DONE]")) + if got := evs[len(evs)-1]; got != stop(want) { + t.Errorf("finish_reason %q: last event %+v, want stop %q", reason, got, want) + } + } +} + +// Usage falls back to prompt_tokens_details.cached_tokens (rule 22). +func TestReplyDecoder_CachedTokens(t *testing.T) { + evs := decodeAll(200, true, sse(chunk(`{"content":"a"}`), finish("stop"), + usageChunk(`{"prompt_tokens":100,"completion_tokens":5,"prompt_tokens_details":{"cached_tokens":60}}`), "[DONE]")) + if got := evs[len(evs)-2]; got != usage(pivot.Usage{Input: 40, Output: 5, CacheRead: 60}) { + t.Errorf("usage event = %+v", got) + } +} + +// A body that ends after finish_reason with no [DONE] still ends the reply. +func TestReplyDecoder_NoDone(t *testing.T) { + checkEvents(t, 200, true, sse(chunk(`{"content":"a"}`), finish("stop")), []pivot.Event{ + start(), open(0, pivot.BlockText), delta(0, pivot.DeltaText, "a"), closeBlock(0), + usage(pivot.Usage{}), stop("end_turn"), + }) +} + +const endedEarly = "the inference server's stream ended before the reply was complete" + +func TestReplyDecoder_EndsEarly(t *testing.T) { + want := []pivot.Event{ + start(), open(0, pivot.BlockText), delta(0, pivot.DeltaText, "half"), + apiError(502, "api_error", endedEarly), + } + checkEvents(t, 200, true, sse(chunk(`{"content":"half"}`)), want) + checkEvents(t, 200, true, sse(chunk(`{"content":"half"}`), "[DONE]"), want) +} + +func TestReplyDecoder_ClosedIndexPiece(t *testing.T) { + body := sse( + chunk(`{"tool_calls":[{"id":"a","function":{"arguments":"{}","name":"A"},"type":"function","index":0}]}`), + chunk(`{"tool_calls":[{"id":"b","function":{"arguments":"{","name":"B"},"type":"function","index":1}]}`), + chunk(`{"tool_calls":[{"function":{"arguments":"\"late\""},"type":"function","index":0}]}`), + chunk(`{"tool_calls":[{"function":{"arguments":"}"},"type":"function","index":1}]}`), + finish("tool_calls"), + "[DONE]", + ) + checkEvents(t, 200, true, body, []pivot.Event{ + start(), + openTool(0, "a", "A"), delta(0, pivot.DeltaInputJSON, "{}"), closeBlock(0), + openTool(1, "b", "B"), delta(1, pivot.DeltaInputJSON, "{"), + apiError(502, "api_error", "the inference server sent arguments for tool call 0 after that tool call ended"), + }) +} + +func TestReplyDecoder_ErrorChunk(t *testing.T) { + checkEvents(t, 200, true, sse(chunk(`{"content":"a"}`), `{"error":{"message":"overloaded","type":"overloaded_error"}}`), []pivot.Event{ + start(), open(0, pivot.BlockText), delta(0, pivot.DeltaText, "a"), + apiError(500, "overloaded_error", "overloaded"), + }) + checkEvents(t, 200, true, sse(`{"error":{"message":"boom","type":"litellm_error","code":429}}`), []pivot.Event{ + apiError(429, "api_error", "boom"), + }) +} + +func TestReplyDecoder_HTMLErrorPage(t *testing.T) { + checkEvents(t, 502, true, "bad gateway\n", []pivot.Event{ + apiError(502, "api_error", "bad gateway"), + }) + long := strings.Repeat("x", 600) + if got := decodeAll(503, false, long); got[0].Message != long[:512] { + t.Errorf("long body message has %d bytes, want 512", len(got[0].Message)) + } + checkEvents(t, 503, false, "", []pivot.Event{apiError(503, "api_error", "Service Unavailable")}) +} + +func TestReplyDecoder_JSONError(t *testing.T) { + checkEvents(t, 404, false, `{"error":{"message":"model \"claude-opus-5-5\" not found or not accessible","type":"not_found_error"}}`, []pivot.Event{ + apiError(404, "not_found_error", `model "claude-opus-5-5" not found or not accessible`), + }) + checkEvents(t, 403, false, `{"error":{"message":"no access","type":"team_model_access_denied"}}`, []pivot.Event{ + apiError(403, "permission_error", "no access"), + }) +} + +func TestReplyDecoder_NotStream(t *testing.T) { + body := `{"id":"chatcmpl-1","model":"claude","choices":[{"index":0,"finish_reason":"tool_calls","message":{"role":"assistant",` + + `"content":"hi","thinking_blocks":[{"type":"thinking","thinking":"hmm","signature":"SIG"}],"reasoning_content":"hmm",` + + `"tool_calls":[{"id":"call_1__thought__a+b=","type":"function","function":{"name":"Bash","arguments":"{\"command\":\"ls\"}"}}]}}],` + + `"usage":{"prompt_tokens":10,"completion_tokens":3,"cache_read_input_tokens":4}}` + checkEvents(t, 200, false, body, []pivot.Event{ + start(), + open(0, pivot.BlockThinking), delta(0, pivot.DeltaThinking, "hmm"), delta(0, pivot.DeltaSignature, "SIG"), closeBlock(0), + open(1, pivot.BlockText), delta(1, pivot.DeltaText, "hi"), closeBlock(1), + openTool(2, "call_1__thought__a+b=", "Bash"), delta(2, pivot.DeltaInputJSON, `{"command":"ls"}`), closeBlock(2), + usage(pivot.Usage{Input: 6, Output: 3, CacheRead: 4}), + stop("tool_use"), + }) + + // No thinking_blocks: one unsigned thinking block from reasoning. + checkEvents(t, 200, false, `{"id":"chatcmpl-1","model":"claude","choices":[{"finish_reason":"stop","message":{"content":"a","reasoning":"r"}}]}`, []pivot.Event{ + start(), + open(0, pivot.BlockThinking), delta(0, pivot.DeltaThinking, "r"), closeBlock(0), + open(1, pivot.BlockText), delta(1, pivot.DeltaText, "a"), closeBlock(1), + usage(pivot.Usage{}), stop("end_turn"), + }) +} + +func TestReplyDecoder_InvalidArgumentsNotStream(t *testing.T) { + body := `{"id":"chatcmpl-1","model":"claude","choices":[{"finish_reason":"tool_calls","message":{"content":"x",` + + `"tool_calls":[{"id":"c","type":"function","function":{"name":"Bash","arguments":"{\"command\":"}}]}}]}` + checkEvents(t, 200, false, body, []pivot.Event{ + apiError(502, "api_error", "the inference server sent tool Bash arguments that are not valid JSON"), + }) +} + +func TestEncodeError(t *testing.T) { + got := string(Protocol{}.EncodeError(400, "invalid_request_error", "bad ")) + if want := `{"error":{"message":"bad ","type":"invalid_request_error","code":null}}`; got != want { + t.Errorf("got %s, want %s", got, want) + } +} + +// Two whole tool calls at one index are two tool_use blocks (rule 20a). +func TestReplyDecoder_NewIDOnOpenIndex(t *testing.T) { + body := sse( + chunk(`{"tool_calls":[{"id":"a","function":{"arguments":"{\"p\":1}","name":"A"},"type":"function","index":0},{"id":"b","function":{"arguments":"{\"q\":2}","name":"B"},"type":"function","index":0}]}`), + chunk(`{"tool_calls":[{"id":"b","function":{"arguments":""},"type":"function","index":0}]}`), + finish("tool_calls"), + "[DONE]", + ) + checkEvents(t, 200, true, body, []pivot.Event{ + start(), + openTool(0, "a", "A"), delta(0, pivot.DeltaInputJSON, `{"p":1}`), closeBlock(0), + openTool(1, "b", "B"), delta(1, pivot.DeltaInputJSON, `{"q":2}`), closeBlock(1), + usage(pivot.Usage{}), stop("tool_use"), + }) +} + +// Three calls at index 0, each in its own chunk: closing a and b must not +// refuse c. +func TestReplyDecoder_ThreeNewIDsOnIndexZero(t *testing.T) { + body := sse( + chunk(`{"tool_calls":[{"id":"a","function":{"arguments":"{}","name":"A"},"type":"function","index":0}]}`), + chunk(`{"tool_calls":[{"id":"b","function":{"arguments":"{}","name":"B"},"type":"function","index":0}]}`), + chunk(`{"tool_calls":[{"id":"c","function":{"arguments":"{}","name":"C"},"type":"function","index":0}]}`), + finish("tool_calls"), + "[DONE]", + ) + checkEvents(t, 200, true, body, []pivot.Event{ + start(), + openTool(0, "a", "A"), delta(0, pivot.DeltaInputJSON, "{}"), closeBlock(0), + openTool(1, "b", "B"), delta(1, pivot.DeltaInputJSON, "{}"), closeBlock(1), + openTool(2, "c", "C"), delta(2, pivot.DeltaInputJSON, "{}"), closeBlock(2), + usage(pivot.Usage{}), stop("tool_use"), + }) +} + +// An id whose block ended comes back: refuse it. +func TestReplyDecoder_ClosedIDAgain(t *testing.T) { + body := sse( + chunk(`{"tool_calls":[{"id":"a","function":{"arguments":"{}","name":"A"},"type":"function","index":0}]}`), + chunk(`{"tool_calls":[{"id":"b","function":{"arguments":"{}","name":"B"},"type":"function","index":1}]}`), + chunk(`{"tool_calls":[{"id":"a","function":{"arguments":"{}","name":"A"},"type":"function","index":0}]}`), + finish("tool_calls"), + "[DONE]", + ) + checkEvents(t, 200, true, body, []pivot.Event{ + start(), + openTool(0, "a", "A"), delta(0, pivot.DeltaInputJSON, "{}"), closeBlock(0), + openTool(1, "b", "B"), delta(1, pivot.DeltaInputJSON, "{}"), + apiError(502, "api_error", "the inference server sent tool call a again after that tool call ended"), + }) +} + +// A reply with a tool call stops with tool_use when finish_reason is stop: +// vLLM and OpenAI send stop for a forced named tool. A length cut +// stays max_tokens. +func TestReplyDecoder_ToolCallWithStop(t *testing.T) { + call := chunk(`{"tool_calls":[{"id":"a","function":{"arguments":"{}","name":"A"},"type":"function","index":0}]}`) + for reason, want := range map[string]string{"stop": "tool_use", "length": "max_tokens"} { + checkEvents(t, 200, true, sse(call, finish(reason), "[DONE]"), []pivot.Event{ + start(), + openTool(0, "a", "A"), delta(0, pivot.DeltaInputJSON, `{}`), closeBlock(0), + usage(pivot.Usage{}), stop(want), + }) + body := `{"id":"chatcmpl-1","model":"claude","choices":[{"finish_reason":"` + reason + `","message":{` + + `"tool_calls":[{"id":"a","type":"function","function":{"name":"A","arguments":"{}"}}]}}]}` + checkEvents(t, 200, false, body, []pivot.Event{ + start(), + openTool(0, "a", "A"), delta(0, pivot.DeltaInputJSON, `{}`), closeBlock(0), + usage(pivot.Usage{}), stop(want), + }) + } +} + +// A not-streamed reply refuses a tool call with no id or a repeated id +// (rule 20a). +func TestReplyDecoder_ToolCallIDsNotStream(t *testing.T) { + for calls, msg := range map[string]string{ + `{"type":"function","function":{"name":"A","arguments":"{}"}}`: "the inference server sent tool A with no id", + `{"id":"a","type":"function","function":{"name":"A","arguments":"{}"}},{"id":"a","type":"function","function":{"name":"B","arguments":"{}"}}`: "the inference server sent tool call a twice", + } { + body := `{"id":"chatcmpl-1","model":"claude","choices":[{"finish_reason":"tool_calls","message":{"content":"x","tool_calls":[` + calls + `]}}]}` + checkEvents(t, 200, false, body, []pivot.Event{apiError(502, "api_error", msg)}) + } +} diff --git a/core/translate/chat/request.go b/core/translate/chat/request.go new file mode 100644 index 000000000..6380d8181 --- /dev/null +++ b/core/translate/chat/request.go @@ -0,0 +1,313 @@ +package chat + +import ( + "encoding/json" + "net/http" + "sort" + "strings" + + "github.com/rossoctl/cortex/core/translate/pivot" +) + +// request is the chat-completions body. It has only the top-level fields +// of rule 11, in that order. Do not add a field that is not in rule 11. +type request struct { + Model string `json:"model"` + Messages []message `json:"messages"` + MaxTokens int `json:"max_tokens,omitempty"` + Stream bool `json:"stream,omitempty"` + StreamOptions *streamOptions `json:"stream_options,omitempty"` + Tools []tool `json:"tools,omitempty"` + ToolChoice json.RawMessage `json:"tool_choice,omitempty"` + ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"` + Stop []string `json:"stop,omitempty"` + Temperature *float64 `json:"temperature,omitempty"` + TopP *float64 `json:"top_p,omitempty"` + ReasoningEffort string `json:"reasoning_effort,omitempty"` +} + +type streamOptions struct { + IncludeUsage bool `json:"include_usage"` +} + +// message is one chat message. Content is a []part for system, user and +// tool messages (rule 9a), and a string or nil for assistant messages. +type message struct { + Role string `json:"role"` + Content any `json:"content,omitempty"` + ThinkingBlocks []thinkingBlock `json:"thinking_blocks,omitempty"` + ToolCalls []toolCall `json:"tool_calls,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` +} + +type part struct { + Type string `json:"type"` + Text *string `json:"text,omitempty"` + ImageURL *imageURL `json:"image_url,omitempty"` + CacheControl json.RawMessage `json:"cache_control,omitempty"` +} + +type imageURL struct { + URL string `json:"url"` +} + +// thinkingBlock is LiteLLM's thinking_blocks entry (rule 15). +type thinkingBlock struct { + Type string `json:"type"` + Thinking *string `json:"thinking,omitempty"` + Signature string `json:"signature,omitempty"` + Data string `json:"data,omitempty"` +} + +// toolCall has no index field: Bob answers 422 to it (rule 15). +type toolCall struct { + ID string `json:"id"` + Type string `json:"type"` + Function functionCall `json:"function"` +} + +type functionCall struct { + Name string `json:"name"` + Arguments string `json:"arguments"` +} + +type tool struct { + Type string `json:"type"` + Function toolFunction `json:"function"` +} + +type toolFunction struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + Parameters json.RawMessage `json:"parameters,omitempty"` +} + +func refuse(msg string) error { + return &pivot.RequestError{Status: http.StatusBadRequest, Type: "invalid_request_error", Message: msg} +} + +var errDocument = refuse("this inference server cannot read documents (PDF)") + +// EncodeRequest converts a pivot request to a chat-completions body +// (rules 11-19). It removes the Anthropic headers from h. dropped is the +// sorted list of the fields that it did not send. +func (Protocol) EncodeRequest(req pivot.Request, h http.Header) ([]byte, []string, error) { + h.Del("Anthropic-Version") + h.Del("Anthropic-Beta") + + e := encoder{dropped: map[string]bool{}} + e.extra(req.Extra) + out := request{ + Model: req.Model, + Messages: []message{}, + MaxTokens: req.MaxTokens, + Stream: req.Stream, + Stop: req.Stop, + Temperature: req.Temperature, + TopP: req.TopP, + } + if req.Stream { + out.StreamOptions = &streamOptions{IncludeUsage: true} + } + if len(req.System) > 0 { + out.Messages = append(out.Messages, message{Role: "system", Content: e.textParts(req.System)}) + } + for _, m := range req.Messages { + e.extra(m.Extra) + var msgs []message + var err error + if m.Role == "assistant" { + msgs = []message{e.assistant(m.Content)} + } else { + // Rule 13a: a system message after the first message is + // sent as a user message. + msgs, err = e.user(m.Content) + } + if err != nil { + return nil, nil, err + } + out.Messages = append(out.Messages, msgs...) + } + for _, t := range req.Tools { + if t.Type != "" { + return nil, nil, refuse(`this inference server cannot run the server tool "` + t.Name + `" (type ` + t.Type + `)`) + } + // Rule 16: no cache_control on a tool. + out.Tools = append(out.Tools, tool{Type: "function", Function: toolFunction{Name: t.Name, Description: t.Description, Parameters: t.InputSchema}}) + } + if c := req.ToolChoice; c != nil { + out.ToolChoice = toolChoice(*c) + if c.DisableParallel { + f := false + out.ParallelToolCalls = &f + } + } + out.ReasoningEffort = effort(req) + + body, err := marshal(out) + if err != nil { + return nil, nil, refuse("the request has a field that is not valid JSON: " + err.Error()) + } + dropped := make([]string, 0, len(e.dropped)) + for k := range e.dropped { + dropped = append(dropped, k) + } + sort.Strings(dropped) + return body, dropped, nil +} + +// encoder collects the names of the dropped fields (decision 11). +type encoder struct { + dropped map[string]bool +} + +// extra names each field of another protocol, without its prefix. +func (e *encoder) extra(x pivot.Extra) { + for k := range x { + proto, field, ok := strings.Cut(k, ".") + if ok && proto != "chat-completions" { + e.dropped[field] = true + } + } +} + +func (e *encoder) textParts(blocks []pivot.Block) []part { + parts := make([]part, 0, len(blocks)) + for _, b := range blocks { + e.extra(b.Extra) + parts = append(parts, textPart(b.Text, b.CacheControl)) + } + return parts +} + +func textPart(text string, cc json.RawMessage) part { + return part{Type: "text", Text: &text, CacheControl: cc} +} + +func imagePart(b pivot.Block) part { + url := b.URL + if b.SourceType == "base64" { + url = "data:" + b.MediaType + ";base64," + b.Data + } + return part{Type: "image_url", ImageURL: &imageURL{URL: url}, CacheControl: b.CacheControl} +} + +// user converts a user (or system) turn by rule 14: the tool messages +// first, then a user message with the images of the tool results, then a +// user message with the other content. +func (e *encoder) user(blocks []pivot.Block) ([]message, error) { + var msgs []message + var toolImages, rest []part + for _, b := range blocks { + e.extra(b.Extra) + switch b.Kind { + case pivot.BlockText: + rest = append(rest, textPart(b.Text, b.CacheControl)) + case pivot.BlockImage: + rest = append(rest, imagePart(b)) + case pivot.BlockDocument: + return nil, errDocument + case pivot.BlockToolResult: + if b.IsError { + e.dropped["is_error"] = true + } + var text []part + for _, c := range b.Content { + e.extra(c.Extra) + switch c.Kind { + case pivot.BlockText: + text = append(text, textPart(c.Text, nil)) + case pivot.BlockImage: + toolImages = append(toolImages, imagePart(c)) + case pivot.BlockDocument: + return nil, errDocument + } + } + if len(text) == 0 { + // One empty part keeps the content a list and gives the + // cache marker a place. + text = []part{textPart("", nil)} + } + text[len(text)-1].CacheControl = b.CacheControl + msgs = append(msgs, message{Role: "tool", Content: text, ToolCallID: b.ToolUseID}) + } + } + if len(toolImages) > 0 { + msgs = append(msgs, message{Role: "user", Content: toolImages}) + } + if len(rest) > 0 { + msgs = append(msgs, message{Role: "user", Content: rest}) + } + return msgs, nil +} + +// assistant converts an assistant turn by rule 15 and decisions 9-10. +func (e *encoder) assistant(blocks []pivot.Block) message { + m := message{Role: "assistant"} + var text strings.Builder + for _, b := range blocks { + e.extra(b.Extra) + switch b.Kind { + case pivot.BlockText: + text.WriteString(b.Text) + case pivot.BlockToolUse: + args := string(b.Input) + if args == "" { + args = "{}" + } + // Rule 20a: the id goes back byte-exact. + m.ToolCalls = append(m.ToolCalls, toolCall{ID: b.ID, Type: "function", Function: functionCall{Name: b.Name, Arguments: args}}) + case pivot.BlockThinking: + if b.Signature == "" { + e.dropped["thinking"] = true + continue + } + t := b.Text + m.ThinkingBlocks = append(m.ThinkingBlocks, thinkingBlock{Type: "thinking", Thinking: &t, Signature: b.Signature}) + case pivot.BlockRedactedThinking: + m.ThinkingBlocks = append(m.ThinkingBlocks, thinkingBlock{Type: "redacted_thinking", Data: b.Data}) + } + } + if text.Len() > 0 { + m.Content = text.String() + } + return m +} + +// toolChoice converts a tool choice by rule 17. +func toolChoice(c pivot.ToolChoice) json.RawMessage { + switch c.Type { + case "auto": + return json.RawMessage(`"auto"`) + case "any": + return json.RawMessage(`"required"`) + case "none": + return json.RawMessage(`"none"`) + case "tool": + b, _ := marshal(struct { + Type string `json:"type"` + Function struct { + Name string `json:"name"` + } `json:"function"` + }{Type: "function", Function: struct { + Name string `json:"name"` + }{c.Name}}) + return b + } + return nil +} + +// effort gives reasoning_effort by rule 19. +func effort(req pivot.Request) string { + switch req.Effort { + case "low", "medium", "high": + return req.Effort + case "": + default: + return "high" + } + if req.Thinking != nil && (req.Thinking.Type == "enabled" || req.Thinking.Type == "adaptive") { + return "medium" + } + return "" +} diff --git a/core/translate/chat/request_test.go b/core/translate/chat/request_test.go new file mode 100644 index 000000000..7d575faaa --- /dev/null +++ b/core/translate/chat/request_test.go @@ -0,0 +1,213 @@ +package chat + +import ( + "encoding/json" + "errors" + "net/http" + "reflect" + "strings" + "testing" + + "github.com/rossoctl/cortex/core/translate/pivot" +) + +var ephemeral = json.RawMessage(`{"type":"ephemeral"}`) + +func ptr(f float64) *float64 { return &f } + +// fullRequest has one block of each kind except document. +func fullRequest() pivot.Request { + const id = "call_1__thought__a+b/c=" + return pivot.Request{ + Model: "m", + System: []pivot.Block{{Kind: pivot.BlockText, Text: "sys", CacheControl: ephemeral}}, + Messages: []pivot.Message{ + {Role: "user", Content: []pivot.Block{ + {Kind: pivot.BlockText, Text: "hi ", CacheControl: ephemeral}, + {Kind: pivot.BlockImage, SourceType: "base64", MediaType: "image/png", Data: "AAA"}, + {Kind: pivot.BlockImage, SourceType: "url", URL: "https://x/a.png"}, + }}, + {Role: "assistant", Extra: pivot.Extra{"anthropic.id": json.RawMessage(`"x"`)}, Content: []pivot.Block{ + {Kind: pivot.BlockThinking, Text: "hmm", Signature: "SIG"}, + {Kind: pivot.BlockThinking, Text: "plain"}, + {Kind: pivot.BlockRedactedThinking, Data: "RD"}, + {Kind: pivot.BlockText, Text: "o"}, + {Kind: pivot.BlockText, Text: "k"}, + {Kind: pivot.BlockToolUse, ID: id, Name: "Bash", Input: json.RawMessage(`{"command":"ls"}`)}, + }}, + {Role: "user", Content: []pivot.Block{ + {Kind: pivot.BlockToolResult, ToolUseID: id, IsError: true, CacheControl: ephemeral, Content: []pivot.Block{ + {Kind: pivot.BlockText, Text: "out"}, + {Kind: pivot.BlockImage, SourceType: "base64", MediaType: "image/png", Data: "BBB", Extra: pivot.Extra{"anthropic.citations": json.RawMessage(`[]`)}}, + }}, + {Kind: pivot.BlockText, Text: "next"}, + }}, + {Role: "system", Content: []pivot.Block{{Kind: pivot.BlockText, Text: "hook"}}}, + }, + Tools: []pivot.Tool{{Name: "Bash", Description: "run", InputSchema: json.RawMessage(`{"type":"object"}`), CacheControl: ephemeral}}, + ToolChoice: &pivot.ToolChoice{Type: "any", DisableParallel: true}, + MaxTokens: 1000, + Stop: []string{"END"}, + Temperature: ptr(0.5), + TopP: ptr(0.9), + Stream: true, + Effort: "high", + Thinking: &pivot.Thinking{Type: "enabled", BudgetTokens: 1000}, + Extra: pivot.Extra{ + "anthropic.top_k": json.RawMessage(`5`), + "anthropic.metadata": json.RawMessage(`{"user_id":"u"}`), + }, + } +} + +func TestEncodeRequest_Shape(t *testing.T) { + body, dropped, err := Protocol{}.EncodeRequest(fullRequest(), nil) + if err != nil { + t.Fatal(err) + } + want := `{"model":"m","messages":[` + + `{"role":"system","content":[{"type":"text","text":"sys","cache_control":{"type":"ephemeral"}}]},` + + `{"role":"user","content":[{"type":"text","text":"hi ","cache_control":{"type":"ephemeral"}},{"type":"image_url","image_url":{"url":"data:image/png;base64,AAA"}},{"type":"image_url","image_url":{"url":"https://x/a.png"}}]},` + + `{"role":"assistant","content":"ok","thinking_blocks":[{"type":"thinking","thinking":"hmm","signature":"SIG"},{"type":"redacted_thinking","data":"RD"}],"tool_calls":[{"id":"call_1__thought__a+b/c=","type":"function","function":{"name":"Bash","arguments":"{\"command\":\"ls\"}"}}]},` + + `{"role":"tool","content":[{"type":"text","text":"out","cache_control":{"type":"ephemeral"}}],"tool_call_id":"call_1__thought__a+b/c="},` + + `{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,BBB"}}]},` + + `{"role":"user","content":[{"type":"text","text":"next"}]},` + + `{"role":"user","content":[{"type":"text","text":"hook"}]}` + + `],"max_tokens":1000,"stream":true,"stream_options":{"include_usage":true},` + + `"tools":[{"type":"function","function":{"name":"Bash","description":"run","parameters":{"type":"object"}}}],` + + `"tool_choice":"required","parallel_tool_calls":false,"stop":["END"],"temperature":0.5,"top_p":0.9,"reasoning_effort":"high"}` + if string(body) != want { + t.Errorf("body:\n got %s\nwant %s", body, want) + } + wantDropped := []string{"citations", "id", "is_error", "metadata", "thinking", "top_k"} + if !reflect.DeepEqual(dropped, wantDropped) { + t.Errorf("dropped = %q, want %q", dropped, wantDropped) + } + + // Rule 9a: the same request gives the same bytes. + again, _, _ := Protocol{}.EncodeRequest(fullRequest(), nil) + if string(again) != string(body) { + t.Error("two encodings of one request differ") + } +} + +func TestEncodeRequest_ToolChoice(t *testing.T) { + for _, tc := range []struct { + choice pivot.ToolChoice + want string + }{ + {pivot.ToolChoice{Type: "auto"}, `"tool_choice":"auto"}`}, + {pivot.ToolChoice{Type: "none"}, `"tool_choice":"none"}`}, + {pivot.ToolChoice{Type: "tool", Name: "Bash"}, `"tool_choice":{"type":"function","function":{"name":"Bash"}}}`}, + } { + req := pivot.Request{Model: "m", ToolChoice: &tc.choice} + body, _, err := Protocol{}.EncodeRequest(req, nil) + if err != nil { + t.Fatal(err) + } + want := `{"model":"m","messages":[],` + tc.want + if string(body) != want { + t.Errorf("%s: got %s, want %s", tc.choice.Type, body, want) + } + } +} + +func wantRefusal(t *testing.T, err error, msg string) { + t.Helper() + var re *pivot.RequestError + if !errors.As(err, &re) { + t.Fatalf("err = %v, want *pivot.RequestError", err) + } + if re.Status != 400 || re.Type != "invalid_request_error" || re.Message != msg { + t.Errorf("err = %+v, want 400 invalid_request_error %q", re, msg) + } +} + +func TestEncodeRequest_RefusesDocument(t *testing.T) { + doc := pivot.Block{Kind: pivot.BlockDocument, SourceType: "base64", MediaType: "application/pdf", Data: "JVBE"} + for name, content := range map[string][]pivot.Block{ + "user": {doc}, + "tool_result": {{Kind: pivot.BlockToolResult, ToolUseID: "t", Content: []pivot.Block{doc}}}, + } { + req := pivot.Request{Model: "m", Messages: []pivot.Message{{Role: "user", Content: content}}} + _, _, err := Protocol{}.EncodeRequest(req, nil) + t.Run(name, func(t *testing.T) { wantRefusal(t, err, "this inference server cannot read documents (PDF)") }) + } +} + +func TestEncodeRequest_RefusesServerTool(t *testing.T) { + req := pivot.Request{Model: "m", Tools: []pivot.Tool{{Name: "web_search", Type: "web_search_20250305"}}} + _, _, err := Protocol{}.EncodeRequest(req, nil) + wantRefusal(t, err, `this inference server cannot run the server tool "web_search" (type web_search_20250305)`) +} + +func TestEncodeRequest_DeletesAnthropicHeaders(t *testing.T) { + h := http.Header{} + h.Set("Anthropic-Version", "2023-06-01") + h.Set("Anthropic-Beta", "x") + h.Set("X-Keep", "1") + if _, _, err := (Protocol{}).EncodeRequest(pivot.Request{Model: "m"}, h); err != nil { + t.Fatal(err) + } + if len(h) != 1 || h.Get("X-Keep") != "1" { + t.Errorf("headers = %v, want only X-Keep", h) + } +} + +func TestEncodeRequest_Effort(t *testing.T) { + for _, tc := range []struct { + effort, thinking, want string + }{ + {"low", "", "low"}, + {"medium", "", "medium"}, + {"high", "", "high"}, + {"max", "", "high"}, + {"xhigh", "disabled", "high"}, + {"", "enabled", "medium"}, + {"", "adaptive", "medium"}, + {"", "disabled", ""}, + {"", "", ""}, + } { + req := pivot.Request{Model: "m", Effort: tc.effort} + if tc.thinking != "" { + req.Thinking = &pivot.Thinking{Type: tc.thinking} + } + body, _, err := Protocol{}.EncodeRequest(req, nil) + if err != nil { + t.Fatal(err) + } + want := `{"model":"m","messages":[]}` + if tc.want != "" { + want = `{"model":"m","messages":[],"reasoning_effort":"` + tc.want + `"}` + } + if string(body) != want { + t.Errorf("effort %q thinking %q: got %s, want %s", tc.effort, tc.thinking, body, want) + } + } +} + +// Rule 9a: a new turn keeps the bytes before it, and a moved cache +// marker changes only the marker's bytes. +func TestEncodeRequest_PrefixStable(t *testing.T) { + req := fullRequest() + req.Stream = false + base, _, _ := Protocol{}.EncodeRequest(req, nil) + + req.Messages = append(req.Messages, pivot.Message{Role: "assistant", Content: []pivot.Block{{Kind: pivot.BlockText, Text: "done"}}}) + longer, _, _ := Protocol{}.EncodeRequest(req, nil) + idx := strings.Index(string(base), `],"max_tokens"`) + if idx < 0 || string(longer[:idx]) != string(base[:idx]) { + t.Errorf("bytes before the new turn changed:\n%s\n%s", base, longer) + } + + moved := fullRequest() + moved.Stream = false + moved.Messages[0].Content[0].CacheControl = nil + moved.Messages[2].Content[1].CacheControl = ephemeral + got, _, _ := Protocol{}.EncodeRequest(moved, nil) + marker := `,"cache_control":{"type":"ephemeral"}` + wantMoved := strings.Replace(strings.Replace(string(base), `"hi "`+marker, `"hi "`, 1), `"next"}`, `"next"`+marker+`}`, 1) + if string(got) != wantMoved { + t.Errorf("moved marker:\n got %s\nwant %s", got, wantMoved) + } +} diff --git a/core/translate/doc.go b/core/translate/doc.go new file mode 100644 index 000000000..3ed3276e5 --- /dev/null +++ b/core/translate/doc.go @@ -0,0 +1,5 @@ +// Package translate converts inference requests and replies from one +// protocol to another through the pivot type in package pivot. Each +// protocol has its own subpackage (anthropic, chat). This package holds +// the compiled-in registry of protocols. +package translate diff --git a/core/translate/pivot/event.go b/core/translate/pivot/event.go new file mode 100644 index 000000000..e9431a206 --- /dev/null +++ b/core/translate/pivot/event.go @@ -0,0 +1,43 @@ +package pivot + +// EventKind is the kind of a reply event. A reply is a list of events, +// for streamed and not-streamed replies alike (rule H3). +type EventKind int + +// The reply event kinds of rule H3. +const ( + EventStart EventKind = iota + EventBlockStart + EventDelta + EventBlockStop + EventStop + EventUsage + EventError +) + +// DeltaKind is the kind of payload in an EventDelta. +type DeltaKind int + +// The delta kinds of rule H3. +const ( + DeltaText DeltaKind = iota + DeltaThinking + DeltaSignature + DeltaInputJSON // a JSON piece of tool input +) + +// Event is one reply event. Kind tells which fields apply. +type Event struct { + Kind EventKind + ID, Model string // start + Index int // block_start, delta, block_stop + Block BlockKind // block_start + ToolID, ToolName string // block_start of tool_use + Delta DeltaKind // delta + Value string // delta payload + StopReason string // stop: Anthropic vocabulary (end_turn, max_tokens, tool_use, refusal) + Usage Usage // usage + Status int // error + ErrorType string // error: Anthropic error type vocabulary + Message string // error +} diff --git a/core/translate/pivot/pivot.go b/core/translate/pivot/pivot.go new file mode 100644 index 000000000..755d3b3d0 --- /dev/null +++ b/core/translate/pivot/pivot.go @@ -0,0 +1,100 @@ +// Package pivot holds the protocol-neutral form of an inference request +// and reply, and the contract that each protocol package implements. +// +// The pivot type holds the union of what the protocols can say, and loses +// nothing that a later turn needs (rule H2). +package pivot + +import "encoding/json" + +// Extra holds the fields that the pivot type has no place for. A decoder +// puts each such field here under the key ".". An encoder +// reads only the keys of its own protocol. +type Extra map[string]json.RawMessage + +// BlockKind is the kind of a content block. +type BlockKind string + +// The block kinds of rule H2. +const ( + BlockText BlockKind = "text" + BlockImage BlockKind = "image" + BlockDocument BlockKind = "document" + BlockToolUse BlockKind = "tool_use" + BlockToolResult BlockKind = "tool_result" + BlockThinking BlockKind = "thinking" + BlockRedactedThinking BlockKind = "redacted_thinking" +) + +// Request is one inference request in the pivot form. +type Request struct { + Model string + System []Block // text blocks + Messages []Message + Tools []Tool + ToolChoice *ToolChoice + MaxTokens int + Stop []string + Temperature *float64 + TopP *float64 + Stream bool + Effort string // "" when none + Thinking *Thinking // nil when none + Extra Extra +} + +// Message is one turn of the conversation. Role is user, assistant or +// system. +type Message struct { + Role string + Content []Block + Extra Extra +} + +// Block is one content block. Kind tells which fields apply. +type Block struct { + Kind BlockKind + Text string // text; thinking text + SourceType string // image/document: "base64" or "url" + MediaType string // image/document + Data string // base64 data; redacted_thinking data + URL string // image/document with SourceType "url" + ID, Name string // tool_use + Input json.RawMessage // tool_use input, raw + ToolUseID string // tool_result + Content []Block // tool_result content + IsError bool // tool_result + Signature string // thinking + CacheControl json.RawMessage // nil when none + Extra Extra +} + +// Tool is one tool that the model can call. +type Tool struct { + Name, Description string + InputSchema json.RawMessage + Type string // "" for a function tool; else a server tool type + CacheControl json.RawMessage + Extra Extra +} + +// ToolChoice tells the model how to use the tools. Type is auto, any, +// tool or none. Name is set when Type is tool. +type ToolChoice struct { + Type string + Name string + DisableParallel bool +} + +// Thinking is the extended-thinking setting. Type is enabled, adaptive or +// disabled. +type Thinking struct { + Type string + BudgetTokens int +} + +// Usage counts tokens with the Anthropic meaning: Input does not include +// the cache tokens. Each decoder converts to this meaning. +type Usage struct { + Input, Output, CacheRead, CacheCreation int +} diff --git a/core/translate/pivot/protocol.go b/core/translate/pivot/protocol.go new file mode 100644 index 000000000..b27ada5ed --- /dev/null +++ b/core/translate/pivot/protocol.go @@ -0,0 +1,86 @@ +package pivot + +import ( + "net/http" + "strconv" +) + +// RequestError is a refusal to send to the agent, in the shape of the +// agent's protocol (EncodeError). +type RequestError struct { + Status int + Type, Message string +} + +// Error gives the status, the type and the message. +func (e *RequestError) Error() string { + return strconv.Itoa(e.Status) + " " + e.Type + ": " + e.Message +} + +// ErrorType maps an HTTP status to the Anthropic error type of rule 24: +// 400,422 invalid_request_error; 401 authentication_error; 402,403 +// permission_error; 404 not_found_error; 429 rate_limit_error; 5xx and +// every other status api_error. +func ErrorType(status int) string { + switch status { + case 400, 422: + return "invalid_request_error" + case 401: + return "authentication_error" + case 402, 403: + return "permission_error" + case 404: + return "not_found_error" + case 429: + return "rate_limit_error" + } + return "api_error" +} + +// Protocol is one inference protocol (rule H4). Each direction is an +// optional interface below, and a protocol can leave a direction out. +// Every protocol must supply EncodeError. +type Protocol interface { + Name() string + // EncodeError gives an error body in this protocol's shape. + EncodeError(status int, errType, message string) []byte +} + +// RequestDecoder converts a request body of this protocol to the pivot form. +type RequestDecoder interface { + DecodeRequest(body []byte) (Request, error) +} + +// RequestEncoder converts a pivot request to a body of this protocol. It +// removes the source protocol's own headers from h. It does not choose the +// path or the auth header. dropped names each field that it could not send. +type RequestEncoder interface { + EncodeRequest(req Request, h http.Header) (body []byte, dropped []string, err error) +} + +// ReplyDecoding makes a decoder for one upstream reply. status is the +// upstream status, and stream tells if the upstream body is SSE. +type ReplyDecoding interface { + NewReplyDecoder(status int, stream bool) ReplyDecoder +} + +// ReplyEncoding makes an encoder for one reply to the agent. stream tells +// if the agent asked for a stream. +type ReplyEncoding interface { + NewReplyEncoder(stream bool) ReplyEncoder +} + +// ReplyDecoder converts the bytes of an upstream reply to events. It keeps +// state between calls and never holds a whole 2xx stream. +type ReplyDecoder interface { + Feed(p []byte) []Event + End() []Event +} + +// ReplyEncoder converts events to bytes for the agent. For a stream it +// gives bytes at once. For a not-streamed reply it gives nil until End, +// which gives the one body. +type ReplyEncoder interface { + Encode(ev Event) []byte + End() []byte +} diff --git a/core/translate/pivot/protocol_test.go b/core/translate/pivot/protocol_test.go new file mode 100644 index 000000000..96915929e --- /dev/null +++ b/core/translate/pivot/protocol_test.go @@ -0,0 +1,32 @@ +package pivot + +import "testing" + +func TestErrorType(t *testing.T) { + for _, tc := range []struct { + status int + want string + }{ + {400, "invalid_request_error"}, + {422, "invalid_request_error"}, + {401, "authentication_error"}, + {402, "permission_error"}, + {403, "permission_error"}, + {404, "not_found_error"}, + {429, "rate_limit_error"}, + {500, "api_error"}, + {503, "api_error"}, + {418, "api_error"}, + } { + if got := ErrorType(tc.status); got != tc.want { + t.Errorf("ErrorType(%d) = %q, want %q", tc.status, got, tc.want) + } + } +} + +func TestRequestError(t *testing.T) { + var err error = &RequestError{Status: 400, Type: "invalid_request_error", Message: "bad"} + if got := err.Error(); got != "400 invalid_request_error: bad" { + t.Errorf("Error() = %q", got) + } +} diff --git a/docs/superpowers/plans/2026-11-17-translation-hub-1911.md b/docs/superpowers/plans/2026-11-17-translation-hub-1911.md new file mode 100644 index 000000000..65dc7ccd0 --- /dev/null +++ b/docs/superpowers/plans/2026-11-17-translation-hub-1911.md @@ -0,0 +1,483 @@ +# Translation hub (Anthropic Messages to chat completions) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** The inference router can send Claude Code's Anthropic `/v1/messages` requests to a server that serves only OpenAI `/v1/chat/completions`, and Claude Code gets valid Anthropic replies, streamed or not. + +**Architecture:** A new stdlib-only hub, `core/translate`: one pivot type (`core/translate/pivot`), one package per protocol (`anthropic`, `chat`), and a compiled-in registry (`core/translate`). The router reads three new server fields (`api`, `auth_header`, `headers`) and asks for translation through a new `pipeline.Context` call. The forward proxy converts the request after the last request plugin and converts the reply after the resend loop, before any reply plugin or recorder reads it. + +**Tech Stack:** Go 1.26.10, standard library only. No new module dependency. + +**Spec:** GitHub issue rossoctl/cortex#1911 (`gh issue view 1911 -R rossoctl/cortex`). The issue is the spec. Rule numbers in this plan (H1-H6, C1-C4, 4-10, F1, 11-24) are the issue's rule numbers. **Every implementer reads the issue rules that their task names before starting.** + +## Global Constraints + +- Standard library only. `go mod tidy -diff` must print nothing in `core` and `cmd/agentop`. +- `core/translate/...` imports only the standard library and other `core/translate` packages. +- `core/plugins/inferencerouter/routerconfig` stays stdlib-only (agentop imports it). It must not import `core/translate`. +- Protocol names (exact): `anthropic-messages`, `chat-completions`, `responses`. +- Deterministic encoding (rule 9a): structs with fixed field order; any map is marshalled by `encoding/json` (sorted keys). User, system and tool-message content is always a list of parts, never a plain string. +- Tool-call ids pass byte-exact in both directions (rule 20a). Never check, shorten or rewrite an id. +- Never copy an unknown top-level field into a chat body (rule 11). +- Test fixtures are written by hand from the issue's "Measured Bob stream shape" section. Do not commit a captured Claude Code request. +- Golden expected-output files live under `testdata/` and are written by the test's `-update` flag: `var update = flag.Bool("update", false, "rewrite golden files")`. A reviewer reads each golden file against the issue rules. Input fixtures are hand-written. +- Run every Go command per module with `GOWORK=off`. Commits use `git commit -s`. Comments and commit messages in ASD-STE100 Simplified Technical English. +- The server URL has no path today (`routerconfig.Server.URL`: "no path"). The issue's "path prefix" comes in #1912. In this plan the wire path is exactly `api[]`. +- Fallback doc URL (used by the rule 4a error, exact): `https://github.com/rossoctl/cortex/blob/main/docs/translation-server.md`. + +## Decisions made for this plan (with reasons) + +1. **Package layout:** `core/translate/pivot` (types and interfaces), `core/translate/anthropic`, `core/translate/chat`, `core/translate` (registry, helpers, cross-protocol goldens). Reason: a registry in the same package as the pivot type would import the protocol packages, which import the pivot type: an import cycle. +2. **Directions are optional interfaces** that a protocol value implements; the registry type-asserts them. Reason: rule H4 "a protocol can leave a direction out". +3. **`NewReplyDecoder(status int, stream bool)`** takes the upstream status and whether the upstream body is SSE. Reason: a chat server can answer a stream request with a JSON error; the decoder must know which body it reads. A non-2xx decoder reads the whole body (at most 1 MiB) and gives one `error` event. +4. **Encoder `stream` follows the agent's request** (`stream` in the agent's body), decoder `stream` follows the upstream `Content-Type`. Reason: the agent must get the shape it asked for even when the server ignores `stream`. +5. **Event stream end:** the chat decoder emits `usage` then `stop` at `[DONE]`. When the body ends after `finish_reason` with no `[DONE]`, it emits them at `End()`. When the body ends before any `finish_reason`, it emits an `error` event (502 `api_error`, "the inference server's stream ended before the reply was complete"). Reason: a broken stream must not look like a finished reply. +6. **Tool-call pieces out of order:** an `arguments` piece for a tool-call index whose block is already closed gives an `error` event (502 `api_error`, naming the index). Reason: no measured server does this; buffering it is code for no known case. +7. **Rule 20 invalid tool JSON** is checked only for a not-streamed reply. Reason: a stream cannot be checked without buffering it (rule 10). +8. **Tool messages (rule 14):** `content` is a list of `text` parts, with the `tool_result`'s `cache_control` on its last part. Reason: rule 9a (always parts) and the cache marker. Whether Bob accepts list content on `role: tool` is a check for a human (#1912's list). +9. **Assistant messages (rule 15):** `content` is a string of the joined text blocks, left out when there is no text. +10. **Unsigned `thinking` in assistant history** is not sent and is named in `dropped` as `thinking`. +11. **`dropped`** is a sorted, unique list: each request, message and block `Extra` key of another protocol with its `.` prefix removed (for example `top_k`, `metadata`), plus `is_error` when a `tool_result` had `is_error: true`, plus `thinking` (decision 10). +12. **Preference order for the target protocol** (rule 4): `chat-completions`, `anthropic-messages`, `responses`; the first one in the `api` map whose protocol has a request encoder. +13. **The rule 4a message** names every `api` key, sorted, joined with " or ": `Cortex cannot translate anthropic-messages to responses yet. Use a translation server for this pair (see https://github.com/rossoctl/cortex/blob/main/docs/translation-server.md).` +14. **Source protocol** comes from a new exported `inference.ProtocolFor(path string) string` in `core/parse/inference/dialect.go`, which calls `dialectFor`. It gives `anthropic-messages` for the Anthropic dialect, `chat-completions` only when the path ends in `/chat/completions`, `responses` for the Responses dialect, and `""` otherwise (legacy `/v1/completions` is `""`). Reason: rule 4 (one rule for parser and translator) without renaming the unexported type through the parser. +15. **routerconfig keeps its own list** `APIProtocols = []string{"anthropic-messages", "chat-completions", "responses"}`. A test in `core/plugins/inferencerouter` pins it equal to `translate.Names()`. Reason: routerconfig must stay free of `core/translate`; this repeats the `NoAgent` pattern that the package already uses. +16. **Two pipeline calls**, beside `Redirect` and `SetRedirectPath`: `Translate(source, target, path)` and `SetUpstreamHeaders(h)`. Both have the gates of `SetRedirectPath`. Reason: rule 5 (translation target kept privately), and rule C3 (fixed headers "after the plugins run": only the listener can apply them after the last plugin). +17. **The listener records `modify/translated`**, because only the encoder knows `dropped`. It converts the request right after the pipeline run and the reject check, before the request row is recorded, so the invocation is on the request row. All plugins have run at that point (rule 8). +18. **The listener passes `pctx.Headers` to `EncodeRequest`**, which deletes `anthropic-version` and `anthropic-beta` there. The sync to the wire request then removes them. +19. **The refused attempt row of a resend** (`resendIfAsked`) records the Anthropic form of the refused body (`translate.Reply`). The raw head is still what `Resend` reads (rule 8) and what the capture keeps. +20. **Capture field names:** `model.RequestCapture.Translated []byte`; `model.ResponseCapture.Upstream []byte` and `UpstreamTruncated bool`; `model.RawCapture.Translated *string` / `TranslatedBase64 []byte` (`json:"translated"`, `"translatedBase64"`); `model.RawBody.Upstream *string` / `UpstreamBase64 []byte` / `UpstreamTruncated bool` (`json:"upstream"`, `"upstreamBase64"`, `"upstreamTruncated"`). +21. **The hop path (`s.Hop`, praxis) gets no special code.** The request conversion runs before the `hopped` branch in `serveOutbound`, and the reply conversion runs after `hopDo` returns, so both arms convert. When `hopDo` answers its own 503 `gateway_unavailable` (`answered` true), that answer is not converted. No hop test in this plan. +22. **`record.go` does not change.** `recordCapturedRequest` copies the capture struct, so a `Translated` field set before the call is recorded. +23. **Three PRs, stacked.** The total estimate is about 5,000 counted lines, so three PRs is the fewest under the 2,000-line limit. The issue's split (a)/(b) is kept, with (b) cut in two: PR 2 adds the listener layer with its own test plugin; PR 3 adds the config fields with the router that reads them, so no config field merges before its reader. + +## Review Focus + +1. **A chat stream that breaks before `finish_reason`** must reach the agent as an Anthropic `event: error`, not as a short reply that looks complete. Test: T3 `TestReplyDecoder_EndsEarly`. +2. **A gzip-encoded chat reply** must be decoded before conversion. Test: T8 `TestTranslate_GzipUpstream`. +3. **A server that ignores `stream: true` and answers JSON** (or the reverse) must still give the agent the shape it asked for. Test: T8 `TestTranslate_AgentStreamsServerAnswersJSON`. +4. **A non-JSON error page** (an HTML 502 from a gateway) must become an Anthropic `api_error` with the first 512 bytes of the body text as the message. Test: T3 `TestReplyDecoder_HTMLErrorPage`. +5. **Two tool calls in one stream** (index 0, then index 1) must give two `tool_use` blocks in order; a piece for a closed index gives an error event. Tests: T4 golden `stream_two_tools`, T3 `TestReplyDecoder_ClosedIndexPiece`. + +--- + +## File structure + +| Path | Task | Responsibility | +|---|---|---| +| `core/translate/pivot/pivot.go` | T1 | Request, Message, Block, Tool, ToolChoice, Thinking, Usage, Extra | +| `core/translate/pivot/event.go` | T1 | Event, EventKind, DeltaKind | +| `core/translate/pivot/protocol.go` | T1 | Protocol and direction interfaces, RequestError, ErrorType | +| `core/translate/anthropic/{request,reply,error}.go` | T2 | DecodeRequest, NewReplyEncoder, EncodeError | +| `core/translate/chat/{request,reply,error}.go` | T3 | EncodeRequest, NewReplyDecoder, EncodeError | +| `core/translate/{registry,convert,reader}.go` | T4 | Lookup, Names, Target, Request, Reply, ErrorBody, NewReplyReader | +| `core/translate/{golden_test.go,testdata/}` | T4 | Cross-protocol goldens | +| `core/pipeline/destination.go` | T6 | Translate, TranslateTarget, SetUpstreamHeaders, UpstreamHeaders | +| `core/listener/forwardproxy/translate.go` | T8 | Request and reply conversion in the forward proxy | +| `core/model/capture.go`, `core/session/archive/capture.go`, `core/sessionapi/{raw,ingest}.go`, `cmd/agentop/tui/detail_raw.go` | T9 | Rule 8a capture fields | +| `core/plugins/inferencerouter/routerconfig/config.go` | T5 | `api`, `auth_header`, `headers` fields and their checks | +| `core/redact/redact.go` | T5 | Redact the values under a key named exactly `headers` | +| `core/parse/inference/dialect.go`, `core/plugins/inferencerouter/{api.go,plugin.go}` | T7 | ProtocolFor; router rules 4-7, C1-C3 | +| `cmd/agentop/{cmd_server_write.go,cmd_server.go,README.md}` | T10 | Rule C4 flags and display | +| `docs/translation-server.md`, `docs/inference-router-plugin.md`, `docs/README.md` | T11 | F1 and the field reference | + +## PR list + +| PR | Title | Parent | Tasks | estLines | +|---|---|---|---|---| +| P1 | `feat: Add the core/translate hub with Anthropic Messages and chat completions protocols` | "" | T1, T2, T3 | 1700 | +| P2 | `feat: Translate inference requests and replies in the forward proxy` | P1 | T4, T6, T8, T9 | 1900 | +| P3 | `feat: Add the api, auth_header and headers server fields to the inference router` | P2 | T5, T7, T10, T11 | 1500 | + +## Gate + +From the worktree root: + +```sh +(cd core && GOWORK=off go vet ./... && test -z "$(gofmt -l .)" && GOWORK=off go test ./translate/... ./pipeline/... ./redact/... ./parse/... ./plugins/inferencerouter/... ./listener/... ./model/... ./session/... ./sessionapi/... && GOWORK=off go mod tidy -diff) && (cd cmd/cortex && GOWORK=off go vet ./...) && (cd cmd/agentop && GOWORK=off go vet ./... && test -z "$(gofmt -l .)" && GOWORK=off go mod tidy -diff && GOWORK=off go test $(GOWORK=off go list ./... | grep -v 'agentop$')) && scripts/dev/go-test-shards.sh -short cmd/agentop +``` + +On the base commit 8695f5f0d, the format and vet steps of this gate were run one by one from the worktree root (`core` vet and gofmt, `cmd/cortex` vet, `cmd/agentop` vet and gofmt). All passed. The test, tidy and shard steps were not run. + +--- + +## PR P1 + +### Task T1: The pivot type + +**Depends on:** none. **Estimate:** 230 lines. **Rules:** H2, H3, H4. + +**Files:** +- Create: `core/translate/pivot/pivot.go`, `core/translate/pivot/event.go`, `core/translate/pivot/protocol.go`, `core/translate/doc.go` (package comment only for `translate`; T4 adds code) +- Test: `core/translate/pivot/protocol_test.go` + +**Interfaces (Produces):** + +```go +package pivot + +type Extra map[string]json.RawMessage // keys "." + +type BlockKind string +const ( + BlockText BlockKind = "text"; BlockImage = "image"; BlockDocument = "document" + BlockToolUse = "tool_use"; BlockToolResult = "tool_result" + BlockThinking = "thinking"; BlockRedactedThinking = "redacted_thinking" +) + +type Request struct { + Model string + System []Block // text blocks + Messages []Message + Tools []Tool + ToolChoice *ToolChoice + MaxTokens int + Stop []string + Temperature *float64 + TopP *float64 + Stream bool + Effort string // "" when none + Thinking *Thinking // nil when none + Extra Extra +} +type Message struct { Role string; Content []Block; Extra Extra } // Role: user, assistant, system +type Block struct { + Kind BlockKind + Text string // text; thinking text + SourceType string // image/document: "base64" or "url" + MediaType string + Data string // base64 data; redacted_thinking data + URL string + ID, Name string // tool_use + Input json.RawMessage // tool_use input, raw + ToolUseID string // tool_result + Content []Block // tool_result content + IsError bool // tool_result + Signature string // thinking + CacheControl json.RawMessage // nil when none + Extra Extra +} +type Tool struct { Name, Description string; InputSchema json.RawMessage; Type string /* "" for a function tool; else a server tool type */; CacheControl json.RawMessage; Extra Extra } +type ToolChoice struct { Type string /* auto, any, tool, none */; Name string; DisableParallel bool } +type Thinking struct { Type string /* enabled, adaptive, disabled */; BudgetTokens int } +type Usage struct { Input, Output, CacheRead, CacheCreation int } // Anthropic meaning: Input excludes cache tokens + +type EventKind int +const ( EventStart EventKind = iota; EventBlockStart; EventDelta; EventBlockStop; EventStop; EventUsage; EventError ) +type DeltaKind int +const ( DeltaText DeltaKind = iota; DeltaThinking; DeltaSignature; DeltaInputJSON ) +type Event struct { + Kind EventKind + ID, Model string // start + Index int // block_start, delta, block_stop + Block BlockKind // block_start + ToolID, ToolName string // block_start of tool_use + Delta DeltaKind // delta + Value string // delta payload + StopReason string // stop: Anthropic vocabulary (end_turn, max_tokens, tool_use, refusal) + Usage Usage // usage + Status int // error + ErrorType string // error: Anthropic error type vocabulary + Message string // error +} + +type RequestError struct { Status int; Type, Message string } // implements error; a refusal to send to the agent +func (e *RequestError) Error() string + +// ErrorType maps an HTTP status to the Anthropic error type of rule 24: +// 400,422 invalid_request_error; 401 authentication_error; 402,403 permission_error; +// 404 not_found_error; 429 rate_limit_error; 5xx and every other status api_error. +func ErrorType(status int) string + +type Protocol interface { Name() string; EncodeError(status int, errType, message string) []byte } +type RequestDecoder interface { DecodeRequest(body []byte) (Request, error) } +type RequestEncoder interface { EncodeRequest(req Request, h http.Header) (body []byte, dropped []string, err error) } +type ReplyDecoding interface { NewReplyDecoder(status int, stream bool) ReplyDecoder } +type ReplyEncoding interface { NewReplyEncoder(stream bool) ReplyEncoder } +type ReplyDecoder interface { Feed(p []byte) []Event; End() []Event } // keeps state between calls; never holds a whole 2xx stream +type ReplyEncoder interface { Encode(ev Event) []byte; End() []byte } // stream: bytes now; not stream: nil until End +``` + +- [ ] **Step 1:** Write `TestErrorType` (table: 400, 422, 401, 402, 403, 404, 429, 500, 503, 418 → the types above; 418 → `api_error`). Run `cd core && GOWORK=off go test ./translate/pivot/` → FAIL (no package). +- [ ] **Step 2:** Write the three files with the declarations above and doc comments that state the rule each one carries. +- [ ] **Step 3:** Run the test → PASS. `GOWORK=off go vet ./translate/...`. +- [ ] **Step 4:** Commit `feat: Add the translation pivot type`. + +### Task T2: The anthropic protocol + +**Depends on:** T1. **Estimate:** 600 lines. **Rules:** H2 (decoder side), H3, H4, H6, 13, 13a (keeps `system` role), 14-20a (Anthropic side), 23 (frame shapes), 24. + +**Files:** +- Create: `core/translate/anthropic/protocol.go` (`type Protocol struct{}`, `Name()` = `"anthropic-messages"`), `request.go`, `reply.go`, `error.go` +- Test: `core/translate/anthropic/request_test.go`, `reply_test.go` + +**Interfaces:** Consumes T1. Produces `anthropic.Protocol` implementing `pivot.Protocol`, `pivot.RequestDecoder`, `pivot.ReplyEncoding`. + +- `DecodeRequest`: reads `model`, `system` (string → one text block; array → text blocks with `cache_control`), `messages` (string content → one text block), `tools` (`type` set → `Tool.Type`), `tool_choice` (`disable_parallel_tool_use` → `DisableParallel`), `max_tokens`, `stop_sequences`, `temperature`, `top_p`, `stream`, `output_config.effort` → `Effort`, `thinking` → `Thinking`. Every other top-level field goes to `Request.Extra` as `anthropic.`, every unknown message or block field to that `Extra` (same key form), and other `output_config` fields as `anthropic.output_config`. Image: `source.type` `base64` (media_type, data) or `url`. Document: kind `document`, same source fields. Bad JSON → `*pivot.RequestError{400, "invalid_request_error", …}`. +- `NewReplyEncoder(true)`: SSE frames `event: \ndata: \n\n`. `message_start` at `EventStart` with `{"id","type":"message","role":"assistant","model","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}`. `content_block_start` per kind: text `{"type":"text","text":""}`, thinking `{"type":"thinking","thinking":"","signature":""}`, tool_use `{"type":"tool_use","id","name","input":{}}`. Deltas `text_delta`/`thinking_delta`/`signature_delta`/`input_json_delta` (`partial_json`). `EventUsage` is kept; `EventStop` writes `message_delta` `{"type":"message_delta","delta":{"stop_reason":…,"stop_sequence":null},"usage":{"input_tokens","cache_creation_input_tokens","cache_read_input_tokens","output_tokens"}}` then `message_stop`. `EventError` writes `event: error` with `{"type":"error","error":{"type","message"}}`. +- `NewReplyEncoder(false)`: collects events; `End()` writes one message `{"id","type":"message","role":"assistant","model","content":[…],"stop_reason","stop_sequence":null,"usage":{the four fields}}`. Tool input is the joined `input_json_delta` pieces, written as raw JSON (`{}` when empty). An `EventError` makes `End()` write only the error body. +- `EncodeError(status, t, m)` → `{"type":"error","error":{"type":t,"message":m}}`. + +- [ ] **Step 1:** Write `TestDecodeRequest_Fields` (one request with system string, a mid-conversation `system` message, image base64 and url, document, tool_use, tool_result with `is_error`, thinking with signature, redacted_thinking, `cache_control` on system and on a block, `top_k`, `metadata`, a server tool `web_search_20250305`, `output_config.effort: "xhigh"`; assert the pivot fields with `reflect.DeepEqual` on a hand-built `pivot.Request`, and `Extra` keys `anthropic.top_k`, `anthropic.metadata`). Write `TestDecodeRequest_BadJSON` (→ `*pivot.RequestError` status 400). +- [ ] **Step 2:** Write `TestReplyEncoder_Stream` (events for: start, thinking block with text and signature, text block, tool_use block with two JSON pieces, usage, stop `tool_use`; assert the exact SSE bytes as a string literal) and `TestReplyEncoder_NotStream` (same events → exact JSON body) and `TestReplyEncoder_Error` (stream: `event: error` frame; not stream: error body). +- [ ] **Step 3:** Run `cd core && GOWORK=off go test ./translate/anthropic/` → FAIL. +- [ ] **Step 4:** Implement with structs of fixed field order (no `map[string]any` for output). +- [ ] **Step 5:** Run → PASS. Commit `feat: Add the anthropic-messages translation protocol`. + +### Task T3: The chat protocol + +**Depends on:** T1. **Estimate:** 870 lines. **Rules:** 9a, 11-24. + +**Files:** +- Create: `core/translate/chat/protocol.go` (`type Protocol struct{}`, `Name()` = `"chat-completions"`), `request.go`, `reply.go`, `error.go` +- Test: `core/translate/chat/request_test.go`, `reply_test.go` + +**Interfaces:** Consumes T1. Produces `chat.Protocol` implementing `pivot.Protocol`, `pivot.RequestEncoder`, `pivot.ReplyDecoding`. + +- `EncodeRequest(req, h)`: deletes `Anthropic-Version` and `Anthropic-Beta` from `h` (nil `h` is valid). Top-level fields only from rule 11, in that order: `model`, `messages`, `max_tokens`, `stream`, `stream_options`, `tools`, `tool_choice`, `parallel_tool_calls`, `stop`, `temperature`, `top_p`, `reasoning_effort` (all `omitempty` except `model` and `messages`). Rules 12-19 and decisions 8-11 of this plan. Refusals return `*pivot.RequestError{400, "invalid_request_error", msg}`: document → `this inference server cannot read documents (PDF)`; server tool → `this inference server cannot run the server tool "" (type )`. Effort map: `low`, `medium`, `high` stay; any other non-empty value → `high`; else `Thinking.Type` `enabled`/`adaptive` → `medium`; else none. +- `NewReplyDecoder(status, stream)`: + - 2xx, `stream`: SSE line parser that keeps a partial line between `Feed` calls; skips lines that start with `:`; reads `data:` payloads; rule 23 and decisions 5-6. `EventStart` at the first chunk (chunk `id`, `model`). Block index counts from 0 in emit order. `finish_reason` maps by rule 21 (unknown value → `end_turn`) and closes the open block. Usage by rule 22. A chunk `{"error":{…}}` → `EventError` (status 500 when the chunk has none, type from `error.type` when it is an Anthropic type, else `api_error`). `provider_specific_fields` is never read. + - 2xx, not `stream`: buffers the JSON body (a not-streamed body is one value) and at `End()` emits the same events as a stream would, by rule 20; invalid tool `arguments` → only `EventError{502, "api_error", "the inference server sent tool arguments that are not valid JSON"}`. + - non-2xx: buffers at most 1 MiB; `End()` emits one `EventError{status, pivot.ErrorType(status), message}`. Message: `error.message` when the body is JSON with one; else the body text, trimmed, first 512 bytes; else `http.StatusText(status)`. +- `EncodeError` → `{"error":{"message":m,"type":t,"code":null}}`. + +- [ ] **Step 1:** Write `TestEncodeRequest_Shape` (a hand-built `pivot.Request` with one of each block kind except document; assert exact bytes as a literal), `TestEncodeRequest_RefusesDocument`, `TestEncodeRequest_RefusesServerTool`, `TestEncodeRequest_DeletesAnthropicHeaders`, `TestEncodeRequest_Effort` (table: `low`→`low`, `max`→`high`, none + `enabled`→`medium`, none + `disabled`→ field absent). +- [ ] **Step 2:** Write `TestReplyDecoder_BobShape1` (the issue's first stream, as `data:` lines with a `: ping` comment line; assert the event list), `TestReplyDecoder_EndsEarly` (no `finish_reason` → last event is `EventError` 502), `TestReplyDecoder_ClosedIndexPiece`, `TestReplyDecoder_HTMLErrorPage` (status 502, body `bad gateway` → message `bad gateway`, type `api_error`), `TestReplyDecoder_InvalidArgumentsNotStream`. +- [ ] **Step 3:** Run `cd core && GOWORK=off go test ./translate/chat/` → FAIL. Implement. Run → PASS. +- [ ] **Step 4:** Commit `feat: Add the chat-completions translation protocol`. + +--- + +## PR P2 + +### Task T4: Registry, helpers and cross-protocol goldens + +**Depends on:** T2, T3. **Estimate:** 550 lines. **Rules:** H5, H6, 9a, 10, 11-24, and the issue's "Golden request tests" and "Golden reply tests". + +**Files:** +- Create: `core/translate/registry.go`, `convert.go`, `reader.go` +- Test: `core/translate/golden_test.go`, `core/translate/reader_test.go`, `core/translate/testdata/request/*.json` (hand-written Anthropic inputs), `testdata/request/*.golden`, `testdata/reply/*.sse|*.json` (hand-written chat inputs), `testdata/reply/*.golden` + +**Interfaces (Produces):** + +```go +package translate +const ( Anthropic = "anthropic-messages"; Chat = "chat-completions"; Responses = "responses" ) +func Lookup(name string) (pivot.Protocol, bool) // compiled-in map; responses is an unexported type with only Name and EncodeError (OpenAI error shape {"error":{"message","type","code":null}}) +func Names() []string // sorted +func Decodes(name string) bool // has pivot.RequestDecoder +func Target(served []string) (string, bool) // decision 12 +type Converted struct { Body []byte; Dropped []string; Stream bool } // Stream: the agent asked for a stream +func Request(source, target string, body []byte, h http.Header) (Converted, error) // *pivot.RequestError for a refusal +func Reply(source, target string, status int, upstreamStream, agentStream bool, body []byte) []byte // whole-body conversion +func ErrorBody(protocol string, status int, errType, message string) []byte +func NewReplyReader(body io.ReadCloser, dec pivot.ReplyDecoder, enc pivot.ReplyEncoder) io.ReadCloser +``` + +`NewReplyReader`: each `Read` reads at most 32 KiB from `body`, feeds it, encodes each event, and returns the bytes; it returns as soon as it has bytes (never waits for the whole stream). At upstream EOF it calls `dec.End()` and `enc.End()`. `Close` closes `body`. An upstream read error other than EOF is returned after the bytes already converted. + +- [ ] **Step 1:** Write `TestNames` (`[anthropic-messages chat-completions responses]`), `TestTarget` (`[responses]` → false; `[responses chat-completions]` → `chat-completions`; `[anthropic-messages responses]` → false), `TestDecodes`. +- [ ] **Step 2:** Write the request goldens: one input per rule 11, 12, 13, 13a, 14 (each bullet, with a `tool_result` that holds an image), 15, 16, 17, 18, 19 and 20a (`call_3833__thought__AY89a1+8zd/x=` id in a `tool_use` and its `tool_result`), and `full_multiturn.json` (text, image, tool_use, tool_result, thinking with signature, `cache_control` on system and a message, a mid-conversation `system` message). The test runs `Request(Anthropic, Chat, in, nil)` and compares the body to `.golden` byte for byte, and `Dropped` to the golden's first line. With `-update` it writes the golden. +- [ ] **Step 3:** Write `TestDeterministic` (rule 9a): encode `full_multiturn.json` twice → equal bytes; append one user turn → old output is a byte prefix up to the end of the old last message; move `cache_control` from the last message to the new message → the two outputs differ only in the marker bytes (compare after removing `,"cache_control":{"type":"ephemeral"}` from both). +- [ ] **Step 4:** Write the reply goldens for the issue's three Bob shapes, `stream_two_tools`, and errors (403 LiteLLM `team_model_access_denied`, 404 Bob `not_found_error`, 429, an in-stream `{"error":…}` chunk), each streamed and not streamed: `Reply(Anthropic, Chat, …)` vs golden. Shape 3: `output_tokens` 296 and no thinking block. Shape 2: a thinking block with empty text and its `signature_delta`. +- [ ] **Step 5:** Write `TestReplyReader_EveryByteSplit`: for each streamed reply input, feed it through `NewReplyReader` over a reader that returns one byte per `Read`; the output equals the golden. Write `TestReplyReader_Incremental`: an `io.Pipe` upstream; after writing the first chunk only, a `Read` returns the `message_start` frame. +- [ ] **Step 6:** Run `cd core && GOWORK=off go test ./translate/... -update`, read every golden against the issue rules, then run without `-update` → PASS. Commit `feat: Add the translation registry and golden tests`. + +### Task T6: The pipeline calls + +**Depends on:** none. **Estimate:** 250 lines. **Rules:** 5, C3 (apply-after-plugins half). + +**Files:** +- Modify: `core/pipeline/destination.go` (after `RedirectPath`), `core/pipeline/context.go` (private fields beside `redirectPath`; reset them wherever `redirectPath` is reset) +- Test: `core/pipeline/translate_test.go` + +**Interfaces (Produces):** + +```go +// Translate asks the listener to convert this redirected request from source to +// target and send it to path, with no query string. It does not change Path. +// Gates: as SetRedirectPath (OnRequest only, WritesDestination, a Redirect took +// effect, path absolute with no query, fragment or ".." segment), plus source and +// target non-empty and different. Records nothing: the listener records +// modify/translated. A second call replaces the first. +func (c *Context) Translate(source, target, path string) error +func (c *Context) TranslateTarget() (source, target, path string, ok bool) // ok only when Redirected and Translate was accepted +// SetUpstreamHeaders gives headers that the listener sets on the upstream request +// after every plugin has run, replacing a header of the same name. Same gates as +// SetRedirectPath without the path. Keeps a clone. +func (c *Context) SetUpstreamHeaders(h http.Header) error +func (c *Context) UpstreamHeaders() http.Header // nil unless Redirected +``` + +- [ ] **Step 1:** Write tests, following the setup of the existing `SetRedirectPath` tests in `core/pipeline/resend_test.go`: accepted after a Redirect (Path unchanged, `TranslateTarget` ok); refused before a Redirect, without WritesDestination, outside OnRequest, under observe (no Redirect took effect), with `path` `v1/x`, `/a?b`, `/a/../b`, with `source == target`; `SetUpstreamHeaders` stores a clone (a later change to the caller's map does not show). +- [ ] **Step 2:** Run `cd core && GOWORK=off go test ./pipeline/ -run 'Translate|UpstreamHeaders'` → FAIL. Implement. Run → PASS. Commit `feat: Add pipeline calls for translation and fixed upstream headers`. + +### Task T8: The forward-proxy layer + +**Depends on:** T4, T6. **Estimate:** 660 lines. **Rules:** 5, 8, 9, 10, C3 (apply), decisions 4, 17, 18, 19. + +**Files:** +- Create: `core/listener/forwardproxy/translate.go` +- Modify: `core/listener/forwardproxy/server.go` (`serveOutbound`, `resendIfAsked`) +- Test: `core/listener/forwardproxy/translate_test.go` + +**Behavior (in `serveOutbound`, in this order):** +1. After the redirect-apply block and the `Reject` return: if `pctx.TranslateTarget()` is ok, call `translate.Request(src, tgt, pctx.Body, pctx.Headers)`. `pctx.Body == nil` → treat as a 502 `api_error` "the request body was not buffered, so Cortex cannot translate it". A `*pivot.RequestError` or any other error (502 `api_error`) → a `pipeline.Reject` action with `Violation{Code: "inference.translation_refused", Status, Body: translate.ErrorBody(src, …), BodyType: "application/json"}`, recorded and written the way the existing reject branch does (`recordOutboundRejectIn` + `httpx.WriteRejectionForRequest`); nothing goes upstream. On success record `pipeline.Invocation{Plugin: "forward-proxy", Action: ActionModify, Reason: "translated", Details: {"from": src, "to": tgt, "path": path}}`, plus `"dropped": strings.Join(dropped, ",")` when not empty. Keep `Converted` in a local. +2. After `httpx.SyncRequestHeaders`: set each `pctx.UpstreamHeaders()` header on `r.Header` (replace). This applies to translated and not-translated requests. +3. After the `BodyMutated` block, for a translated request: `httpx.SetRequestBody(r, conv.Body)`; `r.URL.Path = path`; `r.URL.RawPath, r.URL.RawQuery, r.URL.ForceQuery = "", "", false`. `pctx.Path` is not changed. +4. `ownEncoding` is true for a translated request, so the transport decodes gzip. +5. In `resendIfAsked`, for a translated request: re-run `translate.Request` on `pctx.Body` and send `conv.Body` (on error, return the current answer unchanged); re-apply upstream headers after `SyncRequestHeaders`; the refused attempt row's `pctx.ResponseBody` is `translate.Reply(...)` of `head` (decision 19). Pass the translation through a small struct parameter or a field on a per-request value, not through `pctx`. +6. After the resend loop and the error branch, before `pctx.StatusCode = resp.StatusCode`, for a translated request: `dec := target.NewReplyDecoder(resp.StatusCode, httpx.IsEventStream(ct))`, `enc := source.NewReplyEncoder(conv.Stream && resp.StatusCode/100 == 2)`, `resp.Body = translate.NewReplyReader(resp.Body, dec, enc)`; set `Content-Type` to `text/event-stream` (stream) or `application/json`; delete `Content-Length` and `Content-Encoding`; `resp.ContentLength = -1`. Every later arm (buffered, streaming responders, relay) then reads Anthropic bytes. + +**Tests** (use the setup of `redirect_test.go`: a test plugin with `WritesDestination` that calls `Redirect` to an `httptest` server, then `Translate("anthropic-messages", "chat-completions", "/v1/chat/completions")`, and optionally `SetUpstreamHeaders`; include the inference-parser plugin where a test checks the session store): +- `TestTranslate_StreamedReplyRecordedAndIncremental`: upstream streams the issue's shape 1 and blocks after the first two chunks; the client reads `message_start` before the upstream finishes; after the end, the session store's response event has the Anthropic inference (text `391 is not prime`, tool `Bash`), and the request event has a `modify/translated` invocation with `from`, `to`, `path`. +- `TestTranslate_NoQueryString`: client `POST /v1/messages?beta=true` → upstream sees path `/v1/chat/completions` and empty `RawQuery`; the recorded request row path is `/v1/messages`. +- `TestTranslate_UpstreamHeadersReplace`: client sends `User-Agent: claude-cli/2`; `SetUpstreamHeaders({"User-Agent": ["bob-shell"]})` → upstream sees only `bob-shell`. +- `TestTranslate_GzipUpstream`: upstream gzips when the request has `Accept-Encoding: gzip`; client gets valid Anthropic JSON. +- `TestTranslate_AgentStreamsServerAnswersJSON`: agent `stream: true`, upstream answers `application/json` → client gets SSE. +- `TestTranslate_ErrorStatus`: upstream 429 → client 429 `rate_limit_error` body. +- `TestTranslate_DocumentRefused`: a request with a document → client 400 `invalid_request_error`; the upstream got no request. +- `TestTranslate_DroppedRecorded`: `top_k` and `metadata` → `dropped` `metadata,top_k`. +- `TestTranslate_ResendSeesRawReply`: a test `Resender` records the head it got; upstream answers 403 LiteLLM `team_model_access_denied` first, then 200; the Resender saw `team_model_access_denied`; the second upstream body is chat-shaped. +- `TestTranslate_NotTranslatedPassesBytes`: Redirect only, no Translate → upstream body equals the client body byte for byte. + +- [ ] **Step 1:** Write the tests. Run `cd core && GOWORK=off go test ./listener/forwardproxy/ -run TestTranslate` → FAIL. +- [ ] **Step 2:** Implement `translate.go` helpers and the `server.go` edits. Run → PASS. Run the whole `./listener/...` → PASS. Commit `feat: Translate requests and replies in the forward proxy`. + +### Task T9: Raw capture of the translated hop + +**Depends on:** T8. **Estimate:** 400 lines. **Rules:** 8a, decision 20. + +**Files:** +- Modify: `core/model/capture.go` (fields of decision 20), `core/listener/forwardproxy/server.go` and `translate.go` (fill them), `core/session/archive/capture.go` (write and read the new bodies the way `Forwarded` and the response `Body` are written, with `splitBody`/`rebuild`), `core/session/archive/archive.go` (`captureSize`: count the new bodies in the size), `core/sessionapi/raw.go` (`textOr64` for both), `core/sessionapi/ingest.go` (accept `translated` and the response `upstream` and carry them to `model.RequestCapture`), `cmd/agentop/tui/detail_raw.go` (labels below) +- Test: `core/listener/forwardproxy/translate_test.go` (`TestTranslate_CaptureHoldsFourBodies`), `core/session/archive/capture_test.go`, `core/sessionapi/archive_test.go` (beside `TestHandleGetEvent_RawAddsTheCapture`), `core/sessionapi/ingest_test.go`, `cmd/agentop/tui/detail_raw_test.go` + +Behavior: `capture.Translated = conv.Body` before the request row is recorded. For the reply, wrap the upstream body **before** `NewReplyReader` with `bodycount.WrapCapture(body, &unused, &rc.Upstream, &rc.UpstreamTruncated, &ended, pipeline.MaxResponseCapture)`; the present `rc.Body` stays the converted reply as the agent got it. agentop labels: `translated` → `sent upstream, translated by Cortex: ` with body key `translatedBody`; `upstream` → `server's reply before translation: ` with body key `upstreamBody`. The `forwarded` label ("rewritten by a plugin") does not change. + +- [ ] **Step 1:** Write `TestTranslate_CaptureHoldsFourBodies` (a plugin that changes the body, so `Forwarded` is set; assert `Original`, `Forwarded`, `Translated` chat body, `Response.Body` Anthropic, `Response.Upstream` chat SSE), an archive round-trip test with both new fields, a raw GET test (`translated` text; a non-UTF-8 `upstream` → `upstreamBase64`), and an agentop render test for both labels. +- [ ] **Step 2:** Run the four tests → FAIL. Implement. Run `cd core && GOWORK=off go test ./listener/... ./session/... ./sessionapi/... ./model/...` and `cd cmd/agentop && GOWORK=off go test ./tui/` → PASS. Commit `feat: Capture the translated request and the raw server reply`. + +--- + +## PR P3 + +### Task T5: Server config fields and header redaction + +**Depends on:** none. **Estimate:** 450 lines. **Rules:** C1 (validation), C2 (values), C3 (validation, redaction), C2a. + +**Files:** +- Modify: `core/plugins/inferencerouter/routerconfig/config.go`, `core/redact/redact.go` +- Test: `core/plugins/inferencerouter/routerconfig/config_test.go`, `core/redact/redact_test.go` + +**Interfaces (Produces):** + +```go +// in routerconfig.Server, after Helper: +API map[string]string `json:"api" description:"Protocols the server serves, by protocol name, to the path it serves each at. Empty: requests go in the agent's protocol, to the agent's path."` +AuthHeader string `json:"auth_header" description:"Header that carries key. authorization sends Bearer ; any other name sends the raw key. Empty: the agent's header, or the target protocol's usual one for a translated request."` +Headers map[string]string `json:"headers" description:"Fixed headers set on every request to this server after the plugins run. Values may use ${ENV_VAR}."` + +var APIProtocols = []string{"anthropic-messages", "chat-completions", "responses"} +func CheckAPI(api map[string]string) error // keys sorted for a stable first error +func CheckAuthHeader(name string) error +func CheckHeaders(h map[string]string, authHeader string) error +``` + +Errors (prefix `servers..` added by `Validate`): +- unknown key: `unknown protocol ""; valid protocols: anthropic-messages, chat-completions, responses` +- bad path: `api.: path "

" must be absolute, with no query, fragment or ".." segment` +- `auth_header`: must be a valid header token; refused names (case-insensitive): `Host`, `Content-Length`, and the hop-by-hop set `Connection`, `Keep-Alive`, `Proxy-Connection`, `Transfer-Encoding`, `Upgrade`, `TE`, `Trailer`, `Proxy-Authenticate`, `Proxy-Authorization`. +- `headers`: refused names = the `auth_header` refused set plus `Authorization`, `X-Api-Key`, `Api-Key`, and the server's own `auth_header`; a value with CR, LF or NUL is refused; names must be valid tokens. Error: `headers: "" cannot be set here: ` (reason "Cortex sets it" for auth headers, "it is a hop-by-hop or framing header" otherwise). + +`redact.redactMap`: when a key equals `headers` exactly (case-sensitive) and the value is an object, replace each value in it with `"[REDACTED]"` and keep the names. Do not add `headers` to `sensitiveKeys`. + +- [ ] **Step 1:** Write tests: `api` valid (`{chat-completions: /v1/chat/completions}`, DeepSeek's two entries, `{responses: /v1/responses}`), unknown key error lists the valid keys, relative path refused, `?` refused; `auth_header` `authorization`, `x-api-key`, `api-key` valid, `Host` and `Connection` refused; `headers` `User-Agent: bob-shell` valid, `Authorization`, `X-Api-Key`, `Content-Length`, `Transfer-Encoding` refused, a value with `\n` refused; a strict decode of a config with all three fields succeeds. Redact, through `redact.JSON` (the function that `/config` in `core/observe/statserver.go` and `/v1/pipeline` in `core/sessionapi/server.go` call): `{"servers":{"bob":{"headers":{"User-Agent":"bob-shell"}}},"session":{"id_headers":["X-Session"]}}` → `User-Agent` value `[REDACTED]`, `id_headers` unchanged; `delegated_credential_headers` and `remove_headers` unchanged. +- [ ] **Step 2:** Run `cd core && GOWORK=off go test ./plugins/inferencerouter/routerconfig/ ./redact/` → FAIL. Implement (the path rule is the same as `SetRedirectPath`'s). Run → PASS. Commit `feat: Add the api, auth_header and headers server fields`. + +### Task T7: Router rules + +**Depends on:** T4, T5, T6, T8. **Estimate:** 600 lines. **Rules:** 4, 4a, 5, 6, 7, 9, C1 (routing), C2, C3 (handing over), decisions 12-15. + +**Files:** +- Modify: `core/parse/inference/dialect.go` (add `ProtocolFor`), `core/plugins/inferencerouter/plugin.go` (`route` gets `api map[string]string`, `authHeader string`, `headers http.Header`; `Configure` fills them; `OnRequest` uses the plan; `setKey` call replaced by `setAuth`) +- Create: `core/plugins/inferencerouter/api.go` +- Test: `core/parse/inference/dialect_test.go`, `core/plugins/inferencerouter/api_test.go`, `core/listener/forwardproxy/inferencerouter_test.go` (end-to-end cases) + +**Interfaces:** + +```go +// core/parse/inference +func ProtocolFor(path string) string // decision 14 + +// core/plugins/inferencerouter/api.go +type planKind int +const ( planAsIs planKind = iota; planPath; planTranslate; planRefuse; planCountTokens ) +type apiPlan struct { kind planKind; source, target, path string } +func (r route) plan(path string) apiPlan +func setAuth(pctx *pipeline.Context, r route, translatedTo string) // translatedTo "" when not translated +``` + +`plan` rules, in this order: (1) empty `api` → `planAsIs`; (2) path ends in `/v1/messages/count_tokens` and `api` has no `anthropic-messages` → `planCountTokens`; (3) `src := ProtocolFor(path)`; `api[src]` set → `planPath` with that path; (4) `translate.Decodes(src)` → `translate.Target(keys)`: ok → `planTranslate{src, tgt, api[tgt]}`, else `planRefuse{src}`; (5) else `planAsIs`. + +`OnRequest`, once `redirected` is true: compute the plan before `chooseModel`. `planPath` → `SetRedirectPath(plan.path)` in place of `SetRedirectPath(captured)`. `planTranslate` → `pctx.Translate(...)`, and no `SetRedirectPath`. `planRefuse` → record `pipeline.Invocation{Action: ActionDeny, Reason: "cannot_translate", Details: details()+{"from": src}}` and return `pipeline.Action{Type: pipeline.Reject, Violation: &pipeline.Violation{Code: "inference.translation_unavailable", Reason: msg, Status: 502, Body: translate.ErrorBody(src, 502, "api_error", msg), BodyType: "application/json"}}` with the decision 13 message. `planCountTokens` → record `ActionDeny` reason `count_tokens_local`, reject with status 404 and body `translate.ErrorBody("anthropic-messages", 404, "not_found_error", "this inference server does not count tokens")`. An error from `Translate` or `SetRedirectPath` takes the existing `redirect_failed` branch. Then `setAuth`: `authHeader` set → delete `Authorization`, `X-Api-Key`, `Api-Key`, then `authorization` → `Authorization: Bearer `, any other name → that header = key. `authHeader` empty and translated → delete the three, then `chat-completions` → `Authorization: Bearer `, `anthropic-messages` → `X-Api-Key: `. Else `setKey` as today. Last: when `len(headers) > 0`, `pctx.SetUpstreamHeaders(headers)`. + +- [ ] **Step 1:** Write `TestProtocolFor` (`/v1/messages` → anthropic-messages, `/anthropic/v1/messages` → anthropic-messages, `/v1/chat/completions` → chat-completions, `/v1/completions` → "", `/v1/responses` → responses, `/v1/messages/count_tokens` → ""). +- [ ] **Step 2:** Write `TestAPIProtocolsMatchRegistry` (`routerconfig.APIProtocols` equals `translate.Names()`) and `TestPlan` table: empty map; `{chat-completions}` + `/v1/messages` → translate to `/v1/chat/completions`; `{anthropic-messages: /anthropic/v1/messages}` + `/v1/messages` → path; `{responses}` + `/v1/messages` → refuse; `{anthropic-messages, responses}` + `/v1/chat/completions` → as is (rule 7); `{chat-completions}` + `/v1/messages/count_tokens` → count tokens; `{anthropic-messages}` + `/v1/messages/count_tokens` → as is. +- [ ] **Step 3:** Write plugin tests in `api_test.go` with the existing `plugin_test.go` setup: `setAuth` cases (rule C2 table: `X-Api-Key` agent + empty `auth_header` + not translated → `setKey` behavior unchanged; translated to chat → only `Authorization: Bearer`; `auth_header: x-api-key` → only `X-Api-Key`; `auth_header: api-key` → only `Api-Key`); translated request leaves `pctx.Path` unchanged and `RedirectPath` not ok; `headers` reach `UpstreamHeaders`. +- [ ] **Step 4:** Write end-to-end tests in `core/listener/forwardproxy/inferencerouter_test.go` (its existing router setup): Claude Code `X-Api-Key` request to a `{chat-completions}` server → upstream has only `Authorization: Bearer ` and a chat body; `{responses: /v1/responses}` → client 502 Anthropic `api_error` with the decision 13 text, upstream got nothing; `{anthropic-messages: /anthropic/v1/messages}` → upstream path `/anthropic/v1/messages`, body unchanged; a chat-completions request to a `{anthropic-messages: …}` server → not translated, no 502; `count_tokens` routed to a chat server → client 404 `not_found_error`, upstream got nothing; translated request + LiteLLM 403 `team_model_access_denied` → the router resends with its substitute model; Bob 404 `model "claude-opus-5-5" not found or not accessible` → resend as today. +- [ ] **Step 5:** Run `cd core && GOWORK=off go test ./parse/... ./plugins/inferencerouter/... ./listener/forwardproxy/` → FAIL. Implement. Run → PASS. Commit `feat: Route requests by the server api map and translate them`. + +### Task T10: agentop flags and display + +**Depends on:** T5. **Estimate:** 340 lines. **Rules:** C4. + +**Files:** +- Modify: `cmd/agentop/cmd_server_write.go` (`serverAdd`), `cmd/agentop/cmd_server.go` (`serverList`), `cmd/agentop/README.md` +- Test: `cmd/agentop/cmd_server_write_test.go`, `cmd/agentop/cmd_server_test.go` + +Flags on `server add`: `--api =` (repeatable, a `flag.Value` that fills a map; a second value for the same protocol replaces the first), `--auth-header `, `--header =` (repeatable). Check them with `routerconfig.CheckAPI`, `CheckAuthHeader`, `CheckHeaders` before the key is read, like `checkWordFlags`. `--header` with a `$` in the value is refused with: `--header : the value contains $, which Cortex reads as an environment variable when it loads the config; write headers: {: ${NAME}} by hand`. A flag value with no `=` is refused. Write the fields into the server. `agentop server` shows, for each server that has them, lines `api: chat-completions=/v1/chat/completions` (sorted by protocol), `auth header: x-api-key`, `headers: User-Agent` (names only, sorted). The README lists the three flags. + +- [ ] **Step 1:** Write tests: `--api chat-completions=/v1/chat/completions --auth-header x-api-key --header User-Agent=bob-shell` writes the three fields; `--api bogus=/x` fails with exit 2 and the valid-keys text, and reads no key; `--header X-A=$HOME` fails; `--header Authorization=x` fails; `server` list shows the names and not the value `bob-shell`. +- [ ] **Step 2:** Run `scripts/dev/go-test-shards.sh -short cmd/agentop` → FAIL. Implement. Run → PASS. Commit `feat: Add --api, --auth-header and --header to agentop server add`. + +### Task T11: Docs + +**Depends on:** none. **Estimate:** 120 lines. **Rules:** F1, C1-C3. + +**Files:** +- Create: `docs/translation-server.md` +- Modify: `docs/inference-router-plugin.md` (a section "Servers that serve another protocol"), `docs/README.md` (index row for the new page) + +Facts for `docs/translation-server.md`: +- When: Cortex translates only Anthropic Messages to chat completions. For any other pair, Cortex answers 502 "Cortex cannot translate to yet". Run a translation server on loopback for that pair. +- LiteLLM config that worked (2026-11-16): + ```yaml + model_list: + - model_name: + litellm_params: + model: hosted_vllm/ + api_base: + api_key: os.environ/UPSTREAM_API_KEY + additional_drop_params: ["prompt_cache_key"] + general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + ``` + Use `hosted_vllm/`, not `openai/`: LiteLLM's `openai/` provider sent requests to `/v1/responses`, which gave 403. Run `litellm --config litellm.yaml --host 127.0.0.1 --port 4000`. Bind to loopback only and always set a master key. +- Add it: `agentop server add --main --helper --api anthropic-messages=/v1/messages http://127.0.0.1:4000`, and give the master key as the server key (prompt or `--key-stdin`). With `anthropic-messages` in `api`, Cortex sends Claude Code's requests to LiteLLM untranslated. +- Cortex records only the agent's side of the hop to the translation server. The translation server's own cost and errors are its own, and Cortex does not see them. + +Facts for `docs/inference-router-plugin.md`: the three fields, their descriptions from T5, the valid `api` keys, the DeepSeek example `{chat-completions: /chat/completions, anthropic-messages: /anthropic/v1/messages}`, rules 4 (when Cortex translates), 4a (502), 6 (`count_tokens` 404), 7, C2's header choice table, C3's refused names and that `/config` and `/v1/pipeline` hide the header values, C2a (the model-list fetch still sends Bearer only), and the new `modify/translated` record with `from`, `to`, `path`, `dropped`. Link `translation-server.md`. + +- [ ] **Step 1:** Write the pages. Run `pre-commit run --files docs/translation-server.md docs/inference-router-plugin.md docs/README.md` → PASS. Commit `docs: Add the translation server page and the router's api fields`. + +--- + +## For a human after review + +These steps need a live agent, a real server or a person's decision. They are not tasks in this workflow. + +1. **Needs T1-T11 (all PRs).** Run Claude Code (`claude -p`, with tools) against a chat-completions server through the router for at least 3 tool-call turns. A local Ollama or llama-server works when its `api` map has only `chat-completions` (`agentop server add --api chat-completions=/v1/chat/completions …`). Put the session id and its turn count in the PR P3 body. +2. **Needs T7.** Check that Claude Code's `/context` still works with the local 404 for `count_tokens` (rule 6). If it does not, report what Claude Code does. Do not guess a token count. +3. **Needs T3, T8 (and #1912 for Bob).** Check that Claude Code accepts a `__thought__` tool id with `+` or `/` and sends it back unchanged. If Claude Code refuses or changes it, report what it does. Do not encode the id without a decision on that report. +4. **Needs T3.** On Bob (#1912's list), check the rule 11 fields not yet tested there (`tool_choice`, `parallel_tool_calls`, `stop`, `temperature`, `top_p`), and that Bob accepts `role: tool` messages whose `content` is a list of text parts (decision 8). +5. **Needs P1-P3 opened.** Post nothing outside the repository except the PRs themselves; issue comments on #1911 or #1910 are a person's call.