Repository navigation
fix(adk): fail the task when the model stops at max_tokens instead of completing with a cut answer #2896
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
fix(adk): fail the task when the model stops at max_tokens instead of completing with a cut answer #2896
Changes from all commits
6c11bd3
3f0b9a2
c14d593
1df12c6
c2e2288
0616846
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -99,6 +99,7 @@ func NewKAgentExecutor(cfg KAgentExecutorConfig) (*KAgentExecutor, error) { | |
| A2APartConverter: a2aPartConverter, | ||
| GenAIPartConverter: structuredOutputPartConverter(output, rootName), | ||
| AfterEventCallback: func(ctx adka2a.ExecutorContext, event *adksession.Event, processed *a2atype.TaskArtifactUpdateEvent) error { | ||
| outputLimitWatchFrom(ctx).observe(event) | ||
| // Preserve the artifact's protocol type and assign kagent's ordering key. | ||
| if processed.Artifact != nil { | ||
| position := event.Timestamp | ||
|
|
@@ -113,7 +114,11 @@ func NewKAgentExecutor(cfg KAgentExecutorConfig) (*KAgentExecutor, error) { | |
| // input-required or failed), whose metadata a2a-go merges into the | ||
| // stored task. | ||
| AfterExecuteCallback: func(ctx adka2a.ExecutorContext, finalEvent *a2atype.TaskStatusUpdateEvent, _ error) error { | ||
| outputLimitWatchFrom(ctx).failTruncatedAnswer(finalEvent) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. remote tool reads artifacts first, does parent see this?
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. you are right, it did not. I pushed a new commit, a failed task now yields the status message as error and the artifacts as result, so the parent reads both. |
||
| turnUsageFrom(ctx).stampEvent(finalEvent) | ||
| // Never return the execution error: ADK drops the terminal status | ||
| // event and reports the error instead, which would lose the failure | ||
| // it is already carrying. | ||
| return nil | ||
| }, | ||
| OutputMode: adka2a.OutputArtifactPerEvent, | ||
|
|
@@ -267,6 +272,7 @@ func (e *KAgentExecutor) Execute(ctx context.Context, reqCtx *a2asrv.ExecutorCon | |
| // The synthetic ADK user ID is only a native session lookup key. Memory | ||
| // and outgoing credentials must receive only the passed-through caller. | ||
| ctx = auth.WithUserID(ctx, trustedUserID) | ||
| ctx = withOutputLimitWatch(ctx) | ||
| // Resumed tasks (HITL cycles, follow-up messages) carry the previously | ||
| // persisted total, so the usage total stays a task-lifetime sum. | ||
| usage := &turnUsage{} | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| package a2a | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
|
|
||
| a2atype "github.com/a2aproject/a2a-go/v2/a2a" | ||
| "google.golang.org/adk/v2/server/adka2a/v2" | ||
| adksession "google.golang.org/adk/v2/session" | ||
| "google.golang.org/genai" | ||
| ) | ||
|
|
||
| // isErrorMessageMetadataKey marks a status message as carrying an error. It is | ||
| // the key ADK sets on the failures it reports itself, so a client renders this | ||
| // one the same way. | ||
| var isErrorMessageMetadataKey = adka2a.ToA2AMetaKey("is_error_message") | ||
|
|
||
| // outputLimitWatch remembers how the turn's last model response ended. | ||
| // | ||
| // A model that stops at its output limit returns an answer cut mid-sentence, | ||
| // and the task must not report that as completed: an A2A client — a person's, | ||
| // or a parent agent that calls this one through the remote agent tool — has no | ||
| // other way to tell a whole answer from a truncated one. The finish reason is | ||
| // the provider-neutral signal; every model adapter maps its own stop reason | ||
| // onto it. | ||
| type outputLimitWatch struct { | ||
| reason genai.FinishReason | ||
| // generated is the output the response was billed for. When the model | ||
| // stopped at the limit, that is the limit. Every adapter in pkg/models | ||
| // reports the total billed output in CandidatesTokenCount, reasoning | ||
| // included; ThoughtsTokenCount, where an adapter sets it, is a breakdown of | ||
| // that total and must not be added to it. | ||
| generated int32 | ||
| } | ||
|
|
||
| type outputLimitWatchKey struct{} | ||
|
|
||
| // withOutputLimitWatch gives one execution its own watch. The callbacks below | ||
| // are configured once per executor and shared by every concurrent execution, | ||
| // so the state they read and write has to travel in the request's context. | ||
| func withOutputLimitWatch(ctx context.Context) context.Context { | ||
| return context.WithValue(ctx, outputLimitWatchKey{}, &outputLimitWatch{}) | ||
| } | ||
|
|
||
| func outputLimitWatchFrom(ctx context.Context) *outputLimitWatch { | ||
| watch, _ := ctx.Value(outputLimitWatchKey{}).(*outputLimitWatch) | ||
| return watch | ||
| } | ||
|
|
||
| // observe records how a model response ended. Streaming chunks and the events | ||
| // tools produce carry no finish reason, so only a completed model response | ||
| // moves the watch, and the last one of the turn is the one that decides. | ||
| func (w *outputLimitWatch) observe(event *adksession.Event) { | ||
| if w == nil || event == nil || event.Partial { | ||
| return | ||
| } | ||
| if event.FinishReason == "" || event.FinishReason == genai.FinishReasonUnspecified { | ||
| return | ||
| } | ||
| w.reason = event.FinishReason | ||
| w.generated = 0 | ||
| if usage := event.UsageMetadata; usage != nil { | ||
| w.generated = usage.CandidatesTokenCount | ||
| } | ||
| } | ||
|
|
||
| // failTruncatedAnswer turns the turn's completed terminal status into a failure | ||
| // when the last model response stopped at the output limit. The artifacts | ||
| // already streamed are left untouched, so the caller keeps the partial answer | ||
| // and reads the failure beside it. | ||
| // | ||
| // A status that is not completed is left alone: ADK is already reporting a | ||
| // failure of its own — a broken function call the truncation caused, say — or a | ||
| // pause waiting for input, and overwriting it would replace the cause with a | ||
| // symptom. | ||
| func (w *outputLimitWatch) failTruncatedAnswer(final *a2atype.TaskStatusUpdateEvent) { | ||
| if w == nil || final == nil || w.reason != genai.FinishReasonMaxTokens { | ||
| return | ||
| } | ||
| if final.Status.State != a2atype.TaskStateCompleted { | ||
| return | ||
| } | ||
| part := a2atype.NewTextPart(outputLimitMessage(w.generated)) | ||
| part.Metadata = map[string]any{isErrorMessageMetadataKey: true} | ||
| final.Status.State = a2atype.TaskStateFailed | ||
| final.Status.Message = a2atype.NewMessageForTask(a2atype.MessageRoleAgent, final, part) | ||
| } | ||
|
|
||
| // outputLimitMessage names the limit when it is knowable. The response does not | ||
| // carry the request's max_tokens, but a model that stopped at the limit | ||
| // generated exactly that many tokens, so usage names it; without usage the | ||
| // number is left out rather than guessed. | ||
| func outputLimitMessage(generated int32) string { | ||
| if generated <= 0 { | ||
| return "model output limit reached: the answer is incomplete" | ||
| } | ||
| return fmt.Sprintf("model output limit reached (max_tokens=%d): the answer is incomplete", generated) | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,163 @@ | ||
| package a2a | ||
|
|
||
| import ( | ||
| "context" | ||
| "iter" | ||
| "log/slog" | ||
| "strings" | ||
| "testing" | ||
|
|
||
| a2atype "github.com/a2aproject/a2a-go/v2/a2a" | ||
| "github.com/a2aproject/a2a-go/v2/a2asrv" | ||
| adkagent "google.golang.org/adk/v2/agent" | ||
| "google.golang.org/adk/v2/model" | ||
| "google.golang.org/adk/v2/runner" | ||
| adksession "google.golang.org/adk/v2/session" | ||
| "google.golang.org/genai" | ||
| ) | ||
|
|
||
| // runTurn runs one turn of an agent that yields the given responses and returns | ||
| // the artifact updates and the terminal task status the executor produced. | ||
| func runTurn(t *testing.T, responses []model.LLMResponse) ([]*a2atype.TaskArtifactUpdateEvent, *a2atype.TaskStatusUpdateEvent) { | ||
| t.Helper() | ||
| const appName = "output-limit-app" | ||
|
|
||
| agent, err := adkagent.New(adkagent.Config{ | ||
| Name: "output-limit-agent", | ||
| Run: func(ic adkagent.InvocationContext) iter.Seq2[*adksession.Event, error] { | ||
| return func(yield func(*adksession.Event, error) bool) { | ||
| for _, response := range responses { | ||
| event := &adksession.Event{ | ||
| Author: ic.Agent().Name(), | ||
| InvocationID: ic.InvocationID(), | ||
| Branch: ic.Branch(), | ||
| LLMResponse: response, | ||
| } | ||
| if !yield(event, nil) { | ||
| return | ||
| } | ||
| } | ||
| } | ||
| }, | ||
| }) | ||
| if err != nil { | ||
| t.Fatalf("agent.New() error = %v", err) | ||
| } | ||
|
|
||
| executor, err := NewKAgentExecutor(KAgentExecutorConfig{ | ||
| AppName: appName, | ||
| SessionService: adksession.InMemoryService(), | ||
| Logger: slog.New(slog.DiscardHandler), | ||
| RunnerConfig: runner.Config{ | ||
| AppName: appName, | ||
| Agent: agent, | ||
| }, | ||
| }) | ||
| if err != nil { | ||
| t.Fatalf("NewKAgentExecutor() error = %v", err) | ||
| } | ||
| reqCtx := &a2asrv.ExecutorContext{ | ||
| TaskID: "task-1", | ||
| ContextID: "context-1", | ||
| Message: a2atype.NewMessage(a2atype.MessageRoleUser, a2atype.NewTextPart("hi")), | ||
| } | ||
|
|
||
| var updates []*a2atype.TaskArtifactUpdateEvent | ||
| var terminal *a2atype.TaskStatusUpdateEvent | ||
| for event, err := range executor.Execute(context.Background(), reqCtx) { | ||
| if err != nil { | ||
| t.Fatalf("Execute() error = %v", err) | ||
| } | ||
| switch event := event.(type) { | ||
| case *a2atype.TaskArtifactUpdateEvent: | ||
| updates = append(updates, event) | ||
| case *a2atype.TaskStatusUpdateEvent: | ||
| if event.Status.State.Terminal() { | ||
| terminal = event | ||
| } | ||
| } | ||
| } | ||
| if terminal == nil { | ||
| t.Fatal("Execute() produced no terminal task status") | ||
| } | ||
| return updates, terminal | ||
| } | ||
|
|
||
| // statusText returns the text of a terminal status message. | ||
| func statusText(t *testing.T, status *a2atype.TaskStatusUpdateEvent) string { | ||
| t.Helper() | ||
| if status.Status.Message == nil || len(status.Status.Message.Parts) == 0 { | ||
| t.Fatalf("terminal status = %#v, want a status message", status) | ||
| } | ||
| return status.Status.Message.Parts[0].Text() | ||
| } | ||
|
|
||
| func TestKAgentExecutorFailsTheTaskWhenTheModelStopsAtTheOutputLimit(t *testing.T) { | ||
| updates, terminal := runTurn(t, []model.LLMResponse{ | ||
| { | ||
| Content: genai.NewContentFromText("the answer starts", genai.RoleModel), | ||
| Partial: true, | ||
| }, | ||
| { | ||
| Content: genai.NewContentFromText("the answer starts and is cut", genai.RoleModel), | ||
| FinishReason: genai.FinishReasonMaxTokens, | ||
| // The reasoning tokens are a breakdown of the billed output, so the | ||
| // limit named is 8192 and not 14192. | ||
| UsageMetadata: &genai.GenerateContentResponseUsageMetadata{ | ||
| CandidatesTokenCount: 8192, | ||
| ThoughtsTokenCount: 6000, | ||
| }, | ||
| }, | ||
| }) | ||
|
|
||
| if terminal.Status.State != a2atype.TaskStateFailed { | ||
| t.Fatalf("terminal state = %q, want %q", terminal.Status.State, a2atype.TaskStateFailed) | ||
| } | ||
| text := statusText(t, terminal) | ||
| if !strings.Contains(text, "output limit") || !strings.Contains(text, "max_tokens=8192") { | ||
| t.Fatalf("status message = %q, want the output limit and the limit value", text) | ||
| } | ||
| if len(updates) == 0 { | ||
| t.Fatal("artifact updates = 0, want the partial answer kept") | ||
| } | ||
| last := updates[len(updates)-1] | ||
| if len(last.Artifact.Parts) != 1 || last.Artifact.Parts[0].Text() != "the answer starts and is cut" { | ||
| t.Fatalf("last artifact = %#v, want the streamed partial answer", last.Artifact) | ||
| } | ||
| } | ||
|
|
||
| func TestKAgentExecutorOmitsTheLimitWhenUsageDoesNotNameIt(t *testing.T) { | ||
| _, terminal := runTurn(t, []model.LLMResponse{{ | ||
| Content: genai.NewContentFromText("cut", genai.RoleModel), | ||
| FinishReason: genai.FinishReasonMaxTokens, | ||
| }}) | ||
|
|
||
| if terminal.Status.State != a2atype.TaskStateFailed { | ||
| t.Fatalf("terminal state = %q, want %q", terminal.Status.State, a2atype.TaskStateFailed) | ||
| } | ||
| text := statusText(t, terminal) | ||
| if !strings.Contains(text, "output limit") || strings.Contains(text, "max_tokens=") { | ||
| t.Fatalf("status message = %q, want the output limit without a number", text) | ||
| } | ||
| } | ||
|
|
||
| func TestKAgentExecutorCompletesTheTaskWhenTheLastResponseStopsNormally(t *testing.T) { | ||
| _, terminal := runTurn(t, []model.LLMResponse{ | ||
| { | ||
| Content: genai.NewContentFromText("a chunk", genai.RoleModel), | ||
| Partial: true, | ||
| FinishReason: genai.FinishReasonMaxTokens, | ||
| }, | ||
| { | ||
| Content: genai.NewContentFromText("the whole answer", genai.RoleModel), | ||
| FinishReason: genai.FinishReasonStop, | ||
| }, | ||
| }) | ||
|
|
||
| if terminal.Status.State != a2atype.TaskStateCompleted { | ||
| t.Fatalf("terminal state = %q, want %q", terminal.Status.State, a2atype.TaskStateCompleted) | ||
| } | ||
| if terminal.Status.Message != nil { | ||
| t.Fatalf("completed status message = %#v, want none", terminal.Status.Message) | ||
| } | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
#2890 also rewrites this callback, conflicts on merge
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thanks. #2890 has merged, so I merged main into this branch (c14d593): the observe call sits at the top of the AfterEventCallback that now ends in transformStructuredOutput, the new AfterExecuteCallback is unchanged, and the test follows the new NewKAgentExecutor error return. The two paths do not overlap: a structured-output turn that stops at max_tokens already fails inside transformStructuredOutput, and failTruncatedAnswer leaves a non-completed terminal status alone, so there is no double report. build, vet, the package tests (incl. TestKAgentExecutorFailsCompletedStructuredOutputWithoutResult) and make lint pass on the merged branch.