Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions internal/domain/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -670,6 +670,12 @@ type RunnerWorkBranchHead struct {
SharedHead string `json:"sharedHead,omitempty"`
}

// Runner task event upload limits shared by the worker and the control plane.
const (
RunnerTaskEventBatchLimit = 500
RunnerTaskEventBodyLimit = 2 << 20
)

// RunnerTaskEvent is one batched output record reported by the runner. Seq is
// assigned by the runner and used with the task id for idempotency, so an
// HTTP retry never duplicates log data.
Expand Down
4 changes: 2 additions & 2 deletions internal/httpapi/runners.go
Original file line number Diff line number Diff line change
Expand Up @@ -1080,14 +1080,14 @@ func (h *runnersHandlers) runnerTaskEventsIngest(c *gin.Context) {
var input struct {
Events []domain.RunnerTaskEvent `json:"events"`
}
if !decode(c, &input) {
if !decodeLimit(c, &input, domain.RunnerTaskEventBodyLimit) {
return
}
if len(input.Events) == 0 {
c.JSON(http.StatusOK, map[string]int{"accepted": 0})
return
}
if len(input.Events) > 500 {
if len(input.Events) > domain.RunnerTaskEventBatchLimit {
c.JSON(http.StatusBadRequest, map[string]string{"error": "单批事件数量不能超过 500"})
return
}
Expand Down
85 changes: 84 additions & 1 deletion internal/runner/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"strings"
"sync"
"time"
"unicode/utf8"

"agentflow/internal/domain"

Expand Down Expand Up @@ -283,7 +284,43 @@ func (c *Client) OpenControl(ctx context.Context) (*websocket.Conn, error) {
}

func (c *Client) SendEvents(ctx context.Context, taskID string, events []domain.RunnerTaskEvent) error {
return c.do(ctx, http.MethodPost, "/runners/tasks/"+taskID+"/events", map[string]any{"events": events}, nil)
const envelopeBytes = len(`{"events":[]}`)
path := "/runners/tasks/" + taskID + "/events"
batch := make([]json.RawMessage, 0, min(len(events), domain.RunnerTaskEventBatchLimit))
used := envelopeBytes
flush := func() error {
err := c.do(ctx, http.MethodPost, path, map[string]any{"events": batch}, nil)
batch = batch[:0]
used = envelopeBytes
return err
}
for _, event := range events {
raw, err := json.Marshal(event)
if err != nil {
return err
}
// Include space for a separator, using the encoded size so
// control characters and HTML escaping can't exceed the budget.
size := len(raw) + 1
if envelopeBytes+size > domain.RunnerTaskEventBodyLimit {
raw, err = marshalTruncatedRunnerEvent(event)
if err != nil {
return err
}
size = len(raw) + 1
}
if len(batch) == domain.RunnerTaskEventBatchLimit || used+size > domain.RunnerTaskEventBodyLimit {
if err := flush(); err != nil {
return err
}
}
batch = append(batch, raw)
used += size
}
if len(batch) == 0 {
return nil
}
return flush()
}

type CompleteRequest struct {
Expand Down Expand Up @@ -325,3 +362,49 @@ func (c *Client) UploadPreview(ctx context.Context, previewID, token string, bod
func (c *Client) CompletePreview(ctx context.Context, previewID string) error {
return c.do(ctx, http.MethodPost, "/runners/previews/"+previewID+"/complete", map[string]any{}, nil)
}

// An oversized record must not permanently block the batcher's retry queue.
// Keep one bounded record with the original sequence for task/sequence deduplication.
func marshalTruncatedRunnerEvent(event domain.RunnerTaskEvent) ([]byte, error) {
data := truncateRunnerOutput(event.Data)
if event.Type == "observation" {
// Data is a nested TelemetryEvent, not plain log text. Slicing its JSON
// would make the engine silently discard the observation. Replace it
// with an explicit warning that the ordinary observation consumer accepts.
observation := domain.TelemetryEvent{
Timestamp: event.Ts, Category: "process", Name: "process.output_truncated",
Phase: "progress", Level: "warning", Message: data,
Source: domain.TelemetrySource{Layer: "executor", Adapter: "runner-v1"},
}
raw, err := json.Marshal(observation)
if err != nil {
return nil, err
}
data = string(raw)
} else if len(event.Type) > 256 {
// Real producers use fixed event kinds. Keep even malformed metadata
// from turning a record into an upload that can never succeed.
event.Type = "system"
}
event.Data = data
return json.Marshal(event)
}

func truncateRunnerOutput(data string) string {
// This budget leaves ample room for both JSON escaping layers used by
// observations, even if every retained byte needs an escape sequence.
const retainedBytes = 64 << 10
const marker = "\n[runner output truncated]\n"
if len(data) <= retainedBytes {
return data + marker
}
head := retainedBytes / 2
tail := len(data) - retainedBytes/2
for head > 0 && !utf8.RuneStart(data[head]) {
head--
}
for tail < len(data) && !utf8.RuneStart(data[tail]) {
tail++
}
return data[:head] + marker + data[tail:]
}
174 changes: 174 additions & 0 deletions internal/runner/event_batch_limit_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
package runner

import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"slices"
"strings"
"sync"
"testing"
"time"
"unicode/utf8"

"agentflow/internal/domain"
)

func TestSendEventsSplitsRequestsAtControlPlaneLimits(t *testing.T) {
var mu sync.Mutex
var got []int64
var requests int
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var body struct {
Events []domain.RunnerTaskEvent `json:"events"`
}
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 2<<20)).Decode(&body); err != nil {
http.Error(w, "request too large or invalid", http.StatusBadRequest)
return
}
if len(body.Events) > 500 {
http.Error(w, "too many events", http.StatusBadRequest)
return
}
mu.Lock()
defer mu.Unlock()
requests++
for _, event := range body.Events {
got = append(got, event.Seq)
}
w.WriteHeader(http.StatusNoContent)
}))
t.Cleanup(server.Close)

var events []domain.RunnerTaskEvent
var want []int64
for i := range 520 {
number := int64(i + 1)
data := "small log line"
if i >= 510 {
// HTML characters expand to six bytes each in the JSON body.
data = strings.Repeat("<", 100_000)
}
events = append(events, domain.RunnerTaskEvent{Seq: number, Type: "stdout", Data: data})
want = append(want, number)
}
if err := NewClient(server.URL).SendEvents(context.Background(), "task", events); err != nil {
t.Fatal(err)
}
mu.Lock()
defer mu.Unlock()
if requests < 3 || !slices.Equal(got, want) {
t.Fatalf("sent %d requests with %d events, want all %d events in order", requests, len(got), len(want))
}
}

func TestSendEventsTruncatesOversizedRecordWithoutBlockingFollowingLogs(t *testing.T) {
for _, data := range []string{
strings.Repeat("plain output ", (3<<20)/13),
strings.Repeat("<\"\n", 250_000),
strings.Repeat("你好🌍", 300_000),
} {
t.Run(fmt.Sprintf("bytes-%d", len(data)), func(t *testing.T) {
var got []domain.RunnerTaskEvent
var mu sync.Mutex
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var body struct {
Events []domain.RunnerTaskEvent `json:"events"`
}
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, domain.RunnerTaskEventBodyLimit)).Decode(&body); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if len(body.Events) > domain.RunnerTaskEventBatchLimit {
t.Error("batch limit exceeded")
}
mu.Lock()
got = append(got, body.Events...)
mu.Unlock()
w.WriteHeader(http.StatusNoContent)
}))
t.Cleanup(server.Close)
events := []domain.RunnerTaskEvent{
{Seq: 1, Type: "stdout", Data: "before"},
{Seq: 2, Type: "stderr", Data: "HEAD-" + data + "-TAIL", Ts: time.Unix(1234, 0).UTC()},
{Seq: 3, Type: "stdout", Data: "after"},
}
original := slices.Clone(events)
raw, err := json.Marshal(events[1])
if err != nil || len(raw) <= domain.RunnerTaskEventBodyLimit {
t.Fatal("fixture must exceed body limit")
}
if err := NewClient(server.URL).SendEvents(context.Background(), "task", events); err != nil {
t.Fatal(err)
}
mu.Lock()
defer mu.Unlock()
if len(got) != 3 {
t.Fatalf("got %d events, want 3", len(got))
}
if got[0] != events[0] || got[2] != events[2] {
t.Fatal("neighboring events changed")
}
for i, event := range got {
if event.Seq != int64(i+1) {
t.Fatal("sequence order changed")
}
}
if got[1].Type != events[1].Type || !got[1].Ts.Equal(events[1].Ts) {
t.Fatal("event metadata changed")
}
if !strings.HasPrefix(got[1].Data, "HEAD-") || !strings.HasSuffix(got[1].Data, "-TAIL") || !strings.Contains(got[1].Data, "[runner output truncated]") {
t.Fatal("missing head, tail or truncation marker")
}
if !utf8.ValidString(got[1].Data) {
t.Fatal("truncation split a UTF-8 character")
}
if !slices.Equal(events, original) {
t.Fatal("caller events mutated")
}
})
}
}

func TestSendEventsTruncatesOversizedObservationAsValidTelemetry(t *testing.T) {
original := domain.TelemetryEvent{Category: "tool", Name: "tool.result", Message: "HEAD-" + strings.Repeat("<", 500_000) + "-TAIL"}
payload, err := json.Marshal(original)
if err != nil {
t.Fatal(err)
}
var received []domain.RunnerTaskEvent
var mu sync.Mutex
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var body struct {
Events []domain.RunnerTaskEvent `json:"events"`
}
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, domain.RunnerTaskEventBodyLimit)).Decode(&body); err != nil {
http.Error(w, err.Error(), 400)
return
}
mu.Lock()
received = append(received, body.Events...)
mu.Unlock()
w.WriteHeader(http.StatusNoContent)
}))
t.Cleanup(server.Close)
events := []domain.RunnerTaskEvent{{Seq: 1, Type: "observation", Data: string(payload), Ts: time.Unix(1234, 0).UTC()}, {Seq: 2, Type: "stdout", Data: "following log"}}
if err := NewClient(server.URL).SendEvents(context.Background(), "task", events); err != nil {
t.Fatal(err)
}
mu.Lock()
defer mu.Unlock()
if len(received) != 2 || received[0].Seq != 1 || received[0].Type != "observation" || received[1] != events[1] {
t.Fatal("event order or metadata changed")
}
// This is the same typed decode required by runner_dispatch.go before sink().
var observation domain.TelemetryEvent
if err := json.Unmarshal([]byte(received[0].Data), &observation); err != nil {
t.Fatal(err)
}
if observation.Category != "process" || observation.Name != "process.output_truncated" || observation.Level != "warning" || !strings.Contains(observation.Message, "[runner output truncated]") || !strings.Contains(observation.Message, "HEAD-") || !strings.Contains(observation.Message, "-TAIL") {
t.Fatal("missing bounded warning representation")
}
}