From be29ccd215982ad891d3e5cac14e78b2208047ff Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Sun, 15 Nov 2026 23:59:32 -0500 Subject: [PATCH 1/2] fix(runner): split event uploads before control-plane limits --- internal/domain/runner.go | 6 +++ internal/httpapi/runners.go | 4 +- internal/runner/client.go | 34 ++++++++++++- internal/runner/event_batch_limit_test.go | 62 +++++++++++++++++++++++ 4 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 internal/runner/event_batch_limit_test.go diff --git a/internal/domain/runner.go b/internal/domain/runner.go index 02718a08c..eeebd4176 100644 --- a/internal/domain/runner.go +++ b/internal/domain/runner.go @@ -625,6 +625,12 @@ type RunnerWorkBranchHead struct { SharedHead string `json:"sharedHead,omitempty"` } +// Runner task event upload limits shared by the worker and the control plane. +const ( + RunnerTaskEventBatchLimit = 500 + RunnerTaskEventBodyLimit = 2 << 20 +) + // RunnerTaskEvent is one batched output record reported by the runner. Seq is // assigned by the runner and used with the task id for idempotency, so an // HTTP retry never duplicates log data. diff --git a/internal/httpapi/runners.go b/internal/httpapi/runners.go index 10e7e0d65..99f964c42 100644 --- a/internal/httpapi/runners.go +++ b/internal/httpapi/runners.go @@ -1074,14 +1074,14 @@ func (h *runnersHandlers) runnerTaskEventsIngest(c *gin.Context) { var input struct { Events []domain.RunnerTaskEvent `json:"events"` } - if !decode(c, &input) { + if !decodeLimit(c, &input, domain.RunnerTaskEventBodyLimit) { return } if len(input.Events) == 0 { c.JSON(http.StatusOK, map[string]int{"accepted": 0}) return } - if len(input.Events) > 500 { + if len(input.Events) > domain.RunnerTaskEventBatchLimit { c.JSON(http.StatusBadRequest, map[string]string{"error": "单批事件数量不能超过 500"}) return } diff --git a/internal/runner/client.go b/internal/runner/client.go index 99dffe231..d123db9dc 100644 --- a/internal/runner/client.go +++ b/internal/runner/client.go @@ -283,7 +283,39 @@ func (c *Client) OpenControl(ctx context.Context) (*websocket.Conn, error) { } func (c *Client) SendEvents(ctx context.Context, taskID string, events []domain.RunnerTaskEvent) error { - return c.do(ctx, http.MethodPost, "/runners/tasks/"+taskID+"/events", map[string]any{"events": events}, nil) + const envelopeBytes = len(`{"events":[]}`) + path := "/runners/tasks/" + taskID + "/events" + batch := make([]json.RawMessage, 0, min(len(events), domain.RunnerTaskEventBatchLimit)) + used := envelopeBytes + flush := func() error { + err := c.do(ctx, http.MethodPost, path, map[string]any{"events": batch}, nil) + batch = batch[:0] + used = envelopeBytes + return err + } + for _, event := range events { + raw, err := json.Marshal(event) + if err != nil { + return err + } + // Include space for a separator, using the encoded size so + // control characters and HTML escaping can't exceed the budget. + size := len(raw) + 1 + if envelopeBytes+size > domain.RunnerTaskEventBodyLimit { + return fmt.Errorf("runner event %d exceeds the upload size limit", event.Seq) + } + if len(batch) == domain.RunnerTaskEventBatchLimit || used+size > domain.RunnerTaskEventBodyLimit { + if err := flush(); err != nil { + return err + } + } + batch = append(batch, raw) + used += size + } + if len(batch) == 0 { + return nil + } + return flush() } type CompleteRequest struct { diff --git a/internal/runner/event_batch_limit_test.go b/internal/runner/event_batch_limit_test.go new file mode 100644 index 000000000..a1f1ab486 --- /dev/null +++ b/internal/runner/event_batch_limit_test.go @@ -0,0 +1,62 @@ +package runner + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync" + "testing" + + "agentflow/internal/domain" +) + +func TestSendEventsSplitsRequestsAtControlPlaneLimits(t *testing.T) { + var mu sync.Mutex + var got []int64 + var requests int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body struct { + Events []domain.RunnerTaskEvent `json:"events"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 2<<20)).Decode(&body); err != nil { + http.Error(w, "request too large or invalid", http.StatusBadRequest) + return + } + if len(body.Events) > 500 { + http.Error(w, "too many events", http.StatusBadRequest) + return + } + mu.Lock() + defer mu.Unlock() + requests++ + for _, event := range body.Events { + got = append(got, event.Seq) + } + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(server.Close) + + var events []domain.RunnerTaskEvent + var want []int64 + for i := range 520 { + number := int64(i + 1) + data := "small log line" + if i >= 510 { + // HTML characters expand to six bytes each in the JSON body. + data = strings.Repeat("<", 100_000) + } + events = append(events, domain.RunnerTaskEvent{Seq: number, Type: "stdout", Data: data}) + want = append(want, number) + } + if err := NewClient(server.URL).SendEvents(context.Background(), "task", events); err != nil { + t.Fatal(err) + } + mu.Lock() + defer mu.Unlock() + if requests < 3 || !slices.Equal(got, want) { + t.Fatalf("sent %d requests with %d events, want all %d events in order", requests, len(got), len(want)) + } +} From e1305af064b5a63f704098c414e642a152b2a9ec Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Thu, 19 Nov 2026 21:39:08 -0500 Subject: [PATCH 2/2] fix(runner): bound oversized events without blocking log uploads --- internal/runner/client.go | 53 +++++++++- internal/runner/event_batch_limit_test.go | 112 ++++++++++++++++++++++ 2 files changed, 164 insertions(+), 1 deletion(-) diff --git a/internal/runner/client.go b/internal/runner/client.go index d123db9dc..cc765ed3d 100644 --- a/internal/runner/client.go +++ b/internal/runner/client.go @@ -12,6 +12,7 @@ import ( "strings" "sync" "time" + "unicode/utf8" "agentflow/internal/domain" @@ -302,7 +303,11 @@ func (c *Client) SendEvents(ctx context.Context, taskID string, events []domain. // control characters and HTML escaping can't exceed the budget. size := len(raw) + 1 if envelopeBytes+size > domain.RunnerTaskEventBodyLimit { - return fmt.Errorf("runner event %d exceeds the upload size limit", event.Seq) + raw, err = marshalTruncatedRunnerEvent(event) + if err != nil { + return err + } + size = len(raw) + 1 } if len(batch) == domain.RunnerTaskEventBatchLimit || used+size > domain.RunnerTaskEventBodyLimit { if err := flush(); err != nil { @@ -357,3 +362,49 @@ func (c *Client) UploadPreview(ctx context.Context, previewID, token string, bod func (c *Client) CompletePreview(ctx context.Context, previewID string) error { return c.do(ctx, http.MethodPost, "/runners/previews/"+previewID+"/complete", map[string]any{}, nil) } + +// An oversized record must not permanently block the batcher's retry queue. +// Keep one bounded record with the original sequence for task/sequence deduplication. +func marshalTruncatedRunnerEvent(event domain.RunnerTaskEvent) ([]byte, error) { + data := truncateRunnerOutput(event.Data) + if event.Type == "observation" { + // Data is a nested TelemetryEvent, not plain log text. Slicing its JSON + // would make the engine silently discard the observation. Replace it + // with an explicit warning that the ordinary observation consumer accepts. + observation := domain.TelemetryEvent{ + Timestamp: event.Ts, Category: "process", Name: "process.output_truncated", + Phase: "progress", Level: "warning", Message: data, + Source: domain.TelemetrySource{Layer: "executor", Adapter: "runner-v1"}, + } + raw, err := json.Marshal(observation) + if err != nil { + return nil, err + } + data = string(raw) + } else if len(event.Type) > 256 { + // Real producers use fixed event kinds. Keep even malformed metadata + // from turning a record into an upload that can never succeed. + event.Type = "system" + } + event.Data = data + return json.Marshal(event) +} + +func truncateRunnerOutput(data string) string { + // This budget leaves ample room for both JSON escaping layers used by + // observations, even if every retained byte needs an escape sequence. + const retainedBytes = 64 << 10 + const marker = "\n[runner output truncated]\n" + if len(data) <= retainedBytes { + return data + marker + } + head := retainedBytes / 2 + tail := len(data) - retainedBytes/2 + for head > 0 && !utf8.RuneStart(data[head]) { + head-- + } + for tail < len(data) && !utf8.RuneStart(data[tail]) { + tail++ + } + return data[:head] + marker + data[tail:] +} diff --git a/internal/runner/event_batch_limit_test.go b/internal/runner/event_batch_limit_test.go index a1f1ab486..13ef4e4f9 100644 --- a/internal/runner/event_batch_limit_test.go +++ b/internal/runner/event_batch_limit_test.go @@ -3,12 +3,15 @@ package runner import ( "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "slices" "strings" "sync" "testing" + "time" + "unicode/utf8" "agentflow/internal/domain" ) @@ -60,3 +63,112 @@ func TestSendEventsSplitsRequestsAtControlPlaneLimits(t *testing.T) { t.Fatalf("sent %d requests with %d events, want all %d events in order", requests, len(got), len(want)) } } + +func TestSendEventsTruncatesOversizedRecordWithoutBlockingFollowingLogs(t *testing.T) { + for _, data := range []string{ + strings.Repeat("plain output ", (3<<20)/13), + strings.Repeat("<\"\n", 250_000), + strings.Repeat("你好🌍", 300_000), + } { + t.Run(fmt.Sprintf("bytes-%d", len(data)), func(t *testing.T) { + var got []domain.RunnerTaskEvent + var mu sync.Mutex + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body struct { + Events []domain.RunnerTaskEvent `json:"events"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, domain.RunnerTaskEventBodyLimit)).Decode(&body); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if len(body.Events) > domain.RunnerTaskEventBatchLimit { + t.Error("batch limit exceeded") + } + mu.Lock() + got = append(got, body.Events...) + mu.Unlock() + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(server.Close) + events := []domain.RunnerTaskEvent{ + {Seq: 1, Type: "stdout", Data: "before"}, + {Seq: 2, Type: "stderr", Data: "HEAD-" + data + "-TAIL", Ts: time.Unix(1234, 0).UTC()}, + {Seq: 3, Type: "stdout", Data: "after"}, + } + original := slices.Clone(events) + raw, err := json.Marshal(events[1]) + if err != nil || len(raw) <= domain.RunnerTaskEventBodyLimit { + t.Fatal("fixture must exceed body limit") + } + if err := NewClient(server.URL).SendEvents(context.Background(), "task", events); err != nil { + t.Fatal(err) + } + mu.Lock() + defer mu.Unlock() + if len(got) != 3 { + t.Fatalf("got %d events, want 3", len(got)) + } + if got[0] != events[0] || got[2] != events[2] { + t.Fatal("neighboring events changed") + } + for i, event := range got { + if event.Seq != int64(i+1) { + t.Fatal("sequence order changed") + } + } + if got[1].Type != events[1].Type || !got[1].Ts.Equal(events[1].Ts) { + t.Fatal("event metadata changed") + } + if !strings.HasPrefix(got[1].Data, "HEAD-") || !strings.HasSuffix(got[1].Data, "-TAIL") || !strings.Contains(got[1].Data, "[runner output truncated]") { + t.Fatal("missing head, tail or truncation marker") + } + if !utf8.ValidString(got[1].Data) { + t.Fatal("truncation split a UTF-8 character") + } + if !slices.Equal(events, original) { + t.Fatal("caller events mutated") + } + }) + } +} + +func TestSendEventsTruncatesOversizedObservationAsValidTelemetry(t *testing.T) { + original := domain.TelemetryEvent{Category: "tool", Name: "tool.result", Message: "HEAD-" + strings.Repeat("<", 500_000) + "-TAIL"} + payload, err := json.Marshal(original) + if err != nil { + t.Fatal(err) + } + var received []domain.RunnerTaskEvent + var mu sync.Mutex + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body struct { + Events []domain.RunnerTaskEvent `json:"events"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, domain.RunnerTaskEventBodyLimit)).Decode(&body); err != nil { + http.Error(w, err.Error(), 400) + return + } + mu.Lock() + received = append(received, body.Events...) + mu.Unlock() + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(server.Close) + events := []domain.RunnerTaskEvent{{Seq: 1, Type: "observation", Data: string(payload), Ts: time.Unix(1234, 0).UTC()}, {Seq: 2, Type: "stdout", Data: "following log"}} + if err := NewClient(server.URL).SendEvents(context.Background(), "task", events); err != nil { + t.Fatal(err) + } + mu.Lock() + defer mu.Unlock() + if len(received) != 2 || received[0].Seq != 1 || received[0].Type != "observation" || received[1] != events[1] { + t.Fatal("event order or metadata changed") + } + // This is the same typed decode required by runner_dispatch.go before sink(). + var observation domain.TelemetryEvent + if err := json.Unmarshal([]byte(received[0].Data), &observation); err != nil { + t.Fatal(err) + } + if observation.Category != "process" || observation.Name != "process.output_truncated" || observation.Level != "warning" || !strings.Contains(observation.Message, "[runner output truncated]") || !strings.Contains(observation.Message, "HEAD-") || !strings.Contains(observation.Message, "-TAIL") { + t.Fatal("missing bounded warning representation") + } +}