diff --git a/internal/runner/client.go b/internal/runner/client.go index 99dffe231..fb3cf7bfe 100644 --- a/internal/runner/client.go +++ b/internal/runner/client.go @@ -52,6 +52,15 @@ func (c *Client) sessionToken() string { // ---------- request plumbing ---------- +type runnerResponseError struct { + statusCode int + message string +} + +func (e runnerResponseError) Error() string { + return e.message +} + func (c *Client) do(ctx context.Context, method, path string, body any, out any) error { var reader io.Reader if body != nil { @@ -90,7 +99,10 @@ func (c *Client) do(ctx context.Context, method, path string, body any, out any) if resp.StatusCode == http.StatusConflict && json.Unmarshal(payload, &detail) == nil && detail.Error == ErrTaskLeaseLost.Error() { return ErrTaskLeaseLost } - return fmt.Errorf("%s %s: %s", method, path, strings.TrimSpace(string(payload))) + return runnerResponseError{ + statusCode: resp.StatusCode, + message: fmt.Sprintf("%s %s: %s", method, path, strings.TrimSpace(string(payload))), + } } if out != nil && len(payload) > 0 { if err := json.Unmarshal(payload, out); err != nil { @@ -296,7 +308,22 @@ type CompleteRequest struct { } func (c *Client) Complete(ctx context.Context, taskID string, req CompleteRequest) error { - return c.do(ctx, http.MethodPost, "/runners/tasks/"+taskID+"/complete", req, nil) + for attempt := 1; ; attempt++ { + err := c.do(ctx, http.MethodPost, "/runners/tasks/"+taskID+"/complete", req, nil) + var transport *url.Error + var response runnerResponseError + retryable := errors.As(err, &transport) || (errors.As(err, &response) && response.statusCode >= 500) + if !retryable || attempt == 3 { + return err + } + // Retry the report, not the work. If the server already committed + // the result, it rejects the retry because the lease is gone. + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Duration(attempt) * 200 * time.Millisecond): + } + } } func (c *Client) UploadPreview(ctx context.Context, previewID, token string, body io.Reader) error { diff --git a/internal/runner/completion_retry_test.go b/internal/runner/completion_retry_test.go new file mode 100644 index 000000000..2c5f66256 --- /dev/null +++ b/internal/runner/completion_retry_test.go @@ -0,0 +1,49 @@ +package runner + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "agentflow/internal/domain" +) + +func TestCompletionRetriesTemporaryServerFailure(t *testing.T) { + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req CompleteRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Status != domain.RunnerTaskSuccess || req.Result != "finished" { + http.Error(w, "invalid report", http.StatusBadRequest) + return + } + if attempts.Add(1) == 1 { + http.Error(w, "database restarting", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(server.Close) + err := NewClient(server.URL).Complete(context.Background(), "task", CompleteRequest{Status: domain.RunnerTaskSuccess, Result: "finished"}) + if err != nil || attempts.Load() != 2 { + t.Fatalf("completion attempts=%d, error=%v, want retry and success", attempts.Load(), err) + } +} + +func TestCompletionDoesNotRetryLostLease(t *testing.T) { + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts.Add(1) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusConflict) + _, _ = w.Write([]byte(`{"error":"task lease lost"}`)) + })) + t.Cleanup(server.Close) + err := NewClient(server.URL).Complete(context.Background(), "task", CompleteRequest{Status: domain.RunnerTaskSuccess}) + if !errors.Is(err, ErrTaskLeaseLost) || attempts.Load() != 1 { + t.Fatalf("completion attempts=%d, error=%v, want a single loss", attempts.Load(), err) + } +}