From 7c8c51dfa5a77b3e8704163f8cefef3151087063 Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Sun, 15 Nov 2026 23:55:22 -0500 Subject: [PATCH] fix(runner): preserve database errors during lease checks --- internal/repo/runner_lease_read_test.go | 63 +++++++++++++++++++++++++ internal/repo/runners.go | 29 ++++++------ 2 files changed, 78 insertions(+), 14 deletions(-) create mode 100644 internal/repo/runner_lease_read_test.go diff --git a/internal/repo/runner_lease_read_test.go b/internal/repo/runner_lease_read_test.go new file mode 100644 index 000000000..13391e6bf --- /dev/null +++ b/internal/repo/runner_lease_read_test.go @@ -0,0 +1,63 @@ +package repo + +import ( + "errors" + "testing" + "time" + + "agentflow/internal/domain" + + "gorm.io/gorm" +) + +func TestRunnerLeaseOperationsDistinguishDatabaseFailureFromLostLease(t *testing.T) { + st, err := OpenTest() + if err != nil { + t.Fatal(err) + } + t.Cleanup(st.Close) + now := time.Now().UTC() + if err := st.PutRunnerTask(domain.RunnerTask{ID: "runner-read", Provider: "mock"}); err != nil { + t.Fatal(err) + } + task, found, err := st.ClaimRunnerTask("runner-owner", []string{"mock"}, now, time.Minute) + if err != nil || !found { + t.Fatalf("claim found=%t, error=%v", found, err) + } + + readError := errors.New("transient lease read failure") + failRead := true + if err := st.DB.Callback().Query().Before("gorm:query").Register("test_runner_read_failure", func(tx *gorm.DB) { + if failRead && tx.Statement.Table == "runner_tasks" { + _ = tx.AddError(readError) + } + }); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + run func() error + }{ + {"start", func() error { return st.StartRunnerTask(task.ID, "runner-owner", now) }}, + {"renew", func() error { + _, err := st.RenewRunnerTaskLease(task.ID, "runner-owner", time.Minute, now) + return err + }}, + {"complete", func() error { + return st.CompleteRunnerTask(task.ID, "runner-owner", domain.RunnerTaskSuccess, nil, "", "", nil, nil, now) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + if err := tc.run(); !errors.Is(err, readError) { + t.Errorf("operation error = %v, want original database failure", err) + } + }) + } + failRead = false + if _, err := st.RenewRunnerTaskLease(task.ID, "wrong-owner", time.Minute, now); !errors.Is(err, ErrRunnerTaskLeaseLost) { + t.Fatalf("wrong owner error = %v, want lost lease", err) + } + if _, err := st.RenewRunnerTaskLease(task.ID, "runner-owner", time.Minute, now); err != nil { + t.Fatalf("owner could not renew after recovery: %v", err) + } +} diff --git a/internal/repo/runners.go b/internal/repo/runners.go index e6b65bdb1..20e27f86a 100644 --- a/internal/repo/runners.go +++ b/internal/repo/runners.go @@ -583,9 +583,9 @@ func (r *RunnersRepo) ClaimRunnerTask(runnerID string, providers []string, now t // StartRunnerTask moves a leased task into running. func (r *RunnersRepo) StartRunnerTask(id, runnerID string, now time.Time) error { return r.DB.Transaction(func(tx *gorm.DB) error { - row, ok := r.takeRunnerTaskForLease(tx, id, runnerID) - if !ok { - return ErrRunnerTaskLeaseLost + row, err := r.takeRunnerTaskForLease(tx, id, runnerID) + if err != nil { + return err } task := runnerTaskFromRow(row) task.Status = domain.RunnerTaskRunning @@ -601,14 +601,15 @@ func (r *RunnersRepo) StartRunnerTask(id, runnerID string, now time.Time) error } // takeRunnerTaskForLease loads a task that must currently be leased by runnerID. -func (r *RunnersRepo) takeRunnerTaskForLease(tx *gorm.DB, id, runnerID string) (runnerTaskRow, bool) { +func (r *RunnersRepo) takeRunnerTaskForLease(tx *gorm.DB, id, runnerID string) (runnerTaskRow, error) { var row runnerTaskRow - if err := r.locking(tx). + err := r.locking(tx). Where("id = ? AND lease_owner = ? AND status IN ?", id, runnerID, []string{domain.RunnerTaskLeased, domain.RunnerTaskRunning}). - Take(&row).Error; err != nil { - return runnerTaskRow{}, false + Take(&row).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return runnerTaskRow{}, ErrRunnerTaskLeaseLost } - return row, true + return row, err } // RenewRunnerTaskLease extends the lease and returns the desired state so a @@ -616,9 +617,9 @@ func (r *RunnersRepo) takeRunnerTaskForLease(tx *gorm.DB, id, runnerID string) ( func (r *RunnersRepo) RenewRunnerTaskLease(id, runnerID string, ttl time.Duration, now time.Time) (string, error) { var desired string err := r.DB.Transaction(func(tx *gorm.DB) error { - row, ok := r.takeRunnerTaskForLease(tx, id, runnerID) - if !ok { - return ErrRunnerTaskLeaseLost + row, err := r.takeRunnerTaskForLease(tx, id, runnerID) + if err != nil { + return err } if err := tx.Model(&runnerTaskRow{}).Where("id = ?", id).Updates(map[string]any{ "lease_until": now.Add(ttl), "updated_at": now, @@ -639,9 +640,9 @@ func (r *RunnersRepo) CompleteRunnerTask(id, runnerID, status string, exitCode * return fmt.Errorf("invalid final task status %q", status) } return r.DB.Transaction(func(tx *gorm.DB) error { - row, ok := r.takeRunnerTaskForLease(tx, id, runnerID) - if !ok { - return ErrRunnerTaskLeaseLost + row, err := r.takeRunnerTaskForLease(tx, id, runnerID) + if err != nil { + return err } task := runnerTaskFromRow(row) task.Status = status