Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 63 additions & 0 deletions internal/repo/runner_lease_read_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
package repo

import (
"errors"
"testing"
"time"

"agentflow/internal/domain"

"gorm.io/gorm"
)

func TestRunnerLeaseOperationsDistinguishDatabaseFailureFromLostLease(t *testing.T) {
st, err := OpenTest()
if err != nil {
t.Fatal(err)
}
t.Cleanup(st.Close)
now := time.Now().UTC()
if err := st.PutRunnerTask(domain.RunnerTask{ID: "runner-read", Provider: "mock"}); err != nil {
t.Fatal(err)
}
task, found, err := st.ClaimRunnerTask("runner-owner", []string{"mock"}, now, time.Minute)
if err != nil || !found {
t.Fatalf("claim found=%t, error=%v", found, err)
}

readError := errors.New("transient lease read failure")
failRead := true
if err := st.DB.Callback().Query().Before("gorm:query").Register("test_runner_read_failure", func(tx *gorm.DB) {
if failRead && tx.Statement.Table == "runner_tasks" {
_ = tx.AddError(readError)
}
}); err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
name string
run func() error
}{
{"start", func() error { return st.StartRunnerTask(task.ID, "runner-owner", now) }},
{"renew", func() error {
_, err := st.RenewRunnerTaskLease(task.ID, "runner-owner", time.Minute, now)
return err
}},
{"complete", func() error {
return st.CompleteRunnerTask(task.ID, "runner-owner", domain.RunnerTaskSuccess, nil, "", "", nil, nil, now)
}},
} {
t.Run(tc.name, func(t *testing.T) {
if err := tc.run(); !errors.Is(err, readError) {
t.Errorf("operation error = %v, want original database failure", err)
}
})
}
failRead = false
if _, err := st.RenewRunnerTaskLease(task.ID, "wrong-owner", time.Minute, now); !errors.Is(err, ErrRunnerTaskLeaseLost) {
t.Fatalf("wrong owner error = %v, want lost lease", err)
}
if _, err := st.RenewRunnerTaskLease(task.ID, "runner-owner", time.Minute, now); err != nil {
t.Fatalf("owner could not renew after recovery: %v", err)
}
}
29 changes: 15 additions & 14 deletions internal/repo/runners.go
Original file line number Diff line number Diff line change
Expand Up @@ -583,9 +583,9 @@ func (r *RunnersRepo) ClaimRunnerTask(runnerID string, providers []string, now t
// StartRunnerTask moves a leased task into running.
func (r *RunnersRepo) StartRunnerTask(id, runnerID string, now time.Time) error {
return r.DB.Transaction(func(tx *gorm.DB) error {
row, ok := r.takeRunnerTaskForLease(tx, id, runnerID)
if !ok {
return ErrRunnerTaskLeaseLost
row, err := r.takeRunnerTaskForLease(tx, id, runnerID)
if err != nil {
return err
}
task := runnerTaskFromRow(row)
task.Status = domain.RunnerTaskRunning
Expand All @@ -601,24 +601,25 @@ func (r *RunnersRepo) StartRunnerTask(id, runnerID string, now time.Time) error
}

// takeRunnerTaskForLease loads a task that must currently be leased by runnerID.
func (r *RunnersRepo) takeRunnerTaskForLease(tx *gorm.DB, id, runnerID string) (runnerTaskRow, bool) {
func (r *RunnersRepo) takeRunnerTaskForLease(tx *gorm.DB, id, runnerID string) (runnerTaskRow, error) {
var row runnerTaskRow
if err := r.locking(tx).
err := r.locking(tx).
Where("id = ? AND lease_owner = ? AND status IN ?", id, runnerID, []string{domain.RunnerTaskLeased, domain.RunnerTaskRunning}).
Take(&row).Error; err != nil {
return runnerTaskRow{}, false
Take(&row).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return runnerTaskRow{}, ErrRunnerTaskLeaseLost
}
return row, true
return row, err
}

// RenewRunnerTaskLease extends the lease and returns the desired state so a
// cancel can travel on the same response without extra polling.
func (r *RunnersRepo) RenewRunnerTaskLease(id, runnerID string, ttl time.Duration, now time.Time) (string, error) {
var desired string
err := r.DB.Transaction(func(tx *gorm.DB) error {
row, ok := r.takeRunnerTaskForLease(tx, id, runnerID)
if !ok {
return ErrRunnerTaskLeaseLost
row, err := r.takeRunnerTaskForLease(tx, id, runnerID)
if err != nil {
return err
}
if err := tx.Model(&runnerTaskRow{}).Where("id = ?", id).Updates(map[string]any{
"lease_until": now.Add(ttl), "updated_at": now,
Expand All @@ -639,9 +640,9 @@ func (r *RunnersRepo) CompleteRunnerTask(id, runnerID, status string, exitCode *
return fmt.Errorf("invalid final task status %q", status)
}
return r.DB.Transaction(func(tx *gorm.DB) error {
row, ok := r.takeRunnerTaskForLease(tx, id, runnerID)
if !ok {
return ErrRunnerTaskLeaseLost
row, err := r.takeRunnerTaskForLease(tx, id, runnerID)
if err != nil {
return err
}
task := runnerTaskFromRow(row)
task.Status = status
Expand Down